Repository: pingcap/tidb-lightning
Pull request #524 (2020-12-14 11:17:11)

backend/local: skip split regions if engine total size is smaller than region size
<!--
Thank you for contributing to TiDB-Lightning! Please read the [CONTRIBUTING](https://github.com/pingcap/tidb-lightning/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->
Skip split regions there is only one range within the engine.

### What is changed and how it works?


### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test
 - Manual test (add detailed scripts or steps below)
 - No code

Side effects

Related changes


kennytm: /run-all-tests
lance6716: LGTM
glorv: /merge
ti-srebot: /run-all-tests

Patch:
diff --git a/lightning/backend/local.go b/lightning/backend/local.go
index 83170b6..1607a6b 100644
--- a/lightning/backend/local.go
+++ b/lightning/backend/local.go
@@ -1190,10 +1190,14 @@ func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID) erro
 		log.L().Info("start import engine", zap.Stringer("uuid", engineUUID),
 			zap.Int("ranges", len(ranges)))
 
+		// if all the kv can fit in one region, skip split regions. TiDB will split one region for
+		// the table when table is created.
+		needSplit := len(ranges) > 1 || lf.TotalSize > local.regionSplitSize || lf.Length > regionMaxKeyCount
+
 		// split region by given ranges
 		for i := 0; i < maxRetryTimes; i++ {
-			err = local.SplitAndScatterRegionByRanges(ctx, ranges)
-			if err == nil || errors.Cause(err) == context.Canceled {
+			err = local.SplitAndScatterRegionByRanges(ctx, ranges, needSplit)
+			if err == nil || common.IsContextCanceledError(err) {
 				break
 			}
 
@@ -1220,7 +1224,8 @@ func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID) erro
 		ranges = unfinishedRanges
 	}
 
-	log.L().Info("import engine success", zap.Stringer("uuid", engineUUID))
+	log.L().Info("import engine success", zap.Stringer("uuid", engineUUID),
+		zap.Int64("size", lf.TotalSize), zap.Int64("kvs", lf.Length))
 	return nil
 }
 
diff --git a/lightning/backend/localhelper.go b/lightning/backend/localhelper.go
index 07cefad..a90d53d 100644
--- a/lightning/backend/localhelper.go
+++ b/lightning/backend/localhelper.go
@@ -50,8 +50,8 @@ var (
 // TODO remove this file and use br internal functions
 // This File include region split & scatter operation just like br.
 // we can simply call br function, but we need to change some function signature of br
-
-func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []Range) error {
+// When the ranges total size is small, we can skip the split to avoid generate empty regions.
+func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []Range, needSplit bool) error {
 	if len(ranges) == 0 {
 		return nil
 	}
@@ -97,6 +97,11 @@ func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []
 		log.L().Info("paginate scan region finished", log.ZapRedactBinary("minKey", minKey), log.ZapRedactBinary("maxKey", maxKey),
 			zap.Int("regions", len(regions)))
 
+		if !needSplit {
+			scatterRegions = append(scatterRegions, regions...)
+			break
+		}
+
 		regionMap := make(map[uint64]*split.RegionInfo)
 		for _, region := range regions {
 			regionMap[region.Region.GetId()] = region


Test patch:
diff --git a/lightning/backend/localhelper_test.go b/lightning/backend/localhelper_test.go
index 50d1b96..c8bddbb 100644
--- a/lightning/backend/localhelper_test.go
+++ b/lightning/backend/localhelper_test.go
@@ -363,7 +363,7 @@ func (s *localSuite) doTestBatchSplitRegionByRanges(c *C, hook clientHook, errPa
 		start = end
 	}
 
-	err = local.SplitAndScatterRegionByRanges(ctx, ranges)
+	err = local.SplitAndScatterRegionByRanges(ctx, ranges, true)
 	if len(errPat) == 0 {
 		c.Assert(err, IsNil)
 	} else {
@@ -504,7 +504,7 @@ func (s *localSuite) doTestBatchSplitByRangesWithClusteredIndex(c *C, hook clien
 		start = e
 	}
 
-	err := local.SplitAndScatterRegionByRanges(ctx, ranges)
+	err := local.SplitAndScatterRegionByRanges(ctx, ranges, true)
 	c.Assert(err, IsNil)
 
 	startKey := codec.EncodeBytes([]byte{}, rangeKeys[0])