Repository: pingcap/tidb (Go)

Issue:
lightning import partition tables failed due to "non-retryable error: read index not ready, reason can not read index due to split"
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)
1. Use lightning to import test data of 1.5K partition table 

### 2. What did you expect to see? (Required)
Lightning import should succeed

### 3. What did you see instead (Required)
Lightning import failed due to "non-retryable error: read index not ready, reason can not read index due to split, region 64037"
```
[2022/07/25 04:52:58.753 +00:00] [INFO] [table_restore.go:1015] ["analyze completed"] [table=`new_data`.`PK_LCP9284`] [takeTime=2.401747315s] []
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:1494] ["restore all tables data failed"] [takeTime=55m14.0129788s] [error="[Lightning:Restore:ErrRestoreTable]restore table `new_data`.`UK_LP17285` failed: non-retryable error: read index not ready, reason can not read index due to split, region 64037"]
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:456] ["run failed"] [step=4] [error="[Lightning:Restore:ErrRestoreTable]restore table `new_data`.`UK_LP17285` failed: non-retryable error: read index not ready, reason can not read index due to split, region 64037"]
[2022/07/25 04:52:58.758 +00:00] [INFO] [restore.go:1137] ["everything imported, stopping periodic actions"]
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:466] ["the whole procedure failed"] [takeTime=57m20.978459058s] [error="[Lightning:Restore:ErrRestoreTable]restore table `new_data`.`UK_LP17285` failed: non-retryable error: read index not ready, reason can not read index due to split, region 64037"]
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:167] ["tables failed to be imported"] [count=2]
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:169] [-] [table=`new_data`.`UK_LP17285`] [status=imported] [error="non-retryable error: read index not ready, reason can not read index due to split, region 64037"]
[2022/07/25 04:52:58.758 +00:00] [ERROR] [restore.go:169] [-] [table=`new_data`.`IDT_LP24677`] [status=imported] [error="non-retryable error: read index not ready, reason can not read index due to split, region 71138"]
[2022/07/25 04:52:58.763 +00:00] [INFO] [checksum.go:459] ["service safe point keeper exited"]
[2022/07/25 04:52:58.763 +00:00] [ERROR] [main.go:103] ["tidb lightning encountered error stack info"] [error="[Lightning:Restore:ErrRestoreTable]restore table `new_data`.`UK_LP17285` failed: non-retryable error: read index not ready, reason can not read index due to split, region 64037"]
```

### 4. What is your TiDB version? (Required)
[2022/07/25 03:55:37.272 +00:00] [INFO] [info.go:49] ["Welcome to TiDB-Lightning"] [release-version=v6.2.0] [git-hash=0ea937e56adac89ea7df1ce5406c7dc20a3f83ae] [git-branch=heads/refs/tags/v6.2.0] [go-version=go1.18.2] [utc-build-time="2022-07-23 10:05:04"] [race-enabled=false]




Discussion:

Patch:
diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go
index b0a807f2d7b76..ce9987a5a473c 100644
--- a/br/pkg/lightning/backend/local/local.go
+++ b/br/pkg/lightning/backend/local/local.go
@@ -1926,6 +1926,18 @@ func (local *local) isIngestRetryable(
 		return retryNone, nil, common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
 		return retryNone, nil, common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
+	case errPb.ReadIndexNotReady != nil:
+		// this error happens when this region is splitting, the error might be:
+		//   read index not ready, reason can not read index due to split, region 64037
+		// we have paused schedule, but it's temporary,
+		// if next request takes a long time, there's chance schedule is enabled again
+		// or on key range border, another engine sharing this region tries to split this
+		// region may cause this error too.
+		newRegion, err = getRegion()
+		if err != nil {
+			return retryNone, nil, errors.Trace(err)
+		}
+		return retryWrite, newRegion, common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	}
 	return retryNone, nil, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
 }
diff --git a/br/pkg/lightning/common/errors.go b/br/pkg/lightning/common/errors.go
index ccb465ec4b86a..4f1c598a84535 100644
--- a/br/pkg/lightning/common/errors.go
+++ b/br/pkg/lightning/common/errors.go
@@ -70,13 +70,14 @@ var (
 	ErrCreatePDClient = errors.Normalize("create pd client error", errors.RFCCodeText("Lightning:PD:ErrCreatePDClient"))
 	ErrPauseGC        = errors.Normalize("pause gc error", errors.RFCCodeText("Lightning:PD:ErrPauseGC"))
 
-	ErrCheckKVVersion   = errors.Normalize("check tikv version error", errors.RFCCodeText("Lightning:KV:ErrCheckKVVersion"))
-	ErrCreateKVClient   = errors.Normalize("create kv client error", errors.RFCCodeText("Lightning:KV:ErrCreateKVClient"))
-	ErrCheckMultiIngest = errors.Normalize("check multi-ingest support error", errors.RFCCodeText("Lightning:KV:ErrCheckMultiIngest"))
-	ErrKVEpochNotMatch  = errors.Normalize("epoch not match", errors.RFCCodeText("Lightning:KV:EpochNotMatch"))
-	ErrKVNotLeader      = errors.Normalize("not leader", errors.RFCCodeText("Lightning:KV:NotLeader"))
-	ErrKVServerIsBusy   = errors.Normalize("server is busy", errors.RFCCodeText("Lightning:KV:ServerIsBusy"))
-	ErrKVRegionNotFound = errors.Normalize("region not found", errors.RFCCodeText("Lightning:KV:RegionNotFound"))
+	ErrCheckKVVersion      = errors.Normalize("check tikv version error", errors.RFCCodeText("Lightning:KV:ErrCheckKVVersion"))
+	ErrCreateKVClient      = errors.Normalize("create kv client error", errors.RFCCodeText("Lightning:KV:ErrCreateKVClient"))
+	ErrCheckMultiIngest    = errors.Normalize("check multi-ingest support error", errors.RFCCodeText("Lightning:KV:ErrCheckMultiIngest"))
+	ErrKVEpochNotMatch     = errors.Normalize("epoch not match", errors.RFCCodeText("Lightning:KV:EpochNotMatch"))
+	ErrKVNotLeader         = errors.Normalize("not leader", errors.RFCCodeText("Lightning:KV:NotLeader"))
+	ErrKVServerIsBusy      = errors.Normalize("server is busy", errors.RFCCodeText("Lightning:KV:ServerIsBusy"))
+	ErrKVRegionNotFound    = errors.Normalize("region not found", errors.RFCCodeText("Lightning:KV:RegionNotFound"))
+	ErrKVReadIndexNotReady = errors.Normalize("read index not ready", errors.RFCCodeText("Lightning:KV:ReadIndexNotReady"))
 
 	ErrUnknownBackend     = errors.Normalize("unknown backend %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownBackend"))
 	ErrCheckLocalFile     = errors.Normalize("cannot find local file for table: %s engineDir: %s", errors.RFCCodeText("Lightning:Restore:ErrCheckLocalFile"))
diff --git a/errors.toml b/errors.toml
index 5043c913d0147..f47c968f0cc0e 100755
--- a/errors.toml
+++ b/errors.toml
@@ -386,6 +386,11 @@ error = '''
 not leader
 '''
 
+["Lightning:KV:ReadIndexNotReady"]
+error = '''
+read index not ready
+'''
+
 ["Lightning:KV:RegionNotFound"]
 error = '''
 region not found


Test patch:
diff --git a/br/pkg/lightning/backend/local/local_test.go b/br/pkg/lightning/backend/local/local_test.go
index 45030633b49c9..087c834b6cfde 100644
--- a/br/pkg/lightning/backend/local/local_test.go
+++ b/br/pkg/lightning/backend/local/local_test.go
@@ -512,6 +512,15 @@ func TestIsIngestRetryable(t *testing.T) {
 	retryType, _, err = local.isIngestRetryable(ctx, resp, region, metas)
 	require.Equal(t, retryNone, retryType)
 	require.EqualError(t, err, "non-retryable error: unknown error")
+
+	resp.Error = &errorpb.Error{
+		ReadIndexNotReady: &errorpb.ReadIndexNotReady{
+			Reason: "test",
+		},
+	}
+	retryType, _, err = local.isIngestRetryable(ctx, resp, region, metas)
+	require.Equal(t, retryWrite, retryType)
+	require.Error(t, err)
 }
 
 type testIngester struct{}