Repository: pingcap/tidb-lightning
Pull request #428 (2020-10-26 09:30:56)

backend/tidb: add rebase auto id for tidb backend
<!--
Thank you for contributing to TiDB-Lightning! Please read the [CONTRIBUTING](https://github.com/pingcap/tidb-lightning/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->
Currently, tidb may not rebase auto id properly when use insert rows with auto id column manually. So we do rebase auto id in tidb backend to work around this issue.

Also fix the issue that tidb backend need to communicate to pd-server

### What is changed and how it works?
- Fetch table auto id info by `show table next_row_id` in tidb backend `FetchRemoteTableModels `
- Also rebase auto id for tidb backend

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test
 - Manual test (add detailed scripts or steps below)
 - No code

Side effects

Related changes


kennytm: > Currently, tidb may not rebase auto id properly when use insert rows with auto id column manually. 

is there a TiDB issue?
glorv: > > Currently, tidb may not rebase auto id properly when use insert rows with auto id column manually.
> 
> is there a TiDB issue?

See [Issue](https://internal.pingcap.net/jira/browse/PRM-173). Since there is no easy way to fix this issue easily in tidb side,  we rebase  auto id for tidb backend in lightning side as a workaround.
kennytm: (the TiDB issue is pingcap/tidb#2285.)
kennytm: `alter_random` test failed.

```
[2020-10-26T13:07:33.079Z] column perms: [0], auto inc idx: -1, auto rand idx: 0

[2020-10-26T13:07:33.335Z] Error: restore table `alter_random`.`t` failed: ALTER TABLE `alter_random`.`t` AUTO_RANDOM_BASE=8646911284551352323: alter table auto_random_base failed: Error 8216: Invalid auto random: alter auto_random_base to 8646911284551352323 overflows the incremental bits, max allowed base is 288230376151711743
```
kennytm: (please run a test on the release-3.0 cluster before merging.)
glorv: /run-all-tests tidb=release-3.0 tikv=release-3.0 pd=release-3.0 importer=release-3.0
glorv: /run-all-tests tidb=release-3.0 tikv=release-3.0 pd=release-3.0 importer=release-3.0
glorv: /run-all-tests tidb=release-3.0 tikv=release-3.0 pd=release-3.0 importer=release-3.0


glorv: /run-all-tests tidb=release-3.0 tikv=release-3.0 pd=release-3.0 importer=release-3.0
kennytm: /run-all-tests

(3.0 passed in https://internal.pingcap.net/idc-jenkins/job/lightning_ghpr_test/2363/display/redirect)
jebter: /merge

Patch:
diff --git a/lightning/backend/backend.go b/lightning/backend/backend.go
index 439b0d9..29d7a79 100644
--- a/lightning/backend/backend.go
+++ b/lightning/backend/backend.go
@@ -100,8 +100,7 @@ type AbstractBackend interface {
 	MaxChunkSize() int
 
 	// ShouldPostProcess returns whether KV-specific post-processing should be
-	// performed for this backend. Post-processing includes checksum, adjusting
-	// auto-increment ID, and analyze.
+	// performed for this backend. Post-processing includes checksum and analyze.
 	ShouldPostProcess() bool
 
 	// NewEncoder creates an encoder of a TiDB table.
@@ -140,7 +139,7 @@ type AbstractBackend interface {
 	//     * State (must be model.StatePublic)
 	//     * Offset (must be 0, 1, 2, ...)
 	//  - PKIsHandle (true = do not generate _tidb_rowid)
-	FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, error)
+	FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error)
 }
 
 func fetchRemoteTableModelsFromTLS(tls *common.TLS, schema string) ([]*model.TableInfo, error) {
@@ -209,8 +208,8 @@ func (be Backend) CheckRequirements() error {
 	return be.abstract.CheckRequirements()
 }
 
-func (be Backend) FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, error) {
-	return be.abstract.FetchRemoteTableModels(schemaName)
+func (be Backend) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) {
+	return be.abstract.FetchRemoteTableModels(ctx, schemaName)
 }
 
 // OpenEngine opens an engine with the given table name and engine ID.
diff --git a/lightning/backend/importer.go b/lightning/backend/importer.go
index 9f05999..afa1260 100644
--- a/lightning/backend/importer.go
+++ b/lightning/backend/importer.go
@@ -253,30 +253,6 @@ func (importer *importer) CheckRequirements() error {
 	return nil
 }
 
-func extractTiDBVersion(version string) (*semver.Version, error) {
-	// version format: "5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f"
-	//                               ^~~~~~~~~^ we only want this part
-	// version format: "5.7.10-TiDB-v2.0.4-1-g06a0bf5"
-	//                               ^~~~^
-	// version format: "5.7.10-TiDB-v2.0.7"
-	//                               ^~~~^
-	// version format: "5.7.25-TiDB-v3.0.0-beta-211-g09beefbe0-dirty"
-	//                               ^~~~~~~~~^
-	// The version is generated by `git describe --tags` on the TiDB repository.
-	versions := strings.Split(strings.TrimSuffix(version, "-dirty"), "-")
-	end := len(versions)
-	switch end {
-	case 3, 4:
-	case 5, 6:
-		end -= 2
-	default:
-		return nil, errors.Errorf("not a valid TiDB version: %s", version)
-	}
-	rawVersion := strings.Join(versions[2:end], "-")
-	rawVersion = strings.TrimPrefix(rawVersion, "v")
-	return semver.NewVersion(rawVersion)
-}
-
 func checkTiDBVersion(tls *common.TLS, requiredVersion semver.Version) error {
 	var status struct{ Version string }
 	err := tls.GetJSON("/status", &status)
@@ -284,7 +260,7 @@ func checkTiDBVersion(tls *common.TLS, requiredVersion semver.Version) error {
 		return err
 	}
 
-	version, err := extractTiDBVersion(status.Version)
+	version, err := common.ExtractTiDBVersion(status.Version)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -328,6 +304,6 @@ func checkVersion(component string, expected, actual semver.Version) error {
 	)
 }
 
-func (importer *importer) FetchRemoteTableModels(schema string) ([]*model.TableInfo, error) {
+func (importer *importer) FetchRemoteTableModels(ctx context.Context, schema string) ([]*model.TableInfo, error) {
 	return fetchRemoteTableModelsFromTLS(importer.tls, schema)
 }
diff --git a/lightning/backend/local.go b/lightning/backend/local.go
index 90fd379..76e1790 100644
--- a/lightning/backend/local.go
+++ b/lightning/backend/local.go
@@ -1165,7 +1165,7 @@ func (local *local) CheckRequirements() error {
 	return nil
 }
 
-func (local *local) FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, error) {
+func (local *local) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) {
 	return fetchRemoteTableModelsFromTLS(local.tls, schemaName)
 }
 
diff --git a/lightning/backend/tidb.go b/lightning/backend/tidb.go
index dbbcf1b..448bdad 100644
--- a/lightning/backend/tidb.go
+++ b/lightning/backend/tidb.go
@@ -17,6 +17,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/hex"
+	"fmt"
 	"strconv"
 	"strings"
 	"time"
@@ -279,9 +280,8 @@ func (be *tidbBackend) CheckRequirements() error {
 }
 
 func (be *tidbBackend) NewEncoder(tbl table.Table, options *SessionOptions) Encoder {
-	var se *session
+	se := newSession(options)
 	if options.SQLMode.HasStrictMode() {
-		se = newSession(options)
 		se.vars.SkipUTF8Check = false
 		se.vars.SkipASCIICheck = false
 	}
@@ -356,14 +356,24 @@ func (be *tidbBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName str
 	return errors.Trace(err)
 }
 
-func (be *tidbBackend) FetchRemoteTableModels(schemaName string) (tables []*model.TableInfo, err error) {
+func (be *tidbBackend) FetchRemoteTableModels(ctx context.Context, schemaName string) (tables []*model.TableInfo, err error) {
 	s := common.SQLWithRetry{
 		DB:     be.db,
 		Logger: log.L(),
 	}
-	err = s.Transact(context.Background(), "fetch table columns", func(c context.Context, tx *sql.Tx) error {
+
+	err = s.Transact(ctx, "fetch table columns", func(c context.Context, tx *sql.Tx) error {
+		var versionStr string
+		if err = tx.QueryRowContext(ctx, "SELECT version()").Scan(&versionStr); err != nil {
+			return err
+		}
+		tidbVersion, err := common.ExtractTiDBVersion(versionStr)
+		if err != nil {
+			return err
+		}
+
 		rows, e := tx.Query(`
-			SELECT table_name, column_name
+			SELECT table_name, column_name, column_type, extra
 			FROM information_schema.columns
 			WHERE table_schema = ?
 			ORDER BY table_name, ordinal_position;
@@ -379,8 +389,8 @@ func (be *tidbBackend) FetchRemoteTableModels(schemaName string) (tables []*mode
 			curTable     *model.TableInfo
 		)
 		for rows.Next() {
-			var tableName, columnName string
-			if e := rows.Scan(&tableName, &columnName); e != nil {
+			var tableName, columnName, columnType, columnExtra string
+			if e := rows.Scan(&tableName, &columnName, &columnType, &columnExtra); e != nil {
 				return e
 			}
 			if tableName != curTableName {
@@ -394,14 +404,67 @@ func (be *tidbBackend) FetchRemoteTableModels(schemaName string) (tables []*mode
 				curColOffset = 0
 			}
 
+			// see: https://github.com/pingcap/parser/blob/3b2fb4b41d73710bc6c4e1f4e8679d8be6a4863e/types/field_type.go#L185-L191
+			var flag uint
+			if strings.HasSuffix(columnType, "unsigned") {
+				flag |= mysql.UnsignedFlag
+			}
+			if strings.Contains(columnExtra, "auto_increment") {
+				flag |= mysql.AutoIncrementFlag
+			}
 			curTable.Columns = append(curTable.Columns, &model.ColumnInfo{
 				Name:   model.NewCIStr(columnName),
 				Offset: curColOffset,
 				State:  model.StatePublic,
+				FieldType: types.FieldType{
+					Flag: flag,
+				},
 			})
 			curColOffset++
 		}
-		return rows.Err()
+		if rows.Err() != nil {
+			return rows.Err()
+		}
+		// for version < v4.0.0 we can use `show table next_row_id` to fetch auto id info, so about should be enough
+		if tidbVersion.Major < 4 {
+			return nil
+		}
+		// init auto id column for each table
+		for _, tbl := range tables {
+			tblName := common.UniqueTable(schemaName, tbl.Name.O)
+			rows, e = tx.Query(fmt.Sprintf("SHOW TABLE %s next_row_id", tblName))
+			if e != nil {
+				return e
+			}
+			for rows.Next() {
+				var (
+					dbName, tblName, columnName, idType string
+					nextID                              int64
+				)
+				if e := rows.Scan(&dbName, &tblName, &columnName, &nextID, &idType); e != nil {
+					_ = rows.Close()
+					return e
+				}
+				for _, col := range tbl.Columns {
+					if col.Name.O == columnName {
+						switch idType {
+						case "AUTO_INCREMENT":
+							col.Flag |= mysql.AutoIncrementFlag
+						case "AUTO_RANDOM":
+							col.Flag |= mysql.PriKeyFlag
+							tbl.PKIsHandle = true
+							// set a stub here, since we don't really need the real value
+							tbl.AutoRandomBits = 1
+						}
+					}
+				}
+			}
+			rows.Close()
+			if rows.Err() != nil {
+				return rows.Err()
+			}
+		}
+		return nil
 	})
 	return
 }
diff --git a/lightning/common/version.go b/lightning/common/version.go
index fad832e..f80ed78 100644
--- a/lightning/common/version.go
+++ b/lightning/common/version.go
@@ -15,6 +15,7 @@ package common
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/pingcap/errors"
@@ -71,3 +72,27 @@ func FetchPDVersion(tls *TLS, pdAddr string) (*semver.Version, error) {
 
 	return semver.NewVersion(rawVersion)
 }
+
+func ExtractTiDBVersion(version string) (*semver.Version, error) {
+	// version format: "5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f"
+	//                               ^~~~~~~~~^ we only want this part
+	// version format: "5.7.10-TiDB-v2.0.4-1-g06a0bf5"
+	//                               ^~~~^
+	// version format: "5.7.10-TiDB-v2.0.7"
+	//                               ^~~~^
+	// version format: "5.7.25-TiDB-v3.0.0-beta-211-g09beefbe0-dirty"
+	//                               ^~~~~~~~~^
+	// The version is generated by `git describe --tags` on the TiDB repository.
+	versions := strings.Split(strings.TrimSuffix(version, "-dirty"), "-")
+	end := len(versions)
+	switch end {
+	case 3, 4:
+	case 5, 6:
+		end -= 2
+	default:
+		return nil, errors.Errorf("not a valid TiDB version: %s", version)
+	}
+	rawVersion := strings.Join(versions[2:end], "-")
+	rawVersion = strings.TrimPrefix(rawVersion, "v")
+	return semver.NewVersion(rawVersion)
+}
diff --git a/lightning/restore/checksum.go b/lightning/restore/checksum.go
index 466e359..ce07032 100644
--- a/lightning/restore/checksum.go
+++ b/lightning/restore/checksum.go
@@ -45,7 +45,7 @@ type ChecksumManager interface {
 
 func newChecksumManager(rc *RestoreController) (ChecksumManager, error) {
 	// if we don't need checksum, just return nil
-	if rc.cfg.PostRestore.Checksum == config.OpLevelOff {
+	if rc.cfg.TikvImporter.Backend == config.BackendTiDB || rc.cfg.PostRestore.Checksum == config.OpLevelOff {
 		return nil, nil
 	}
 
diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go
index cbd6335..d8d172a 100644
--- a/lightning/restore/restore.go
+++ b/lightning/restore/restore.go
@@ -639,9 +639,11 @@ var checksumManagerKey struct{}
 func (rc *RestoreController) restoreTables(ctx context.Context) error {
 	logTask := log.L().Begin(zap.InfoLevel, "restore all tables data")
 
-	// for importer/local backend, we should disable some pd scheduler and change some settings, to
+	// for local backend, we should disable some pd scheduler and change some settings, to
 	// make split region and ingest sst more stable
-	if rc.cfg.TikvImporter.Backend != config.BackendTiDB {
+	// because importer backend is mostly use for v3.x cluster which doesn't support these api,
+	// so we also don't do this for import backend
+	if rc.cfg.TikvImporter.Backend == config.BackendLocal {
 		// disable some pd schedulers
 		pdController, err := pdutil.NewPdController(ctx, rc.cfg.TiDB.PdAddr,
 			rc.tls.TLSConfig(), rc.tls.ToPDSecurityOption())
@@ -1159,12 +1161,6 @@ func (t *TableRestore) importEngine(
 }
 
 func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint) error {
-	if !rc.backend.ShouldPostProcess() {
-		t.logger.Debug("skip post-processing, not supported by backend")
-		rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped)
-		return nil
-	}
-
 	// 3. alter table set auto_increment
 	if cp.Status < CheckpointStatusAlteredAutoInc {
 		rc.alterTableLock.Lock()
@@ -1183,6 +1179,13 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c
 		}
 	}
 
+	// tidb backend don't need checksum & analyze
+	if !rc.backend.ShouldPostProcess() {
+		t.logger.Debug("skip checksum & analyze, not supported by this backend")
+		rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped)
+		return nil
+	}
+
 	// 4. do table checksum
 	var localChecksum verify.KVChecksum
 	for _, engine := range cp.Engines {
diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go
index 41f656b..9f14934 100644
--- a/lightning/restore/tidb.go
+++ b/lightning/restore/tidb.go
@@ -195,11 +195,11 @@ func (timgr *TiDBManager) DropTable(ctx context.Context, tableName string) error
 func (timgr *TiDBManager) LoadSchemaInfo(
 	ctx context.Context,
 	schemas []*mydump.MDDatabaseMeta,
-	getTables func(string) ([]*model.TableInfo, error),
+	getTables func(context.Context, string) ([]*model.TableInfo, error),
 ) (map[string]*TidbDBInfo, error) {
 	result := make(map[string]*TidbDBInfo, len(schemas))
 	for _, schema := range schemas {
-		tables, err := getTables(schema.Name)
+		tables, err := getTables(ctx, schema.Name)
 		if err != nil {
 			return nil, err
 		}
@@ -279,6 +279,11 @@ func ObtainNewCollationEnabled(ctx context.Context, db *sql.DB) bool {
 	return newCollationEnabled
 }
 
+// AlterAutoIncrement rebase the table auto increment id
+//
+// NOTE: since tidb can make sure the auto id is always be rebase even if the `incr` value is smaller
+// the the auto incremanet base in tidb side, we needn't fetch currently auto increment value here.
+// See: https://github.com/pingcap/tidb/blob/64698ef9a3358bfd0fdc323996bb7928a56cadca/ddl/ddl_api.go#L2528-L2533
 func AlterAutoIncrement(ctx context.Context, db *sql.DB, tableName string, incr int64) error {
 	sql := common.SQLWithRetry{
 		DB:     db,
diff --git a/mock/backend.go b/mock/backend.go
index a4115ac..273cf32 100644
--- a/mock/backend.go
+++ b/mock/backend.go
@@ -100,18 +100,18 @@ func (mr *MockBackendMockRecorder) CloseEngine(arg0, arg1 interface{}) *gomock.C
 }
 
 // FetchRemoteTableModels mocks base method
-func (m *MockBackend) FetchRemoteTableModels(arg0 string) ([]*model.TableInfo, error) {
+func (m *MockBackend) FetchRemoteTableModels(arg0 context.Context, arg1 string) ([]*model.TableInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "FetchRemoteTableModels", arg0)
+	ret := m.ctrl.Call(m, "FetchRemoteTableModels", arg0, arg1)
 	ret0, _ := ret[0].([]*model.TableInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // FetchRemoteTableModels indicates an expected call of FetchRemoteTableModels
-func (mr *MockBackendMockRecorder) FetchRemoteTableModels(arg0 interface{}) *gomock.Call {
+func (mr *MockBackendMockRecorder) FetchRemoteTableModels(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchRemoteTableModels", reflect.TypeOf((*MockBackend)(nil).FetchRemoteTableModels), arg0)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchRemoteTableModels", reflect.TypeOf((*MockBackend)(nil).FetchRemoteTableModels), arg0, arg1)
 }
 
 // ImportEngine mocks base method


Test patch:
diff --git a/lightning/backend/checkreq_test.go b/lightning/backend/checkreq_test.go
index 1a8d710..53a667b 100644
--- a/lightning/backend/checkreq_test.go
+++ b/lightning/backend/checkreq_test.go
@@ -30,49 +30,6 @@ var _ = Suite(&checkReqSuite{})
 
 type checkReqSuite struct{}
 
-func (s *checkReqSuite) TestExtractTiDBVersion(c *C) {
-	vers, err := extractTiDBVersion("5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("2.1.0-rc.1"))
-
-	vers, err = extractTiDBVersion("5.7.10-TiDB-v2.0.4-1-g06a0bf5")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("2.0.4"))
-
-	vers, err = extractTiDBVersion("5.7.10-TiDB-v2.0.7")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("2.0.7"))
-
-	vers, err = extractTiDBVersion("8.0.12-TiDB-v3.0.5-beta.12")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("3.0.5-beta.12"))
-
-	vers, err = extractTiDBVersion("5.7.25-TiDB-v3.0.0-beta-211-g09beefbe0-dirty")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("3.0.0-beta"))
-
-	vers, err = extractTiDBVersion("8.0.12-TiDB-v3.0.5-dirty")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("3.0.5"))
-
-	vers, err = extractTiDBVersion("8.0.12-TiDB-v3.0.5-beta.12-dirty")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("3.0.5-beta.12"))
-
-	vers, err = extractTiDBVersion("5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f-dirty")
-	c.Assert(err, IsNil)
-	c.Assert(*vers, Equals, *semver.New("2.1.0-rc.1"))
-
-	_, err = extractTiDBVersion("")
-	c.Assert(err, ErrorMatches, "not a valid TiDB version.*")
-
-	_, err = extractTiDBVersion("8.0.12")
-	c.Assert(err, ErrorMatches, "not a valid TiDB version.*")
-
-	_, err = extractTiDBVersion("not-a-valid-version")
-	c.Assert(err, NotNil)
-}
-
 func (s *checkReqSuite) TestCheckVersion(c *C) {
 	err := checkVersion("TiNB", *semver.New("2.1.0"), *semver.New("2.3.5"))
 	c.Assert(err, IsNil)
diff --git a/lightning/backend/tidb_test.go b/lightning/backend/tidb_test.go
index 977fc8c..482a0bc 100644
--- a/lightning/backend/tidb_test.go
+++ b/lightning/backend/tidb_test.go
@@ -86,10 +86,11 @@ func (s *mysqlSuite) TestWriteRowsReplaceOnDup(c *C) {
 	indexChecksum := verification.MakeKVChecksum(0, 0, 0)
 
 	cols := s.tbl.Cols()
-	perms := make([]int, 0, len(s.tbl.Cols()))
+	perms := make([]int, 0, len(s.tbl.Cols())+1)
 	for i := 0; i < len(cols); i++ {
 		perms = append(perms, i)
 	}
+	perms = append(perms, -1)
 	encoder := s.backend.NewEncoder(s.tbl, &kv.SessionOptions{SQLMode: 0, Timestamp: 1234567890, RowFormatVersion: "1"})
 	row, err := encoder.Encode(logger, []types.Datum{
 		types.NewUintDatum(18446744073709551615),
@@ -133,7 +134,7 @@ func (s *mysqlSuite) TestWriteRowsIgnoreOnDup(c *C) {
 	encoder := ignoreBackend.NewEncoder(s.tbl, &kv.SessionOptions{})
 	row, err := encoder.Encode(logger, []types.Datum{
 		types.NewIntDatum(1),
-	}, 1, []int{0})
+	}, 1, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1})
 	c.Assert(err, IsNil)
 	row.ClassifyAndAppend(&dataRows, &dataChecksum, &indexRows, &indexChecksum)
 
@@ -161,7 +162,7 @@ func (s *mysqlSuite) TestWriteRowsErrorOnDup(c *C) {
 	encoder := ignoreBackend.NewEncoder(s.tbl, &kv.SessionOptions{})
 	row, err := encoder.Encode(logger, []types.Datum{
 		types.NewIntDatum(1),
-	}, 1, []int{0})
+	}, 1, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1})
 	c.Assert(err, IsNil)
 
 	row.ClassifyAndAppend(&dataRows, &dataChecksum, &indexRows, &indexChecksum)
@@ -187,16 +188,16 @@ func (s *mysqlSuite) TestStrictMode(c *C) {
 	logger := log.L()
 	_, err = encoder.Encode(logger, []types.Datum{
 		types.NewStringDatum("test"),
-	}, 1, []int{0})
+	}, 1, []int{0, 1, -1})
 	c.Assert(err, IsNil)
 
 	_, err = encoder.Encode(logger, []types.Datum{
 		types.NewStringDatum("\xff\xff\xff\xff"),
-	}, 1, []int{0})
+	}, 1, []int{0, 1, -1})
 	c.Assert(err, ErrorMatches, `.*incorrect utf8 value .* for column s0`)
 
 	_, err = encoder.Encode(logger, []types.Datum{
 		types.NewStringDatum("非 ASCII 字符串"),
-	}, 1, []int{1})
+	}, 1, []int{1, 0, -1})
 	c.Assert(err, ErrorMatches, ".*incorrect ascii value .* for column s1")
 }
diff --git a/lightning/common/version_test.go b/lightning/common/version_test.go
index f02e821..cd75b87 100644
--- a/lightning/common/version_test.go
+++ b/lightning/common/version_test.go
@@ -14,6 +14,7 @@
 package common_test
 
 import (
+	"github.com/coreos/go-semver/semver"
 	. "github.com/pingcap/check"
 
 	"github.com/pingcap/tidb-lightning/lightning/common"
@@ -49,3 +50,46 @@ UTC Build Time: None
 Go Version: None
 `)
 }
+
+func (s *utilSuite) TestExtractTiDBVersion(c *C) {
+	vers, err := common.ExtractTiDBVersion("5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("2.1.0-rc.1"))
+
+	vers, err = common.ExtractTiDBVersion("5.7.10-TiDB-v2.0.4-1-g06a0bf5")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("2.0.4"))
+
+	vers, err = common.ExtractTiDBVersion("5.7.10-TiDB-v2.0.7")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("2.0.7"))
+
+	vers, err = common.ExtractTiDBVersion("8.0.12-TiDB-v3.0.5-beta.12")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("3.0.5-beta.12"))
+
+	vers, err = common.ExtractTiDBVersion("5.7.25-TiDB-v3.0.0-beta-211-g09beefbe0-dirty")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("3.0.0-beta"))
+
+	vers, err = common.ExtractTiDBVersion("8.0.12-TiDB-v3.0.5-dirty")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("3.0.5"))
+
+	vers, err = common.ExtractTiDBVersion("8.0.12-TiDB-v3.0.5-beta.12-dirty")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("3.0.5-beta.12"))
+
+	vers, err = common.ExtractTiDBVersion("5.7.10-TiDB-v2.1.0-rc.1-7-g38c939f-dirty")
+	c.Assert(err, IsNil)
+	c.Assert(*vers, Equals, *semver.New("2.1.0-rc.1"))
+
+	_, err = common.ExtractTiDBVersion("")
+	c.Assert(err, ErrorMatches, "not a valid TiDB version.*")
+
+	_, err = common.ExtractTiDBVersion("8.0.12")
+	c.Assert(err, ErrorMatches, "not a valid TiDB version.*")
+
+	_, err = common.ExtractTiDBVersion("not-a-valid-version")
+	c.Assert(err, NotNil)
+}
diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go
index 14ebe01..b1c8a84 100644
--- a/lightning/restore/tidb_test.go
+++ b/lightning/restore/tidb_test.go
@@ -240,7 +240,7 @@ func (s *tidbSuite) TestLoadSchemaInfo(c *C) {
 		tableInfos = append(tableInfos, info)
 	}
 
-	loaded, err := s.timgr.LoadSchemaInfo(ctx, []*mydump.MDDatabaseMeta{{Name: "db"}}, func(schema string) ([]*model.TableInfo, error) {
+	loaded, err := s.timgr.LoadSchemaInfo(ctx, []*mydump.MDDatabaseMeta{{Name: "db"}}, func(ctx context.Context, schema string) ([]*model.TableInfo, error) {
 		c.Assert(schema, Equals, "db")
 		return tableInfos, nil
 	})
@@ -269,7 +269,7 @@ func (s *tidbSuite) TestLoadSchemaInfo(c *C) {
 func (s *tidbSuite) TestLoadSchemaInfoMissing(c *C) {
 	ctx := context.Background()
 
-	_, err := s.timgr.LoadSchemaInfo(ctx, []*mydump.MDDatabaseMeta{{Name: "asdjalsjdlas"}}, func(schema string) ([]*model.TableInfo, error) {
+	_, err := s.timgr.LoadSchemaInfo(ctx, []*mydump.MDDatabaseMeta{{Name: "asdjalsjdlas"}}, func(ctx context.Context, schema string) ([]*model.TableInfo, error) {
 		return nil, errors.Errorf("[schema:1049]Unknown database '%s'", schema)
 	})
 	c.Assert(err, ErrorMatches, ".*Unknown database.*")
diff --git a/tests/alter_random/run.sh b/tests/alter_random/run.sh
index 84499ad..08967bc 100644
--- a/tests/alter_random/run.sh
+++ b/tests/alter_random/run.sh
@@ -34,5 +34,9 @@ for backend in tidb importer local; do
     # auto random base is 4
     run_sql "INSERT INTO alter_random.t VALUES ();"
     run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM alter_random.t"
-    check_contains 'inc: 4'
+    if [ "$backend" = 'tidb' ]; then
+      check_contains 'inc: 30002'
+    else
+      check_contains 'inc: 4'
+    fi
 done