Repository: pingcap/tidb (Go)

Issue:
txn: pessimistic statement execution panic panicked
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. What did you do?

<!-- If possible, provide a recipe for reproducing the error. -->
Execute statement `insert into xxx select from`

### 2. What did you expect to see?
Execution results ok

### 3. What did you see instead?
tidb panicked
stack
```
"connection running loop panic"] [conn=1] [lastSQL="insert into sbtest2 select * from sbtest1"] [err="runtime error: index out of range [0] with length 0"] [stack="goroutine 854 [running]:
github.com/pingcap/tidb/server.(*clientConn).Run.func1(0x3667a60, 0xc000e1b470, 0xc000c148f0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/server/conn.go:660 +0xee
panic(0x2f79fe0, 0xc023cd2bc0)
/usr/local/go/src/runtime/panic.go:679 +0x1b2
github.com/pingcap/tidb/executor.(*ExecStmt).Exec.func1(0xc000dcdf80, 0xc000e90cc8, 0x3667a60, 0xc000e1b470)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/executor/adapter.go:292 +0x53a
panic(0x2f79fe0, 0xc023cd2bc0)
/usr/local/go/src/runtime/panic.go:679 +0x1b2
github.com/pingcap/tidb/store/tikv.preSplitAndScatterIn2PC(0x36679e0, 0xc0000e2000, 0xc000522100, 0x2, 0x1, 0x18, 0x0, 0x0, 0x0, 0xc02f328000, ...)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/store/tikv/2pc.go:474 +0xa15
github.com/pingcap/tidb/store/tikv.(*twoPhaseCommitter).doActionOnMutations(0xc023cda000, 0xc000ec2080, 0x365dfa0, 0xc0003e42a0, 0x0, 0x0, 0x0, 0xc02f328000, 0xf4240, 0xf4240, ...)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/store/tikv/2pc.go:451 +0x51a
github.com/pingcap/tidb/store/tikv.(*twoPhaseCommitter).pessimisticLockMutations(...)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/store/tikv/2pc.go:1200
github.com/pingcap/tidb/store/tikv.(*tikvTxn).LockKeys(0xc000f18630, 0x3667a60, 0xc000e1b470, 0xc0003e42a0, 0xc02dc44000, 0xf4240, 0xf4240, 0x0, 0x0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/store/tikv/txn.go:388 +0x467
github.com/pingcap/tidb/executor.(*ExecStmt).handlePessimisticDML(0xc000dcdf80, 0x3667a60, 0xc000e1b470, 0x3671260, 0xc000ee4a20, 0x29, 0x5c7b48c92180004)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/executor/adapter.go:552 +0x42a
github.com/pingcap/tidb/executor.(*ExecStmt).handleNoDelay(0xc000dcdf80, 0x3667a60, 0xc000e1b470, 0x3671260, 0xc000ee4a20, 0x5120001, 0x3, 0x0, 0xc000e90b58, 0x115ef67, ...)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/executor/adapter.go:387 +0xc7
github.com/pingcap/tidb/executor.(*ExecStmt).Exec(0xc000dcdf80, 0x3667a60, 0xc000e1b470, 0x0, 0x0, 0x0, 0x0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/executor/adapter.go:357 +0x3c7
github.com/pingcap/tidb/session.runStmt(0x3667a60, 0xc000e1b470, 0x36a9da0, 0xc000c24600, 0x3670f20, 0xc000dcdf80, 0x0, 0x0, 0x0, 0x0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/session/tidb.go:276 +0x304
github.com/pingcap/tidb/session.(*session).ExecuteStmt(0xc000c24600, 0x3667a60, 0xc000e1b470, 0x3670460, 0xc000f0fb80, 0x0, 0x0, 0x0, 0x0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/session/session.go:1163 +0x7e0
github.com/pingcap/tidb/server.(*TiDBContext).ExecuteStmt(0xc000e1b770, 0x3667a60, 0xc000e1b470, 0x3670460, 0xc000f0fb80, 0x8, 0x8, 0xc000e91300, 0x11ce7c8)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/server/driver_tidb.go:248 +0x65
github.com/pingcap/tidb/server.(*clientConn).handleStmt(0xc000c148f0, 0x3667a60, 0xc000e1b470, 0x3670460, 0xc000f0fb80, 0xc00118a801, 0x0, 0x0)
/home/jenkins/agent/workspace/tidb_master/go/src/github.com/pingcap/tidb/server/conn.go:1294 +0x8e
github.com/pingcap/tidb/server.(*clientConn).handleQuery
```

This is because when doing **pessimistic lock keys**, the `committerMutations` struct has only elements in `keys` array but nothing in `values` array, accessing `values` array leads to panic 

### 4. What version of TiDB are you using? (`tidb-server -V` or run `select tidb_version();` on TiDB)

nightly with
```
["Git Commit Hash"=d367a0101e4555fda76f8d33258a6d903991741a] ["Git Branch"=master]
```


Discussion:
@coocood @jackysp @tiancaiamao   The `pre-split` logic  seems incompatible with pessimistic locks logic, we need to fix this
@zyguan  I think we need to daily run some large tests and collect results, we could trace the result is ok and the resource usage is expected



Patch:
diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go
index fdc5bcc5c827b..6f6409a758672 100644
--- a/store/tikv/2pc.go
+++ b/store/tikv/2pc.go
@@ -465,13 +465,18 @@ func (c *twoPhaseCommitter) doActionOnMutations(bo *Backoffer, action twoPhaseCo
 }
 
 func preSplitAndScatterIn2PC(ctx context.Context, store *tikvStore, group groupedMutations) bool {
-	length := group.mutations.len()
 	splitKeys := make([][]byte, 0, 4)
 
 	preSplitSizeThresholdVal := atomic.LoadUint32(&preSplitSizeThreshold)
 	regionSize := 0
-	for i := 0; i < length; i++ {
-		regionSize = regionSize + len(group.mutations.keys[i]) + len(group.mutations.values[i])
+	keysLength := group.mutations.len()
+	valsLength := len(group.mutations.values)
+	// The value length maybe zero for pessimistic lock keys
+	for i := 0; i < keysLength; i++ {
+		regionSize = regionSize + len(group.mutations.keys[i])
+		if i < valsLength {
+			regionSize = regionSize + len(group.mutations.values[i])
+		}
 		// The second condition is used for testing.
 		if regionSize >= int(preSplitSizeThresholdVal) {
 			regionSize = 0
@@ -484,7 +489,8 @@ func preSplitAndScatterIn2PC(ctx context.Context, store *tikvStore, group groupe
 
 	regionIDs, err := store.SplitRegions(ctx, splitKeys, true)
 	if err != nil {
-		logutil.BgLogger().Warn("2PC split regions failed", zap.Uint64("regionID", group.region.id), zap.Int("keys count", length), zap.Error(err))
+		logutil.BgLogger().Warn("2PC split regions failed", zap.Uint64("regionID", group.region.id),
+			zap.Int("keys count", keysLength), zap.Int("values count", valsLength), zap.Error(err))
 		return false
 	}
 


Test patch:
diff --git a/store/tikv/2pc_slow_test.go b/store/tikv/2pc_slow_test.go
index c7a70a188306e..c6f3dbe718e27 100644
--- a/store/tikv/2pc_slow_test.go
+++ b/store/tikv/2pc_slow_test.go
@@ -20,6 +20,7 @@ import (
 	"sync/atomic"
 
 	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb/kv"
 )
 
 // TestCommitMultipleRegions tests commit multiple regions.
@@ -52,8 +53,6 @@ func (s *testTiclientSuite) TestSplitRegionIn2PC(c *C) {
 	atomic.StoreUint32(&preSplitSizeThreshold, 5000)
 
 	bo := NewBackoffer(context.Background(), 1)
-	startKey := encodeKey(s.prefix, s08d("key", 0))
-	endKey := encodeKey(s.prefix, s08d("key", preSplitThresholdInTest))
 	checkKeyRegion := func(bo *Backoffer, start, end []byte, checker Checker) {
 		// Check regions after split.
 		loc1, err := s.store.regionCache.LocateKey(bo, start)
@@ -62,16 +61,47 @@ func (s *testTiclientSuite) TestSplitRegionIn2PC(c *C) {
 		c.Assert(err, IsNil)
 		c.Assert(loc1.Region.id, checker, loc2.Region.id)
 	}
-
-	// Check before test.
-	checkKeyRegion(bo, startKey, endKey, Equals)
-	txn := s.beginTxn(c)
-	for i := 0; i < preSplitThresholdInTest; i++ {
-		err := txn.Set(encodeKey(s.prefix, s08d("key", i)), valueBytes(i))
+	mode := []string{"optimistic", "pessimistic"}
+	var (
+		startKey []byte
+		endKey   []byte
+	)
+	ctx := context.Background()
+	for _, m := range mode {
+		if m == "optimistic" {
+			startKey = encodeKey(s.prefix, s08d("key", 0))
+			endKey = encodeKey(s.prefix, s08d("key", preSplitThresholdInTest))
+		} else {
+			startKey = encodeKey(s.prefix, s08d("pkey", 0))
+			endKey = encodeKey(s.prefix, s08d("pkey", preSplitThresholdInTest))
+		}
+		// Check before test.
+		checkKeyRegion(bo, startKey, endKey, Equals)
+		txn := s.beginTxn(c)
+		if m == "pessimistic" {
+			txn.SetOption(kv.Pessimistic, true)
+			lockCtx := &kv.LockCtx{}
+			lockCtx.ForUpdateTS = txn.startTS
+			keys := make([]kv.Key, 0, preSplitThresholdInTest)
+			for i := 0; i < preSplitThresholdInTest; i++ {
+				keys = append(keys, encodeKey(s.prefix, s08d("pkey", i)))
+			}
+			err := txn.LockKeys(ctx, lockCtx, keys...)
+			c.Assert(err, IsNil)
+			checkKeyRegion(bo, startKey, endKey, Not(Equals))
+		}
+		var err error
+		for i := 0; i < preSplitThresholdInTest; i++ {
+			if m == "optimistic" {
+				err = txn.Set(encodeKey(s.prefix, s08d("key", i)), valueBytes(i))
+			} else {
+				err = txn.Set(encodeKey(s.prefix, s08d("pkey", i)), valueBytes(i))
+			}
+			c.Assert(err, IsNil)
+		}
+		err = txn.Commit(context.Background())
 		c.Assert(err, IsNil)
+		// Check region split after test.
+		checkKeyRegion(bo, startKey, endKey, Not(Equals))
 	}
-	err := txn.Commit(context.Background())
-	c.Assert(err, IsNil)
-	// Check region split after test.
-	checkKeyRegion(bo, startKey, endKey, Not(Equals))
 }