Repository: pingcap/tidb (Go)

Issue:
should show correct duplicate entry while add index
## Bug Report

Please answer these questions before submitting your issue. Thanks!

1. What did you do?
If possible, provide a recipe for reproducing the error.
tidb:
```
mysql> CREATE TABLE binlog_pk_add_duplicate_uk(id INT PRIMARY KEY, a1 INT);
Query OK, 0 rows affected (0.18 sec)

mysql> INSERT INTO binlog_pk_add_duplicate_uk(id, a1) VALUES(1,1),(2,1);
Query OK, 2 rows affected (0.03 sec)
Records: 2  Duplicates: 0  Warnings: 0

mysql> ALTER TABLE binlog_pk_add_duplicate_uk ADD UNIQUE INDEX aidx(a1);
ERROR 1062 (23000): Duplicate entry '' for key 'aidx'
mysql>
```



2. What did you expect to see?
mysql:
```
mysql> use test;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> CREATE TABLE binlog_pk_add_duplicate_uk(id INT PRIMARY KEY, a1 INT);
Query OK, 0 rows affected (0.08 sec)

mysql> INSERT INTO binlog_pk_add_duplicate_uk(id, a1) VALUES(1,1),(2,1);
Query OK, 2 rows affected (0.03 sec)
Records: 2  Duplicates: 0  Warnings: 0

mysql> ALTER TABLE binlog_pk_add_duplicate_uk ADD UNIQUE INDEX aidx(a1);
ERROR 1062 (23000): Duplicate entry '1' for key 'aidx'
```


3. What did you see instead?
Duplicate entry '' for key 'aidx'  


4. What version of TiDB are you using (`tidb-server -V` or run `select tidb_version();` on TiDB)?
```
Release Version: v4.0.0-alpha-689-g11cd41d2b
Git Commit Hash: 11cd41d2b59f17055bc25dfb281262f0665f4cdf
Git Branch: master
UTC Build Time: 2019-11-03 04:53:59
GoVersion: go version go1.13 darwin/amd64
Race Enabled: false
TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
Check Table Before Drop: false
```



Discussion:
Confirming this issue still exists in master:
```sql
DROP TABLE IF EXISTS binlog_pk_add_duplicate_uk;
CREATE TABLE binlog_pk_add_duplicate_uk(id INT PRIMARY KEY, a1 INT);
INSERT INTO binlog_pk_add_duplicate_uk(id, a1) VALUES(1,1),(2,1);
ALTER TABLE binlog_pk_add_duplicate_uk ADD UNIQUE INDEX aidx(a1);

..

mysql> ALTER TABLE binlog_pk_add_duplicate_uk ADD UNIQUE INDEX aidx(a1);
ERROR 1062 (23000): Duplicate entry '' for key 'aidx'
mysql> select tidb_version()\G
*************************** 1. row ***************************
tidb_version(): Release Version: v4.0.0-beta.2-782-gb72e47e6d
Edition: Community
Git Commit Hash: b72e47e6db8863c08e47714391cf937b0f1c3269
Git Branch: master
UTC Build Time: 2020-07-15 01:26:06
GoVersion: go1.13
Race Enabled: false
TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
Check Table Before Drop: false
1 row in set (0.00 sec)
```
## Please edit this comment or add a new comment to complete the following information

### Not a bug

1. Remove the 'type/bug' label
2. Add notes to indicate why it is not a bug

### Duplicate bug

1. Add the 'type/duplicate' label
2. Add the link to the original bug

### Bug

Note: Make Sure that 'component', and 'severity' labels are added
Example for how to fill out the template: https://github.com/pingcap/tidb/issues/20100

#### 1. Root Cause Analysis (RCA) (optional) 
<!-- Write down the reason why this bug occurs -->

#### 2. Symptom (optional)

<!-- What will the user see when this bug occurs. The error message may be in the terminal, log or monitoring -->

#### 3. All Trigger Conditions (optional)

<!-- All the user scenarios that may trigger this bug -->

#### 4. Workaround (optional)

#### 5. Affected versions

<!--
In the format of [start_version:end_version], multiple version ranges are
accepted. If the bug only affects the unreleased version, please input:
"unreleased". For example:

Notes:
  1. Do not use any white spaces in '[]'.
  2. The range in '[]' is a closed interval
  3. The version format is `v$Major.$Minor.$Patch`, the $Majoy and $Minor
     number in a version range should be the same. [v3.0.1:v3.1.2] is
     invalid because the $Minor number of the version range is different.

Example 1: [v3.0.1:v3.0.5], [v4.0.1:v4.0.5]
Example 2: unreleased
-->

#### 6. Fixed versions

<!--
The first released version that contains this fix in each minor version. If the bug's affected version has been released, the fixed version should be a detailed version number; If the bug doesn't affect any released version, the fixed version can be "master";  

Example 1: v3.0.13, v4.0.5
Example 2: master
-->



Patch:
diff --git a/ddl/index.go b/ddl/index.go
index b4f3ac5038bd1..0483388a887f4 100644
--- a/ddl/index.go
+++ b/ddl/index.go
@@ -14,6 +14,7 @@
 package ddl
 
 import (
+	"bytes"
 	"context"
 	"strings"
 	"sync/atomic"
@@ -39,8 +40,10 @@ import (
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util"
+	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/logutil"
 	decoder "github.com/pingcap/tidb/util/rowDecoder"
+	"github.com/pingcap/tidb/util/rowcodec"
 	"github.com/pingcap/tidb/util/timeutil"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
@@ -1013,6 +1016,73 @@ func (w *addIndexWorker) initBatchCheckBufs(batchCount int) {
 	w.distinctCheckFlags = w.distinctCheckFlags[:0]
 }
 
+func (w *addIndexWorker) checkHandleExists(key kv.Key, value []byte, handle kv.Handle) error {
+	idxInfo := w.index.Meta()
+	tblInfo := w.table.Meta()
+	name := w.index.Meta().Name.String()
+
+	colInfo := make([]rowcodec.ColInfo, 0, len(idxInfo.Columns))
+	for _, idxCol := range idxInfo.Columns {
+		col := tblInfo.Columns[idxCol.Offset]
+		colInfo = append(colInfo, rowcodec.ColInfo{
+			ID:         col.ID,
+			IsPKHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.Flag),
+			Ft:         rowcodec.FieldTypeFromModelColumn(col),
+		})
+	}
+
+	values, err := tablecodec.DecodeIndexKV(key, value, len(idxInfo.Columns), tablecodec.HandleDefault, colInfo)
+	if err != nil {
+		return err
+	}
+
+	if !w.table.Meta().IsCommonHandle {
+		_, d, err := codec.DecodeOne(values[len(colInfo)])
+		if err != nil {
+			return errors.Trace(err)
+		}
+		if d.GetInt64() == handle.IntValue() {
+			return nil
+		}
+	} else {
+		// We expect the two handle have the same number of columns, because they come from a same table.
+		// But we still need to check it explicitly, otherwise we will encounter undesired index out of range panic,
+		// or undefined behavior if someone change the format of the value returned by tablecodec.DecodeIndexKV.
+		colsOfHandle := len(values) - len(colInfo)
+		if w.index.Meta().Global {
+			colsOfHandle--
+		}
+		if colsOfHandle != handle.NumCols() {
+			// We can claim these two handle are different, because they have different length.
+			// But we'd better report an error at here to detect compatibility problem introduced in other package during tests.
+			return errors.New("number of columns in two handle is different")
+		}
+
+		for i := 0; i < handle.NumCols(); i++ {
+			if bytes.Equal(values[i+len(colInfo)], handle.EncodedCol(i)) {
+				colsOfHandle--
+			}
+		}
+		if colsOfHandle == 0 {
+			return nil
+		}
+	}
+
+	valueStr := make([]string, 0, len(colInfo))
+	for i, val := range values[:len(colInfo)] {
+		d, err := tablecodec.DecodeColumnValue(val, colInfo[i].Ft, time.Local)
+		if err != nil {
+			return kv.ErrKeyExists.FastGenByArgs(key.String(), name)
+		}
+		str, err := d.ToString()
+		if err != nil {
+			str = string(val)
+		}
+		valueStr = append(valueStr, str)
+	}
+	return kv.ErrKeyExists.FastGenByArgs(strings.Join(valueStr, "-"), name)
+}
+
 func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*indexRecord) error {
 	idxInfo := w.index.Meta()
 	if !idxInfo.Unique {
@@ -1046,13 +1116,8 @@ func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*i
 	for i, key := range w.batchCheckKeys {
 		if val, found := batchVals[string(key)]; found {
 			if w.distinctCheckFlags[i] {
-				handle, err1 := tablecodec.DecodeHandleInUniqueIndexValue(val, w.table.Meta().IsCommonHandle)
-				if err1 != nil {
-					return errors.Trace(err1)
-				}
-
-				if handle != idxRecords[i].handle {
-					return errors.Trace(kv.ErrKeyExists)
+				if err := w.checkHandleExists(key, val, idxRecords[i].handle); err != nil {
+					return errors.Trace(err)
 				}
 			}
 			idxRecords[i].skip = true
@@ -1060,7 +1125,11 @@ func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*i
 			// The keys in w.batchCheckKeys also maybe duplicate,
 			// so we need to backfill the not found key into `batchVals` map.
 			if w.distinctCheckFlags[i] {
-				batchVals[string(key)] = tablecodec.EncodeHandleInUniqueIndexValue(idxRecords[i].handle, false)
+				val, err := w.index.GenIndexValue(stmtCtx, idxRecords[i].vals, w.distinctCheckFlags[i], false, idxRecords[i].handle)
+				if err != nil {
+					return errors.Trace(err)
+				}
+				batchVals[string(key)] = val
 			}
 		}
 	}
diff --git a/ddl/rollingback.go b/ddl/rollingback.go
index ff0bfbc5b3d3e..bdc1f6629f8f7 100644
--- a/ddl/rollingback.go
+++ b/ddl/rollingback.go
@@ -21,7 +21,6 @@ import (
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
-	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
 	"github.com/pingcap/tidb/util/logutil"
 	"go.uber.org/zap"
@@ -70,10 +69,6 @@ func convertAddIdxJob2RollbackJob(t *meta.Meta, job *model.Job, tblInfo *model.T
 		return ver, errors.Trace(err1)
 	}
 
-	if kv.ErrKeyExists.Equal(err) {
-		return ver, kv.ErrKeyExists.GenWithStackByArgs("", indexInfo.Name.O)
-	}
-
 	return ver, errors.Trace(err)
 }
 
diff --git a/table/index.go b/table/index.go
index 90aefe280ccd2..e0554f16ff8d3 100644
--- a/table/index.go
+++ b/table/index.go
@@ -73,6 +73,8 @@ type Index interface {
 	Exist(sc *stmtctx.StatementContext, us kv.UnionStore, indexedValues []types.Datum, h kv.Handle) (bool, kv.Handle, error)
 	// GenIndexKey generates an index key.
 	GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.Datum, h kv.Handle, buf []byte) (key []byte, distinct bool, err error)
+	// GenIndexValue generates an index value.
+	GenIndexValue(sc *stmtctx.StatementContext, indexedValues []types.Datum, distinct bool, untouched bool, h kv.Handle) (val []byte, err error)
 	// Seek supports where clause.
 	Seek(sc *stmtctx.StatementContext, r kv.Retriever, indexedValues []types.Datum) (iter IndexIterator, hit bool, err error)
 	// SeekFirst supports aggregate min and ascend order by.
diff --git a/table/tables/index.go b/table/tables/index.go
index 5d3868463a756..90ce5e22cd0e9 100644
--- a/table/tables/index.go
+++ b/table/tables/index.go
@@ -139,6 +139,10 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.
 	return tablecodec.GenIndexKey(sc, c.tblInfo, c.idxInfo, idxTblID, indexedValues, h, buf)
 }
 
+func (c *index) GenIndexValue(sc *stmtctx.StatementContext, indexedValues []types.Datum, distinct bool, untouched bool, h kv.Handle) (val []byte, err error) {
+	return tablecodec.GenIndexValueNew(sc, c.tblInfo, c.idxInfo, c.containNonBinaryString, distinct, untouched, indexedValues, h, c.phyTblID)
+}
+
 // Create creates a new entry in the kvIndex data.
 // If the index is unique and there is an existing entry with the same key,
 // Create will return the existing entry's handle as the first return value, ErrKeyExists as the second return value.


Test patch:
diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go
index db37d2f4e0d1e..58da97f53bc2d 100644
--- a/ddl/db_integration_test.go
+++ b/ddl/db_integration_test.go
@@ -42,6 +42,7 @@ import (
 	"github.com/pingcap/tidb/store/mockstore/cluster"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/collate"
 	"github.com/pingcap/tidb/util/israce"
 	"github.com/pingcap/tidb/util/mock"
 	"github.com/pingcap/tidb/util/testkit"
@@ -2590,3 +2591,58 @@ func (s *testIntegrationSuite3) TestStrictDoubleTypeCheck(c *C) {
 	defer tk.MustExec("set @@tidb_enable_strict_double_type_check = 'ON'")
 	tk.MustExec(sql)
 }
+
+func (s *testIntegrationSuite7) TestDuplicateErrorMessage(c *C) {
+	defer collate.SetNewCollationEnabledForTest(false)
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+
+	type testdata struct {
+		types  []string
+		values []string
+	}
+	tests := []testdata{
+		{[]string{"int"}, []string{"1"}},
+		{[]string{"datetime"}, []string{"'2020-01-01 00:00:00'"}},
+		{[]string{"varchar(10)"}, []string{"'qwe'"}},
+		{[]string{"enum('r', 'g', 'b')"}, []string{"'r'"}},
+		{[]string{"int", "datetime", "varchar(10)", "enum('r', 'g', 'b')"}, []string{"1", "'2020-01-01 00:00:00'", "'qwe'", "'r'"}},
+	}
+
+	for _, newCollate := range []bool{false, true} {
+		collate.SetNewCollationEnabledForTest(newCollate)
+		for _, globalIndex := range []bool{false, true} {
+			restoreConfig := config.RestoreFunc()
+			config.UpdateGlobal(func(conf *config.Config) {
+				conf.EnableGlobalIndex = globalIndex
+			})
+			for _, clusteredIndex := range []int{0, 1} {
+				tk.MustExec(fmt.Sprintf("set session tidb_enable_clustered_index=%d;", clusteredIndex))
+				for _, t := range tests {
+					tk.MustExec("drop table if exists t;")
+					fields := make([]string, len(t.types))
+
+					for i, tp := range t.types {
+						fields[i] = fmt.Sprintf("a%d %s", i, tp)
+					}
+					tk.MustExec("create table t (id1 int, id2 varchar(10), " + strings.Join(fields, ",") + ",primary key(id1, id2)) " +
+						"collate utf8mb4_general_ci " +
+						"partition by range (id1) (partition p1 values less than (2), partition p2 values less than (maxvalue))")
+
+					vals := strings.Join(t.values, ",")
+					tk.MustExec(fmt.Sprintf("insert into t values (1, 'asd', %s), (1, 'dsa', %s)", vals, vals))
+					for i := range t.types {
+						fields[i] = fmt.Sprintf("a%d", i)
+					}
+					index := strings.Join(fields, ",")
+					for i, val := range t.values {
+						fields[i] = strings.Replace(val, "'", "", -1)
+					}
+					tk.MustGetErrMsg("alter table t add unique index t_idx(id1,"+index+")",
+						fmt.Sprintf("[kv:1062]Duplicate entry '1-%s' for key 't_idx'", strings.Join(fields, "-")))
+				}
+			}
+			restoreConfig()
+		}
+	}
+}
diff --git a/ddl/db_test.go b/ddl/db_test.go
index ab23e40a458e7..7c706d71991df 100644
--- a/ddl/db_test.go
+++ b/ddl/db_test.go
@@ -291,7 +291,7 @@ func (s *testDBSuite5) TestAddPrimaryKeyRollback1(c *C) {
 	hasNullValsInKey := false
 	idxName := "PRIMARY"
 	addIdxSQL := "alter table t1 add primary key c3_index (c3);"
-	errMsg := "[kv:1062]Duplicate entry '' for key 'PRIMARY'"
+	errMsg := "[kv:1062]Duplicate entry '" + strconv.Itoa(defaultBatchSize*2-10) + "' for key 'PRIMARY'"
 	testAddIndexRollback(c, s.store, s.lease, idxName, addIdxSQL, errMsg, hasNullValsInKey)
 }
 
@@ -308,7 +308,7 @@ func (s *testDBSuite2) TestAddUniqueIndexRollback(c *C) {
 	hasNullValsInKey := false
 	idxName := "c3_index"
 	addIdxSQL := "create unique index c3_index on t1 (c3)"
-	errMsg := "[kv:1062]Duplicate entry '' for key 'c3_index'"
+	errMsg := "[kv:1062]Duplicate entry '" + strconv.Itoa(defaultBatchSize*2-10) + "' for key 'c3_index'"
 	testAddIndexRollback(c, s.store, s.lease, idxName, addIdxSQL, errMsg, hasNullValsInKey)
 }