Repository: pingcap/tidb (Go)

Issue:
load-stats statements return an error when column collate is `utf8mb4_unicode_ci`
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)

<!-- a step by step guide for reproducing the bug. -->
```
create table t (
  `actor_login` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
    KEY `index_github_events_on_actor_login` (`actor_login`)
);

-- insert some values randomly
insert into t values ("SteviePlaysMc4");
-- ...
insert into t values ("andr3a809");

analyze table t with 0 top_n;
```

dump the stats and then load it back with `load stats ...`, then you can see some errors like:
```
mysql> load stats 'stats.json';
ERROR 1300 (HY000): Invalid utf8mb4 character string: 'FB'
```

### 2. What did you expect to see? (Required)
Can load it back successfully.

### 3. What did you see instead (Required)
```
mysql> load stats 'stats.json';
ERROR 1300 (HY000): Invalid utf8mb4 character string: 'FB'
```

```
runtime/debug.Stack()
        /Users/zhangyuanjia/Workspace/goroot/go1.18/src/runtime/debug/stack.go:24 +0x65
runtime/debug.PrintStack()
        /Users/zhangyuanjia/Workspace/goroot/go1.18/src/runtime/debug/stack.go:16 +0x19
github.com/pingcap/tidb/parser/charset.generateEncodingErr({0x6707faf, 0x7}, {0xc016048003?, 0xc00fe87b00?, 0x40?})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/vendor/github.com/pingcap/tidb/parser/charset/encoding_base.go:121 +0x79
github.com/pingcap/tidb/parser/charset.encodingBase.Transform.func1({0xc016048003, 0x1, 0xb}, {0xc016048003, 0x1, 0xb}, 0x10?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/vendor/github.com/pingcap/tidb/parser/charset/encoding_base.go:69 +0xab
github.com/pingcap/tidb/parser/charset.(*encodingUTF8).Foreach(0xc015e68510?, {0xc016048000, 0xe, 0xe}, 0x5f88?, 0xc015f7e9c0)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/vendor/github.com/pingcap/tidb/parser/charset/encoding_utf8.go:101 +0xfc
github.com/pingcap/tidb/parser/charset.encodingBase.Transform({{0x6c95418?, 0x8aeeec0?}, {0x6cb98c0?, 0x8abaf40?}}, 0x0, {0xc016048000, 0xe, 0xe}, 0x26)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/vendor/github.com/pingcap/tidb/parser/charset/encoding_base.go:66 +0x1b6
github.com/pingcap/tidb/parser/charset.(*encodingUTF8).Transform(0x8abaf40, 0xc010b61430?, {0xc016048000, 0xe, 0xe}, 0xa888?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/vendor/github.com/pingcap/tidb/parser/charset/encoding_utf8.go:92 +0xab
github.com/pingcap/tidb/types.(*Datum).GetBinaryStringDecoded(0xc012de6750, 0x83b33e0?, {0xc010b61430?, 0x4069779?})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/types/datum.go:209 +0xf4
github.com/pingcap/tidb/types.(*Datum).convertToString(0x4019dcb?, 0x14d24400?, 0xc0005d6e68)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/types/datum.go:1018 +0x247
github.com/pingcap/tidb/types.(*Datum).ConvertTo(0xc015e5a960?, 0x40ddb60?, 0xc000132060?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/types/datum.go:896 +0xde7
github.com/pingcap/tidb/statistics.(*Histogram).ConvertTo(0xc015e59260, 0xc000130008?, 0xc012de6a00?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/statistics/histogram.go:269 +0x1be
github.com/pingcap/tidb/statistics/handle.TableStatsFromJSON(0xc0004f2900, 0x8e, 0xc015fafe60)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/statistics/handle/dump.go:318 +0x830
github.com/pingcap/tidb/statistics/handle.(*Handle).loadStatsFromJSON(0x88020c0?, 0xc015cd1d50?, 0x6cc3e00?, 0xc015cd1d50?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/statistics/handle/dump.go:234 +0x3c
github.com/pingcap/tidb/statistics/handle.(*Handle).LoadStatsFromJSON(0xc00ff426c0?, {0x6cc3ec8, 0xc01241e6e0}, 0xc015fafe60)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/statistics/handle/dump.go:208 +0x158
github.com/pingcap/tidb/executor.(*LoadStatsInfo).Update(0xc015e30880, {0xc01601a000, 0x1e22e, 0x22000})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/executor/load_stats.go:90 +0x155
github.com/pingcap/tidb/server.(*clientConn).handleLoadStats(0xc00ff426c0?, {0x6ca3a40?, 0xc015f0dbf0?}, 0xc015e30880)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:1757 +0x74
github.com/pingcap/tidb/server.(*clientConn).handleQuerySpecial(0xc00f9bb340, {0x6ca3a40, 0xc015f0dbf0}, 0x5220?)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:2078 +0x28e
github.com/pingcap/tidb/server.(*clientConn).handleStmt(0xc00f9bb340, {0x6ca3998, 0xc015fb27c0}, {0x6cb5220, 0xc015e5a5a0}, {0x8aef7b8, 0x0, 0x0}, 0x1)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:2050 +0x487
github.com/pingcap/tidb/server.(*clientConn).handleQuery(0xc00f9bb340, {0x6ca3998, 0xc015fb27c0}, {0xc015c7e8a1, 0x5a})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:1876 +0x774
github.com/pingcap/tidb/server.(*clientConn).dispatch(0xc00f9bb340, {0x6ca3a40?, 0xc00fdb9ad0?}, {0xc015c7e8a0, 0x5b, 0x5b})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:1371 +0xe55
github.com/pingcap/tidb/server.(*clientConn).Run(0xc00f9bb340, {0x6ca3a40, 0xc00fdb9ad0})
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/conn.go:1121 +0x24e
github.com/pingcap/tidb/server.(*Server).onConn(0xc00fa32fc0, 0xc00f9bb340)
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/server.go:559 +0x6c9
created by github.com/pingcap/tidb/server.(*Server).startNetworkListener
        /Users/zhangyuanjia/Workspace/go/src/github.com/pingcap/tidb/server/server.go:455 +0x5ca
```

### 4. What is your TiDB version? (Required)

<!-- Paste the output of SELECT tidb_version() -->
```
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| tidb_version()                                                                                                                                                                                                                                                                                                                                      |
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Release Version: v6.2.0-alpha-86-g597942f75c-dirty
Edition: Community
Git Commit Hash: 597942f75c0168871bbe9e753db1060ff8b823bd
Git Branch: cost-model-2
UTC Build Time: 2022-06-07 10:26:18
GoVersion: go1.18
Race Enabled: false
TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
Check Table Before Drop: false
Store: unistore |
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
```




Discussion:

Patch:
diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go
index 4fd89b6c11b84..5216622dc9a35 100644
--- a/statistics/handle/dump.go
+++ b/statistics/handle/dump.go
@@ -302,18 +302,21 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			}
 			hist := statistics.HistogramFromProto(jsonCol.Histogram)
 			sc := &stmtctx.StatementContext{TimeZone: time.UTC}
-			// Deal with sortKey, the length of sortKey maybe longer than the column's length.
-			orgLen := colInfo.FieldType.GetFlen()
-			if types.IsString(colInfo.FieldType.GetType()) {
-				colInfo.SetFlen(types.UnspecifiedLength)
+			tmpFT := colInfo.FieldType
+			// For new collation data, when storing the bounds of the histogram, we store the collate key instead of the
+			// original value.
+			// But there's additional conversion logic for new collation data, and the collate key might be longer than
+			// the FieldType.flen.
+			// If we use the original FieldType here, there might be errors like "Invalid utf8mb4 character string"
+			// or "Data too long".
+			// So we change it to TypeBlob to bypass those logics here.
+			if colInfo.FieldType.EvalType() == types.ETString && colInfo.FieldType.GetType() != mysql.TypeEnum && colInfo.FieldType.GetType() != mysql.TypeSet {
+				tmpFT = *types.NewFieldType(mysql.TypeBlob)
 			}
-			hist, err := hist.ConvertTo(sc, &colInfo.FieldType)
+			hist, err := hist.ConvertTo(sc, &tmpFT)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			if types.IsString(colInfo.FieldType.GetType()) {
-				colInfo.SetFlen(orgLen)
-			}
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
 			fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation
diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go
index 2e9ba37d9d4a7..2f4d313446f24 100644
--- a/statistics/handle/handle.go
+++ b/statistics/handle/handle.go
@@ -1490,9 +1490,13 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID
 		} else {
 			sc := &stmtctx.StatementContext{TimeZone: time.UTC}
 			d := rows[i].GetDatum(2, &fields[2].Column.FieldType)
-			// When there's new collation data, the length of bounds of histogram(the collate key) might be
-			// longer than the FieldType.flen of this column.
-			// We change it to TypeBlob to bypass the length check here.
+			// For new collation data, when storing the bounds of the histogram, we store the collate key instead of the
+			// original value.
+			// But there's additional conversion logic for new collation data, and the collate key might be longer than
+			// the FieldType.flen.
+			// If we use the original FieldType here, there might be errors like "Invalid utf8mb4 character string"
+			// or "Data too long".
+			// So we change it to TypeBlob to bypass those logics here.
 			if tp.EvalType() == types.ETString && tp.GetType() != mysql.TypeEnum && tp.GetType() != mysql.TypeSet {
 				tp = types.NewFieldType(mysql.TypeBlob)
 			}


Test patch:
diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go
index 2940bb691f17a..26335bd203785 100644
--- a/statistics/handle/dump_test.go
+++ b/statistics/handle/dump_test.go
@@ -400,7 +400,7 @@ func TestLoadStatsForNewCollation(t *testing.T) {
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)")
-	tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
+	tk.MustExec("insert into t value(1, 'aaa'), (1, 'aaa'), (3, 'aab'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc'), (7, 'Ste')")
 	// mark column stats as needed
 	tk.MustExec("select * from t where a = 3")
 	tk.MustExec("select * from t where b = 'bbb'")