Repository: pingcap/tidb (Go)

Issue:
The function `upper` produces error for some special characters
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)

With `new_collation_enabled` is `True`

```
create table t(a varchar(32)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
insert into t values('ʞ');
select binary upper('ʞ');
select binary upper(a) from t;
select distinct upper(a) from t; 
```

### 2. What did you expect to see? (Required)

```
mysql> select binary upper('ʞ');
+----------------------------------------+
| binary upper('ʞ')                      |
+----------------------------------------+
| 0xEA9EB0                               |
+----------------------------------------+
1 row in set (0.03 sec)

mysql>  select binary upper(a) from t;
+----------------------------------+
| binary upper(a)                  |
+----------------------------------+
| 0xEA9EB0                         |
+----------------------------------+
1 row in set (0.03 sec)

mysql> select distinct upper(a) from t;
+----------+
| upper(a) |
+----------+
| Ʞ        |
+----------+
1 row in set (1.76 sec)
```

### 3. What did you see instead (Required)

```
mysql> select binary upper('ʞ');
+----------------------------------------+
| binary upper('ʞ')                      |
+----------------------------------------+
| 0xEA9EB0                               |
+----------------------------------------+
1 row in set (0.03 sec)

mysql>  select binary upper(a) from t;
+----------------------------------+
| binary upper(a)                  |
+----------------------------------+
| 0xEA9E                           |
+----------------------------------+
1 row in set (0.03 sec)

mysql> select distinct upper(a) from t;
ERROR 1105 (HY000): runtime error: index out of range [2] with length 2
```

### 4. What is your TiDB version? (Required)

<!-- Paste the output of SELECT tidb_version() -->

master. But I think it effects all tidb versions

mysql> SELECT tidb_version();
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| tidb_version()                                                                                                                                                                                                                                |
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Release Version: None
Edition: Community
Git Commit Hash: None
Git Branch: None
UTC Build Time: None
GoVersion: go1.16.3
Race Enabled: false
TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
Check Table Before Drop: false |
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.03 sec)




Discussion:
The root cause is here: https://github.com/pingcap/tidb/blob/171a35486036e8ea5b168c2d4d29f4639df1611d/expression/builtin_string_vec.go#L146-L155

In line 147, we reserved the space according to the original character, but the length of its upper case may larger.
/assign
Similar problems may also occur in the `lower()` function.

```sql
select binary 'İ'; -- 0xC4B0
select binary lower('İ');  -- 0x69
```
Some results of `upper()` and `lower()` are different between TiDB and MySQL, because the result of Golang is based on unicode 13.0.0 [ref](https://github.com/golang/go/blob/851ecea4cc99ab276109493477b2c7e30c253ea8/src/unicode/tables.go#L5).
Maybe we can add a `SpecailCase table` for them or just fix panic problem.

Some of results:
```
...
rune: ꮑ, unicode: 43921, mysql upper: 0xEAAE91, tidb upper: 0xE18F81
rune: ꮒ, unicode: 43922, mysql upper: 0xEAAE92, tidb upper: 0xE18F82
rune: ꮓ, unicode: 43923, mysql upper: 0xEAAE93, tidb upper: 0xE18F83
rune: ꮔ, unicode: 43924, mysql upper: 0xEAAE94, tidb upper: 0xE18F84
rune: ꮕ, unicode: 43925, mysql upper: 0xEAAE95, tidb upper: 0xE18F85
rune: ꮖ, unicode: 43926, mysql upper: 0xEAAE96, tidb upper: 0xE18F86
rune: ꮗ, unicode: 43927, mysql upper: 0xEAAE97, tidb upper: 0xE18F87
...
``` 
> Some results of `upper()` and `lower()` are different between TiDB and MySQL, because the result of Golang is based on unicode 13.0.0 [ref](https://github.com/golang/go/blob/851ecea4cc99ab276109493477b2c7e30c253ea8/src/unicode/tables.go#L5). Maybe we can add a `SpecailCase table` for them or just fix panic problem.
> 
> Some of results:
> 
> ```
> ...
> rune: ꮑ, unicode: 43921, mysql upper: 0xEAAE91, tidb upper: 0xE18F81
> rune: ꮒ, unicode: 43922, mysql upper: 0xEAAE92, tidb upper: 0xE18F82
> rune: ꮓ, unicode: 43923, mysql upper: 0xEAAE93, tidb upper: 0xE18F83
> rune: ꮔ, unicode: 43924, mysql upper: 0xEAAE94, tidb upper: 0xE18F84
> rune: ꮕ, unicode: 43925, mysql upper: 0xEAAE95, tidb upper: 0xE18F85
> rune: ꮖ, unicode: 43926, mysql upper: 0xEAAE96, tidb upper: 0xE18F86
> rune: ꮗ, unicode: 43927, mysql upper: 0xEAAE97, tidb upper: 0xE18F87
> ...
> ```

In my local mysql (version: 8.0.23) , it return s the result:
```
mysql> select binary upper('\U+AB91');
+------------------------------------------+
| binary upper('ꮑ')                        |
+------------------------------------------+
| 0xE18F81                                 |
+------------------------------------------+
1 row in set (0.00 sec)
```

The behavior is the same with tidb. 
My MySQL version is 8.0.28, and I tried it in MySQL 8.0.23. I think the result is based on charset and collation.

```mysql
MySQL:
mysql> set names utf8mb4 collate utf8mb4_bin;
Query OK, 0 rows affected (0.00 sec)

mysql> select binary upper('\U+AB91');
+---------------------+
| binary upper('ꮑ')   |
+---------------------+
| ꮑ                   |
+---------------------+
1 row in set, 1 warning (0.00 sec)

mysql> set names utf8mb4 collate utf8mb4_0900_ai_ci;
Query OK, 0 rows affected (0.00 sec)

mysql> select binary upper('\U+AB91');
+---------------------+
| binary upper('ꮑ')   |
+---------------------+
| Ꮑ                   |
+---------------------+
1 row in set, 1 warning (0.00 sec)
```



Patch:
diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go
index 5484d46f953e5..8232fa36754b1 100644
--- a/expression/builtin_string_vec.go
+++ b/expression/builtin_string_vec.go
@@ -43,12 +43,23 @@ func (b *builtinLowerSig) vectorized() bool {
 }
 
 func (b *builtinLowerUTF8Sig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error {
-	if err := b.args[0].VecEvalString(b.ctx, input, result); err != nil {
+	n := input.NumRows()
+	buf, err := b.bufAllocator.get()
+	if err != nil {
+		return err
+	}
+	defer b.bufAllocator.put(buf)
+	if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil {
 		return err
 	}
+	result.ReserveString(n)
 	enc := charset.FindEncoding(b.args[0].GetType().Charset)
-	for i := 0; i < input.NumRows(); i++ {
-		result.SetRaw(i, []byte(enc.ToLower(result.GetString(i))))
+	for i := 0; i < n; i++ {
+		if buf.IsNull(i) {
+			result.AppendNull()
+		} else {
+			result.AppendString(enc.ToLower(buf.GetString(i)))
+		}
 	}
 	return nil
 }
@@ -144,12 +155,23 @@ func (b *builtinStringIsNullSig) vectorized() bool {
 }
 
 func (b *builtinUpperUTF8Sig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error {
-	if err := b.args[0].VecEvalString(b.ctx, input, result); err != nil {
+	n := input.NumRows()
+	buf, err := b.bufAllocator.get()
+	if err != nil {
+		return err
+	}
+	defer b.bufAllocator.put(buf)
+	if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil {
 		return err
 	}
+	result.ReserveString(n)
 	enc := charset.FindEncoding(b.args[0].GetType().Charset)
-	for i := 0; i < input.NumRows(); i++ {
-		result.SetRaw(i, []byte(enc.ToUpper(result.GetString(i))))
+	for i := 0; i < n; i++ {
+		if buf.IsNull(i) {
+			result.AppendNull()
+		} else {
+			result.AppendString(enc.ToUpper(buf.GetString(i)))
+		}
 	}
 	return nil
 }


Test patch:
diff --git a/expression/integration_test.go b/expression/integration_test.go
index f3f4ee44e32a7..7091f50818c4f 100644
--- a/expression/integration_test.go
+++ b/expression/integration_test.go
@@ -7043,3 +7043,19 @@ func TestIssue22206(t *testing.T) {
 	unixTime = time.Unix(5000000000, 0).In(tz).String()[:19]
 	result.Check(testkit.Rows(unixTime))
 }
+
+func TestIssue32488(t *testing.T) {
+	store, clean := testkit.CreateMockStore(t)
+	defer clean()
+
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t(a varchar(32)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;")
+	tk.MustExec("insert into t values('ʞ'), ('İ');")
+	tk.MustExec("set @@tidb_enable_vectorized_expression = false;")
+	tk.MustQuery("select binary upper(a), lower(a) from t order by upper(a);").Check([][]interface{}{{"İ i"}, {"Ʞ ʞ"}})
+	tk.MustQuery("select distinct upper(a), lower(a) from t order by upper(a);").Check([][]interface{}{{"İ i"}, {"Ʞ ʞ"}})
+	tk.MustExec("set @@tidb_enable_vectorized_expression = true;")
+	tk.MustQuery("select binary upper(a), lower(a) from t order by upper(a);").Check([][]interface{}{{"İ i"}, {"Ʞ ʞ"}})
+	tk.MustQuery("select distinct upper(a), lower(a) from t order by upper(a);").Check([][]interface{}{{"İ i"}, {"Ʞ ʞ"}})
+}