Repository: pingcap/tidb-lightning (go)

Issue:
bug: restore data by mistake in tidb backend mode
## Bug Report

Please answer these questions before submitting your issue. Thanks!

1. What did you do? If possible, provide a recipe for reproducing the error.

loader data into tidb using lightning 

2. What did you expect to see?
```
MySQL [test]> select * from t_access3;
+-------------+
| accessKey   |
+-------------+
| @P&FLASHSHA |
+-------------+
```
3. What did you see instead?
```
MySQL [test]> select * from t_access3;
+-------------+
| accessKey   |
+-------------+
| @PFLASHSHA |
+-------------+
```
4. Versions of the cluster

    - TiDB-Lightning version (run `tidb-lightning -V`):

        ```
        3.0.8
        ```
        
    - TiKV version (run `tikv-server -V`):

        ```
        3.0.8
        ```

    - TiDB cluster version (execute `SELECT tidb_version();` in a MySQL client):

        ```
        3.0.8
        ```


Pull request:
backend: fix issue 282
<!--
Thank you for contributing to TiDB-Lightning! Please read the [CONTRIBUTING](https://github.com/pingcap/tidb-lightning/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->

Fixed #282, caused by a typo.


### What is changed and how it works?

Previously `\Z` was treated as `\x26` (which is `&`). It should be `\x1a` (decimal 26). 

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test
 - Manual test (add detailed scripts or steps below)
     * running with the data dump from #282 works.

Side effects

Related changes

 - Need to cherry-pick to the release branch
 - Need to be included in the release note
    * fixed the bug where the characters `&` is replaced by EOF character in TiDB backend.
    * fixed the bug where `\Z` is replaced by `&`.

Patch:
diff --git a/lightning/backend/tidb.go b/lightning/backend/tidb.go
index 76725a8..70358cf 100644
--- a/lightning/backend/tidb.go
+++ b/lightning/backend/tidb.go
@@ -117,7 +117,7 @@ func (enc tidbEncoder) appendSQLBytes(sb *strings.Builder, value []byte) {
 				sb.WriteString(`\r`)
 			case '\t':
 				sb.WriteString(`\t`)
-			case 0x26:
+			case 26:
 				sb.WriteString(`\Z`)
 			case '\'':
 				sb.WriteString(`''`)
diff --git a/lightning/mydump/parser.go b/lightning/mydump/parser.go
index 7749245..78f7197 100644
--- a/lightning/mydump/parser.go
+++ b/lightning/mydump/parser.go
@@ -270,7 +270,7 @@ func unescape(
 			case 't':
 				return "\t"
 			case 'Z':
-				return "\x26"
+				return "\x1a"
 			default:
 				return substr[1:]
 			}


Test patch:
diff --git a/lightning/backend/tidb_test.go b/lightning/backend/tidb_test.go
index 6648eef..46b3c40 100644
--- a/lightning/backend/tidb_test.go
+++ b/lightning/backend/tidb_test.go
@@ -75,7 +75,7 @@ func (s *mysqlSuite) TestWriteRowsReplaceOnDup(c *C) {
 		types.NewFloat64Datum(5e-324),
 		types.NewFloat64Datum(1.7976931348623157e+308),
 		types.NewFloat64Datum(-0.0),
-		types.NewStringDatum("甲乙丙\r\n\x00\x26'\"\\`"),
+		types.NewStringDatum("甲乙丙\r\n\x00\x1a'\"\\`"),
 		types.NewBinaryLiteralDatum(types.NewBinaryLiteralFromUint(0xabcdef, 6)),
 		types.NewMysqlBitDatum(types.NewBinaryLiteralFromUint(0x98765432, 4)),
 		types.NewDecimalDatum(types.NewDecFromFloatForTest(12.5)),
diff --git a/lightning/mydump/parser_test.go b/lightning/mydump/parser_test.go
index 6268e5d..43236ce 100644
--- a/lightning/mydump/parser_test.go
+++ b/lightning/mydump/parser_test.go
@@ -340,7 +340,7 @@ func (s *testMydumpParserSuite) TestVariousSyntax(c *C) {
 		},
 		{
 			input:    `('\0\b\n\r\t\Z\'\a')`,
-			expected: [][]types.Datum{{types.NewStringDatum("\x00\b\n\r\t\x26'a")}},
+			expected: [][]types.Datum{{types.NewStringDatum("\x00\b\n\r\t\x1a'a")}},
 		},
 		{
 			input:    `(CONVERT("[1,2,3]" USING UTF8MB4))`,
diff --git a/tests/check_requirements/run.sh b/tests/check_requirements/run.sh
index 3a9c686..26d6140 100755
--- a/tests/check_requirements/run.sh
+++ b/tests/check_requirements/run.sh
@@ -31,6 +31,7 @@ reset_cluster_version() {
 trap reset_cluster_version EXIT
 
 curl_cluster_version '{"cluster-version":"2.0.0-fake.and.error.expected"}'
+sleep 1
 
 run_sql 'DROP DATABASE IF EXISTS checkreq'
 set +e
diff --git a/tests/checkpoint_chunks/run.sh b/tests/checkpoint_chunks/run.sh
index 0c6468b..8917b79 100755
--- a/tests/checkpoint_chunks/run.sh
+++ b/tests/checkpoint_chunks/run.sh
@@ -86,9 +86,10 @@ set +e
 i=0
 wait_max_time=20
 while [ $i -lt $wait_max_time ]; do
-    lightning_proc=$(ps -ef|grep '[b]in/tidb-lightning.test')
+    lightning_proc=$(ps -ef|grep "[b]in/tidb-lightning\\.test.*$TEST_NAME")
     ret="$?"
     if [ "$ret" -eq 0 ]; then
+        echo "lightning is still running: $lightning_proc"
         sleep 1
     else
         break
diff --git a/tests/issue_282/config.toml b/tests/issue_282/config.toml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/issue_282/data/issue282-schema-create.sql b/tests/issue_282/data/issue282-schema-create.sql
new file mode 100644
index 0000000..4cdf514
--- /dev/null
+++ b/tests/issue_282/data/issue282-schema-create.sql
@@ -0,0 +1,1 @@
+CREATE DATABASE `issue282` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
diff --git a/tests/issue_282/data/issue282.t_access3-schema.sql b/tests/issue_282/data/issue282.t_access3-schema.sql
new file mode 100644
index 0000000..4c955d4
--- /dev/null
+++ b/tests/issue_282/data/issue282.t_access3-schema.sql
@@ -0,0 +1,4 @@
+/*!40103 SET TIME_ZONE='+00:00' */;
+CREATE TABLE `t_access3` (
+  `accessKey` varchar(100) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
diff --git a/tests/issue_282/data/issue282.t_access3.sql b/tests/issue_282/data/issue282.t_access3.sql
new file mode 100644
index 0000000..2c3e038
--- /dev/null
+++ b/tests/issue_282/data/issue282.t_access3.sql
@@ -0,0 +1,5 @@
+/*!40103 SET TIME_ZONE='+00:00' */;
+INSERT INTO `t_access3` VALUES
+('@P&FLASHSHA');
+INSERT INTO `t_access3` VALUES
+('\Z');
diff --git a/tests/issue_282/run.sh b/tests/issue_282/run.sh
new file mode 100644
index 0000000..ce2a0f9
--- /dev/null
+++ b/tests/issue_282/run.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+for backend in tidb importer; do
+    run_sql 'DROP DATABASE IF EXISTS issue282;'
+    run_lightning --backend $backend
+
+    run_sql "SELECT hex(accessKey) FROM issue282.t_access3"
+    check_contains 'hex(accessKey): 405026464C415348534841'
+    check_contains 'hex(accessKey): 1A'
+done