Repository: pingcap/tidb-lightning
Pull request #420 (2020-10-13 07:34:46)

backend: fix a bug about wrong column info
<!--
Thank you for contributing to TiDB-Lightning! Please read the [CONTRIBUTING](https://github.com/pingcap/tidb-lightning/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->
fix a bug about wrong column info

### What is changed and how it works?


### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test

kennytm: LGTM after fixing overvenus's comment.
leoppro: /run-all-tests
glorv: /run-all-tests
glorv: /run-all-tests
glorv: /run-all-tests
kennytm: https://internal.pingcap.net/idc-jenkins/blue/organizations/jenkins/lightning_ghpr_test/detail/lightning_ghpr_test/2287/pipeline

```
[2020-10-22T08:57:36.890Z] Import using importer finished
[2020-10-22T08:57:37.149Z] TEST FAILED: OUTPUT DOES NOT CONTAIN 'set: x00,x06,x07,x09,x17,x20,x23,x24,x27,x37,x44,x46,x49,x54,x55,x58,x61,x62'
[2020-10-22T08:57:37.149Z] ____________________________________
[2020-10-22T08:57:37.149Z] [Thu Oct 22 16:57:36 CST 2020] Executing SQL: SELECT `set` FROM vt.`enum-set` WHERE `enum` = "gcc"
[2020-10-22T08:57:37.149Z] *************************** 1. row ***************************
[2020-10-22T08:57:37.149Z] set: x10,x17,x20,x23,x24,x27,x37,x44,x46,x49,x54,x55,x58,x61,x62
[2020-10-22T08:57:37.149Z] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
```

This looks broken.

Patch:
diff --git a/lightning/backend/tidb.go b/lightning/backend/tidb.go
index 4230757..dbbcf1b 100644
--- a/lightning/backend/tidb.go
+++ b/lightning/backend/tidb.go
@@ -366,7 +366,7 @@ func (be *tidbBackend) FetchRemoteTableModels(schemaName string) (tables []*mode
 			SELECT table_name, column_name
 			FROM information_schema.columns
 			WHERE table_schema = ?
-			ORDER BY table_name;
+			ORDER BY table_name, ordinal_position;
 		`, schemaName)
 		if e != nil {
 			return e


Test patch:
diff --git a/tests/too_many_columns/config.toml b/tests/too_many_columns/config.toml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/too_many_columns/data/too_many_columns-schema-create.sql b/tests/too_many_columns/data/too_many_columns-schema-create.sql
new file mode 100644
index 0000000..0264589
--- /dev/null
+++ b/tests/too_many_columns/data/too_many_columns-schema-create.sql
@@ -0,0 +1,1 @@
+create database too_many_columns;
diff --git a/tests/too_many_columns/data/too_many_columns.t-schema.sql b/tests/too_many_columns/data/too_many_columns.t-schema.sql
new file mode 100644
index 0000000..c21436d
--- /dev/null
+++ b/tests/too_many_columns/data/too_many_columns.t-schema.sql
@@ -0,0 +1,259 @@
+create table t
+(
+    COL001 VARCHAR(16),
+    COL002 VARCHAR(16),
+    COL003 VARCHAR(16),
+    COL004 VARCHAR(16),
+    COL005 VARCHAR(16),
+    COL006 VARCHAR(16),
+    COL007 VARCHAR(16),
+    COL008 VARCHAR(16),
+    COL009 VARCHAR(16),
+    COL010 VARCHAR(16),
+    COL011 VARCHAR(16),
+    COL012 VARCHAR(16),
+    COL013 VARCHAR(16),
+    COL014 VARCHAR(16),
+    COL015 VARCHAR(16),
+    COL016 VARCHAR(16),
+    COL017 VARCHAR(16),
+    COL018 VARCHAR(16),
+    COL019 VARCHAR(16),
+    COL020 VARCHAR(16),
+    COL021 VARCHAR(16),
+    COL022 VARCHAR(16),
+    COL023 VARCHAR(16),
+    COL024 VARCHAR(16),
+    COL025 VARCHAR(16),
+    COL026 VARCHAR(16),
+    COL027 VARCHAR(16),
+    COL028 VARCHAR(16),
+    COL029 VARCHAR(16),
+    COL030 VARCHAR(16),
+    COL031 VARCHAR(16),
+    COL032 VARCHAR(16),
+    COL033 VARCHAR(16),
+    COL034 VARCHAR(16),
+    COL035 VARCHAR(16),
+    COL036 VARCHAR(16),
+    COL037 VARCHAR(16),
+    COL038 VARCHAR(16),
+    COL039 VARCHAR(16),
+    COL040 VARCHAR(16),
+    COL041 VARCHAR(16),
+    COL042 VARCHAR(16),
+    COL043 VARCHAR(16),
+    COL044 VARCHAR(16),
+    COL045 VARCHAR(16),
+    COL046 VARCHAR(16),
+    COL047 VARCHAR(16),
+    COL048 VARCHAR(16),
+    COL049 VARCHAR(16),
+    COL050 VARCHAR(16),
+    COL051 VARCHAR(16),
+    COL052 VARCHAR(16),
+    COL053 VARCHAR(16),
+    COL054 VARCHAR(16),
+    COL055 VARCHAR(16),
+    COL056 VARCHAR(16),
+    COL057 VARCHAR(16),
+    COL058 VARCHAR(16),
+    COL059 VARCHAR(16),
+    COL060 VARCHAR(16),
+    COL061 VARCHAR(16),
+    COL062 VARCHAR(16),
+    COL063 VARCHAR(16),
+    COL064 VARCHAR(16),
+    COL065 VARCHAR(16),
+    COL066 VARCHAR(16),
+    COL067 VARCHAR(16),
+    COL068 VARCHAR(16),
+    COL069 VARCHAR(16),
+    COL070 VARCHAR(16),
+    COL071 VARCHAR(16),
+    COL072 VARCHAR(16),
+    COL073 VARCHAR(16),
+    COL074 VARCHAR(16),
+    COL075 VARCHAR(16),
+    COL076 VARCHAR(16),
+    COL077 VARCHAR(16),
+    COL078 VARCHAR(16),
+    COL079 VARCHAR(16),
+    COL080 VARCHAR(16),
+    COL081 VARCHAR(16),
+    COL082 VARCHAR(16),
+    COL083 VARCHAR(16),
+    COL084 VARCHAR(16),
+    COL085 VARCHAR(16),
+    COL086 VARCHAR(16),
+    COL087 VARCHAR(16),
+    COL088 VARCHAR(16),
+    COL089 VARCHAR(16),
+    COL090 VARCHAR(16),
+    COL091 VARCHAR(16),
+    COL092 VARCHAR(16),
+    COL093 VARCHAR(16),
+    COL094 VARCHAR(16),
+    COL095 VARCHAR(16),
+    COL096 VARCHAR(16),
+    COL097 VARCHAR(16),
+    COL098 VARCHAR(16),
+    COL099 VARCHAR(16),
+    COL100 VARCHAR(16),
+    COL101 VARCHAR(16),
+    COL102 VARCHAR(16),
+    COL103 VARCHAR(16),
+    COL104 VARCHAR(16),
+    COL105 VARCHAR(16),
+    COL106 VARCHAR(16),
+    COL107 VARCHAR(16),
+    COL108 VARCHAR(16),
+    COL109 VARCHAR(16),
+    COL110 VARCHAR(16),
+    COL111 VARCHAR(16),
+    COL112 VARCHAR(16),
+    COL113 VARCHAR(16),
+    COL114 VARCHAR(16),
+    COL115 VARCHAR(16),
+    COL116 VARCHAR(16),
+    COL117 VARCHAR(16),
+    COL118 VARCHAR(16),
+    COL119 VARCHAR(16),
+    COL120 VARCHAR(16),
+    COL121 VARCHAR(16),
+    COL122 VARCHAR(16),
+    COL123 VARCHAR(16),
+    COL124 VARCHAR(16),
+    COL125 VARCHAR(16),
+    COL126 VARCHAR(16),
+    COL127 VARCHAR(16),
+    COL128 VARCHAR(16),
+    COL129 VARCHAR(16),
+    COL130 VARCHAR(16),
+    COL131 VARCHAR(16),
+    COL132 VARCHAR(16),
+    COL133 VARCHAR(16),
+    COL134 VARCHAR(16),
+    COL135 VARCHAR(16),
+    COL136 VARCHAR(16),
+    COL137 VARCHAR(16),
+    COL138 VARCHAR(16),
+    COL139 VARCHAR(16),
+    COL140 VARCHAR(16),
+    COL141 VARCHAR(16),
+    COL142 VARCHAR(16),
+    COL143 VARCHAR(16),
+    COL144 VARCHAR(16),
+    COL145 VARCHAR(16),
+    COL146 VARCHAR(16),
+    COL147 VARCHAR(16),
+    COL148 VARCHAR(16),
+    COL149 VARCHAR(16),
+    COL150 VARCHAR(16),
+    COL151 VARCHAR(16),
+    COL152 VARCHAR(16),
+    COL153 VARCHAR(16),
+    COL154 VARCHAR(16),
+    COL155 VARCHAR(16),
+    COL156 VARCHAR(16),
+    COL157 VARCHAR(16),
+    COL158 VARCHAR(16),
+    COL159 VARCHAR(16),
+    COL160 VARCHAR(16),
+    COL161 VARCHAR(16),
+    COL162 VARCHAR(16),
+    COL163 VARCHAR(16),
+    COL164 VARCHAR(16),
+    COL165 VARCHAR(16),
+    COL166 VARCHAR(16),
+    COL167 VARCHAR(16),
+    COL168 VARCHAR(16),
+    COL169 VARCHAR(16),
+    COL170 VARCHAR(16),
+    COL171 VARCHAR(16),
+    COL172 VARCHAR(16),
+    COL173 VARCHAR(16),
+    COL174 VARCHAR(16),
+    COL175 VARCHAR(16),
+    COL176 VARCHAR(16),
+    COL177 VARCHAR(16),
+    COL178 VARCHAR(16),
+    COL179 VARCHAR(16),
+    COL180 VARCHAR(16),
+    COL181 VARCHAR(16),
+    COL182 VARCHAR(16),
+    COL183 VARCHAR(16),
+    COL184 VARCHAR(16),
+    COL185 VARCHAR(16),
+    COL186 VARCHAR(16),
+    COL187 VARCHAR(16),
+    COL188 VARCHAR(16),
+    COL189 VARCHAR(16),
+    COL190 VARCHAR(16),
+    COL191 VARCHAR(16),
+    COL192 VARCHAR(16),
+    COL193 VARCHAR(16),
+    COL194 VARCHAR(16),
+    COL195 VARCHAR(16),
+    COL196 VARCHAR(16),
+    COL197 VARCHAR(16),
+    COL198 VARCHAR(16),
+    COL199 VARCHAR(16),
+    COL200 VARCHAR(16),
+    COL201 VARCHAR(16),
+    COL202 VARCHAR(16),
+    COL203 VARCHAR(16),
+    COL204 VARCHAR(16),
+    COL205 VARCHAR(16),
+    COL206 VARCHAR(16),
+    COL207 VARCHAR(16),
+    COL208 VARCHAR(16),
+    COL209 VARCHAR(16),
+    COL210 VARCHAR(16),
+    COL211 VARCHAR(16),
+    COL212 VARCHAR(16),
+    COL213 VARCHAR(16),
+    COL214 VARCHAR(16),
+    COL215 VARCHAR(16),
+    COL216 VARCHAR(16),
+    COL217 VARCHAR(16),
+    COL218 VARCHAR(16),
+    COL219 VARCHAR(16),
+    COL220 VARCHAR(16),
+    COL221 VARCHAR(16),
+    COL222 VARCHAR(16),
+    COL223 VARCHAR(16),
+    COL224 VARCHAR(16),
+    COL225 VARCHAR(16),
+    COL226 VARCHAR(16),
+    COL227 VARCHAR(16),
+    COL228 VARCHAR(16),
+    COL229 VARCHAR(16),
+    COL230 VARCHAR(16),
+    COL231 VARCHAR(16),
+    COL232 VARCHAR(16),
+    COL233 VARCHAR(16),
+    COL234 VARCHAR(16),
+    COL235 VARCHAR(16),
+    COL236 VARCHAR(16),
+    COL237 VARCHAR(16),
+    COL238 VARCHAR(16),
+    COL239 VARCHAR(16),
+    COL240 VARCHAR(16),
+    COL241 VARCHAR(16),
+    COL242 VARCHAR(16),
+    COL243 VARCHAR(16),
+    COL244 VARCHAR(16),
+    COL245 VARCHAR(16),
+    COL246 VARCHAR(16),
+    COL247 VARCHAR(16),
+    COL248 VARCHAR(16),
+    COL249 VARCHAR(16),
+    COL250 VARCHAR(16),
+    COL251 VARCHAR(16),
+    COL252 VARCHAR(16),
+    COL253 VARCHAR(16),
+    COL254 VARCHAR(16),
+    COL255 VARCHAR(16),
+    COL256 VARCHAR(16)
+);
diff --git a/tests/too_many_columns/data/too_many_columns.t.0.csv b/tests/too_many_columns/data/too_many_columns.t.0.csv
new file mode 100644
index 0000000..e129b7b
--- /dev/null
+++ b/tests/too_many_columns/data/too_many_columns.t.0.csv
@@ -0,0 +1,2 @@
+COL001,COL002,COL003,COL004,COL005,COL006,COL007,COL008,COL009,COL010,COL011,COL012,COL013,COL014,COL015,COL016,COL017,COL018,COL019,COL020,COL021,COL022,COL023,COL024,COL025,COL026,COL027,COL028,COL029,COL030,COL031,COL032,COL033,COL034,COL035,COL036,COL037,COL038,COL039,COL040,COL041,COL042,COL043,COL044,COL045,COL046,COL047,COL048,COL049,COL050,COL051,COL052,COL053,COL054,COL055,COL056,COL057,COL058,COL059,COL060,COL061,COL062,COL063,COL064,COL065,COL066,COL067,COL068,COL069,COL070,COL071,COL072,COL073,COL074,COL075,COL076,COL077,COL078,COL079,COL080,COL081,COL082,COL083,COL084,COL085,COL086,COL087,COL088,COL089,COL090,COL091,COL092,COL093,COL094,COL095,COL096,COL097,COL098,COL099,COL100,COL101,COL102,COL103,COL104,COL105,COL106,COL107,COL108,COL109,COL110,COL111,COL112,COL113,COL114,COL115,COL116,COL117,COL118,COL119,COL120,COL121,COL122,COL123,COL124,COL125,COL126,COL127,COL128,COL129,COL130,COL131,COL132,COL133,COL134,COL135,COL136,COL137,COL138,COL139,COL140,COL141,COL142,COL143,COL144,COL145,COL146,COL147,COL148,COL149,COL150,COL151,COL152,COL153,COL154,COL155,COL156,COL157,COL158,COL159,COL160,COL161,COL162,COL163,COL164,COL165,COL166,COL167,COL168,COL169,COL170,COL171,COL172,COL173,COL174,COL175,COL176,COL177,COL178,COL179,COL180,COL181,COL182,COL183,COL184,COL185,COL186,COL187,COL188,COL189,COL190,COL191,COL192,COL193,COL194,COL195,COL196,COL197,COL198,COL199,COL200,COL201,COL202,COL203,COL204,COL205,COL206,COL207,COL208,COL209,COL210,COL211,COL212,COL213,COL214,COL215,COL216,COL217,COL218,COL219,COL220,COL221,COL222,COL223,COL224,COL225,COL226,COL227,COL228,COL229,COL230,COL231,COL232,COL233,COL234,COL235,COL236,COL237,COL238,COL239,COL240,COL241,COL242,COL243,COL244,COL245,COL246,COL247,COL248,COL249,COL250,COL251,COL252,COL253,COL254,COL255,COL256
+1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256
\ No newline at end of file
diff --git a/tests/too_many_columns/run.sh b/tests/too_many_columns/run.sh
new file mode 100644
index 0000000..22a5f47
--- /dev/null
+++ b/tests/too_many_columns/run.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+for backend in tidb importer local; do
+    if [ "$backend" = 'local' ]; then
+        check_cluster_version 4 0 0 'local backend' || continue
+    fi
+
+    run_sql 'DROP DATABASE IF EXISTS too_many_columns;'
+    run_lightning --backend $backend
+
+    run_sql "SELECT * FROM too_many_columns.t"
+    check_contains 'COL001: 1001'
+    check_contains 'COL256: 1256'
+    check_contains 'COL100: 1100'
+done