Repository: pingcap/tidb (Go)

Issue:
Duplicated ORDER BY condition causes bad execution plan
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)

<!-- a step by step guide for reproducing the bug. -->

```
create table t1 (id int unsigned not null auto_increment primary key);
insert into t1 values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
explain analyze select id from t1 where id > 10 order by id asc limit 5;
explain analyze select id from t1 where id > 10 order by id asc, id asc limit 5;
```

### 2. What did you expect to see? (Required)

These should generate the same execution plan since the items in the ORDER BY clause are redundant.

### 3. What did you see instead (Required)

```
mysql> explain analyze select id from t1 where id > 10 order by id asc limit 5;
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
| id                          | estRows | actRows | task      | access object | execution info                                                                                                | operator info                                  | memory    | disk |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
| Limit_11                    | 5.00    | 5       | root      |               | time:738.012µs, loops:2                                                                                       | offset:0, count:5                              | N/A       | N/A  |
| └─TableReader_21            | 5.00    | 5       | root      |               | time:735.732µs, loops:1, cop_task: {num: 1, max:708.291µs, proc_keys: 10, rpc_num: 1, rpc_time: 696.397µs}    | data:Limit_20                                  | 220 Bytes | N/A  |
|   └─Limit_20                | 5.00    | 5       | cop[tikv] |               | time:0s, loops:1                                                                                              | offset:0, count:5                              | N/A       | N/A  |
|     └─TableRangeScan_19     | 6.25    | 10      | cop[tikv] | table:t1      | time:0s, loops:1                                                                                              | range:(10,+inf], keep order:true, stats:pseudo | N/A       | N/A  |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
4 rows in set (0.00 sec)

mysql> explain analyze select id from t1 where id > 10 order by id asc, id asc limit 5;
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
| id                          | estRows | actRows | task      | access object | execution info                                                                                                | operator info                                     | memory    | disk |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
| TopN_8                      | 5.00    | 5       | root      |               | time:462.83µs, loops:2                                                                                        | test.t1.id:asc, test.t1.id:asc, offset:0, count:5 | 200 Bytes | N/A  |
| └─TableReader_16            | 5.00    | 5       | root      |               | time:452.103µs, loops:2, cop_task: {num: 1, max:416.112µs, proc_keys: 10, rpc_num: 1, rpc_time: 409.959µs}    | data:TopN_15                                      | 222 Bytes | N/A  |
|   └─TopN_15                 | 5.00    | 5       | cop[tikv] |               | time:0s, loops:1                                                                                              | test.t1.id:asc, test.t1.id:asc, offset:0, count:5 | N/A       | N/A  |
|     └─TableRangeScan_14     | 8.33    | 10      | cop[tikv] | table:t1      | time:0s, loops:1                                                                                              | range:(10,+inf], keep order:false, stats:pseudo   | N/A       | N/A  |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
4 rows in set (0.00 sec)
```

### 4. What is your TiDB version? (Required)

<!-- Paste the output of SELECT tidb_version() -->

```
tidb_version(): Release Version: v4.0.6
Edition: Community
Git Commit Hash: 51d365fc45fdfc039eb204a96268c5bd1c55075f
Git Branch: heads/v4.0.6
UTC Build Time: 2020-09-15 10:07:22
GoVersion: go1.13.4
Race Enabled: false
TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
Check Table Before Drop: false
1 row in set (0.00 sec)
```



Discussion:
Integrity check:
severity RCA symptom trigger_condition affect_version fix_version fields are empty

Please comment /info to get template
## Please edit this comment to complete the following information

### Bug

#### 1. Root Cause Analysis (RCA)

duplicated `order by` item is not removed.

#### 2. Symptom

the execution plan is changed when adding another same order by item:

```
mysql> explain analyze select id from t1 where id > 10 order by id asc limit 5;
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
| id                          | estRows | actRows | task      | access object | execution info                                                                                                | operator info                                  | memory    | disk |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
| Limit_11                    | 5.00    | 5       | root      |               | time:738.012µs, loops:2                                                                                       | offset:0, count:5                              | N/A       | N/A  |
| └─TableReader_21            | 5.00    | 5       | root      |               | time:735.732µs, loops:1, cop_task: {num: 1, max:708.291µs, proc_keys: 10, rpc_num: 1, rpc_time: 696.397µs}    | data:Limit_20                                  | 220 Bytes | N/A  |
|   └─Limit_20                | 5.00    | 5       | cop[tikv] |               | time:0s, loops:1                                                                                              | offset:0, count:5                              | N/A       | N/A  |
|     └─TableRangeScan_19     | 6.25    | 10      | cop[tikv] | table:t1      | time:0s, loops:1                                                                                              | range:(10,+inf], keep order:true, stats:pseudo | N/A       | N/A  |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+------------------------------------------------+-----------+------+
4 rows in set (0.00 sec)

mysql> explain analyze select id from t1 where id > 10 order by id asc, id asc limit 5;
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
| id                          | estRows | actRows | task      | access object | execution info                                                                                                | operator info                                     | memory    | disk |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
| TopN_8                      | 5.00    | 5       | root      |               | time:462.83µs, loops:2                                                                                        | test.t1.id:asc, test.t1.id:asc, offset:0, count:5 | 200 Bytes | N/A  |
| └─TableReader_16            | 5.00    | 5       | root      |               | time:452.103µs, loops:2, cop_task: {num: 1, max:416.112µs, proc_keys: 10, rpc_num: 1, rpc_time: 409.959µs}    | data:TopN_15                                      | 222 Bytes | N/A  |
|   └─TopN_15                 | 5.00    | 5       | cop[tikv] |               | time:0s, loops:1                                                                                              | test.t1.id:asc, test.t1.id:asc, offset:0, count:5 | N/A       | N/A  |
|     └─TableRangeScan_14     | 8.33    | 10      | cop[tikv] | table:t1      | time:0s, loops:1                                                                                              | range:(10,+inf], keep order:false, stats:pseudo   | N/A       | N/A  |
+-----------------------------+---------+---------+-----------+---------------+---------------------------------------------------------------------------------------------------------------+---------------------------------------------------+-----------+------+
4 rows in set (0.00 sec)
```

#### 3. All Trigger Conditions

there are duplicated expressions in the `order by` clause, like this:

```sql
select id
from t1
where id > 10
order by id asc,
         id asc
limit 5;
```

#### 4. Workaround (optional)

remove the duplicated `order by` items in the SQL.

#### 5. Affected versions

[v3.0.0:v3.0.19], [v4.0.0:v4.0.7]

#### 6. Fixed versions

v4.0.8



Patch:
diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go
index 12125040939f0..d482e41724e0b 100644
--- a/planner/core/rule_column_pruning.go
+++ b/planner/core/rule_column_pruning.go
@@ -137,9 +137,15 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column)
 
 func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*expression.Column) {
 	new = make([]*util.ByItems, 0, len(old))
+	seen := make(map[string]struct{}, len(old))
 	for _, byItem := range old {
+		hash := string(byItem.Expr.HashCode(nil))
+		_, hashMatch := seen[hash]
+		seen[hash] = struct{}{}
 		cols := expression.ExtractColumns(byItem.Expr)
-		if len(cols) == 0 {
+		if hashMatch {
+			// do nothing, should be filtered
+		} else if len(cols) == 0 {
 			if !expression.IsRuntimeConstExpr(byItem.Expr) {
 				new = append(new, byItem)
 			}


Test patch:
diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go
index ce26c680af33f..704af732ea96d 100644
--- a/planner/core/logical_plan_test.go
+++ b/planner/core/logical_plan_test.go
@@ -553,6 +553,31 @@ func (s *testPlanSuite) TestColumnPruning(c *C) {
 	}
 }
 
+func (s *testPlanSuite) TestSortByItemsPruning(c *C) {
+	defer testleak.AfterTest(c)()
+	var (
+		input  []string
+		output [][]string
+	)
+	s.testData.GetTestCases(c, &input, &output)
+	s.testData.OnRecord(func() {
+		output = make([][]string, len(input))
+	})
+
+	ctx := context.Background()
+	for i, tt := range input {
+		comment := Commentf("for %s", tt)
+		stmt, err := s.ParseOneStmt(tt, "", "")
+		c.Assert(err, IsNil, comment)
+
+		p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is)
+		c.Assert(err, IsNil)
+		lp, err := logicalOptimize(ctx, flagEliminateProjection|flagPredicatePushDown|flagPrunColumns|flagPrunColumnsAgain, p.(LogicalPlan))
+		c.Assert(err, IsNil)
+		s.checkOrderByItems(lp, c, &output[i], comment)
+	}
+}
+
 func (s *testPlanSuite) TestProjectionEliminator(c *C) {
 	defer testleak.AfterTest(c)()
 	tests := []struct {
@@ -620,6 +645,27 @@ func (s *testPlanSuite) checkDataSourceCols(p LogicalPlan, c *C, ans map[int][]s
 	}
 }
 
+func (s *testPlanSuite) checkOrderByItems(p LogicalPlan, c *C, colList *[]string, comment CommentInterface) {
+	switch p := p.(type) {
+	case *LogicalSort:
+		s.testData.OnRecord(func() {
+			*colList = make([]string, len(p.ByItems))
+		})
+		for i, col := range p.ByItems {
+			s.testData.OnRecord(func() {
+				(*colList)[i] = col.String()
+			})
+			s := col.String()
+			c.Assert(s, Equals, (*colList)[i], comment)
+		}
+	}
+	children := p.Children()
+	c.Assert(len(children), LessEqual, 1, Commentf("For %v Expected <= 1 Child", comment))
+	for _, child := range children {
+		s.checkOrderByItems(child, c, colList, comment)
+	}
+}
+
 func (s *testPlanSuite) TestValidate(c *C) {
 	defer testleak.AfterTest(c)()
 	tests := []struct {
diff --git a/planner/core/testdata/plan_suite_unexported_in.json b/planner/core/testdata/plan_suite_unexported_in.json
index b71d9f9960da7..41aef7bec4ccc 100644
--- a/planner/core/testdata/plan_suite_unexported_in.json
+++ b/planner/core/testdata/plan_suite_unexported_in.json
@@ -402,6 +402,14 @@
       "select count(1) from (select count(b) as cnt from t group by c) t1"
     ]
   },
+  {
+    "name": "TestSortByItemsPruning",
+    "cases": [
+      "select * from t where a > 1 order by a asc, a asc limit 10",
+      "select * from t where a > 1 order by a asc, b asc, a asc, c asc limit 10",
+      "select * from t where a > 1 order by pow(a, 2) asc, b asc, pow(a, 2) asc, c asc limit 10"
+    ]
+  },
   {
     "name": "TestDeriveNotNullConds",
     "cases": [
diff --git a/planner/core/testdata/plan_suite_unexported_out.json b/planner/core/testdata/plan_suite_unexported_out.json
index ade08e163fe7e..699b0d3214326 100644
--- a/planner/core/testdata/plan_suite_unexported_out.json
+++ b/planner/core/testdata/plan_suite_unexported_out.json
@@ -739,6 +739,24 @@
       }
     ]
   },
+  {
+    "Name": "TestSortByItemsPruning",
+    "Cases": [
+      [
+        "test.t.a"
+      ],
+      [
+        "test.t.a",
+        "test.t.b",
+        "test.t.c"
+      ],
+      [
+        "pow(cast(test.t.a, double BINARY), 2)",
+        "test.t.b",
+        "test.t.c"
+      ]
+    ]
+  },
   {
     "Name": "TestDeriveNotNullConds",
     "Cases": [