Repository: pingcap/tidb (Go)

Issue:
enable Selectivity() to handle constants
## Enhancement
Ref: #32758

Currently, the `Selectivity()` function is unable to handle constants like `1`, `0`, `true`, `false`, and will use 0.8 as the selectivity.
Most times this won't cause severe consequences. But for `Selection` that is always true and `TableDual`, this behavior makes the estimation results very unreasonable.


Discussion:

Patch:
diff --git a/statistics/selectivity.go b/statistics/selectivity.go
index 0f5f4792acc71..5fc1f1dcc2b3b 100644
--- a/statistics/selectivity.go
+++ b/statistics/selectivity.go
@@ -317,6 +317,34 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp
 		}
 	}
 
+	// Try to cover Constants
+	if mask > 0 {
+		for i, expr := range remainedExprs {
+			if mask&(1<<uint64(i)) == 0 {
+				continue
+			}
+			if c, ok := expr.(*expression.Constant); ok {
+				if expression.MaybeOverOptimized4PlanCache(ctx, []expression.Expression{c}) {
+					continue
+				}
+				if c.Value.IsNull() {
+					// c is null
+					ret *= 0
+					mask &^= 1 << uint64(i)
+				} else if isTrue, err := c.Value.ToBool(sc); err == nil {
+					if isTrue == 0 {
+						// c is false
+						ret *= 0
+					}
+					// c is true, no need to change ret
+					mask &^= 1 << uint64(i)
+				}
+				// Not expected to come here:
+				// err != nil, no need to do anything.
+			}
+		}
+	}
+
 	// Now we try to cover those still not covered DNF conditions using independence assumption,
 	// i.e., sel(condA or condB) = sel(condA) + sel(condB) - sel(condA) * sel(condB)
 	if mask > 0 {
@@ -354,24 +382,6 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp
 				if ok {
 					continue
 				}
-				// where {"0" / 0 / "false" / false / null} or A or B ... the '0' constant item should be ignored.
-				if c, ok := cond.(*expression.Constant); ok {
-					if !expression.MaybeOverOptimized4PlanCache(ctx, []expression.Expression{cond}) {
-						if c.Value.IsNull() {
-							// constant is null
-							continue
-						}
-						if isTrue, err := c.Value.ToBool(sc); err == nil {
-							if isTrue == 0 {
-								// constant == 0
-								continue
-							}
-							// constant == 1
-							selectivity = 1.0
-							break
-						}
-					}
-				}
 
 				var cnfItems []expression.Expression
 				if scalar, ok := cond.(*expression.ScalarFunction); ok && scalar.FuncName.L == ast.LogicAnd {


Test patch:
diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result
index e3d082d5c7542..766b8760a2faf 100644
--- a/cmd/explaintest/r/explain_easy.result
+++ b/cmd/explaintest/r/explain_easy.result
@@ -433,10 +433,10 @@ IndexReader	10.00	root		index:IndexRangeScan
 └─IndexRangeScan	10.00	cop[tikv]	table:t, index:idx(a, b)	range:[1,1], keep order:false, stats:pseudo
 explain format = 'brief' select * from t where a = 1 and a = 2;
 id	estRows	task	access object	operator info
-TableDual	8000.00	root		rows:0
+TableDual	0.00	root		rows:0
 explain format = 'brief' select * from t where b = 1 and b = 2;
 id	estRows	task	access object	operator info
-TableDual	8000.00	root		rows:0
+TableDual	0.00	root		rows:0
 explain format = 'brief' select * from t t1 join t t2 where t1.b = t2.b and t2.b is null;
 id	estRows	task	access object	operator info
 Projection	0.00	root		test.t.a, test.t.b, test.t.a, test.t.b
@@ -458,7 +458,7 @@ drop table if exists t;
 create table t(a bigint primary key);
 explain format = 'brief' select * from t where a = 1 and a = 2;
 id	estRows	task	access object	operator info
-TableDual	8000.00	root		rows:0
+TableDual	0.00	root		rows:0
 explain format = 'brief' select null or a > 1 from t;
 id	estRows	task	access object	operator info
 Projection	10000.00	root		or(<nil>, gt(test.t.a, 1))->Column#2
@@ -506,17 +506,17 @@ PRIMARY KEY (`id`)
 explain format = 'brief' SELECT COUNT(1) FROM (SELECT COALESCE(b.region_name, '不详') region_name, SUM(a.registration_num) registration_num FROM (SELECT stat_date, show_date, region_id, 0 registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202 UNION ALL SELECT stat_date, show_date, region_id, registration_num registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202) a LEFT JOIN test02 b ON a.region_id = b.id WHERE registration_num > 0 AND a.stat_date >= '20191202' AND a.stat_date <= '20191202' GROUP BY a.stat_date , a.show_date , COALESCE(b.region_name, '不详') ) JLS;
 id	estRows	task	access object	operator info
 StreamAgg	1.00	root		funcs:count(1)->Column#22
-└─HashAgg	8000.00	root		group by:Column#32, Column#33, Column#34, funcs:count(1)->Column#31
-  └─Projection	10000.01	root		Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#34
-    └─HashJoin	10000.01	root		left outer join, equal:[eq(Column#16, test.test02.id)]
-      ├─TableReader(Build)	10000.00	root		data:TableFullScan
-      │ └─TableFullScan	10000.00	cop[tikv]	table:b	keep order:false, stats:pseudo
-      └─Union(Probe)	8000.01	root		
-        ├─TableDual	8000.00	root		rows:0
-        └─Projection	0.01	root		test.test01.stat_date, test.test01.show_date, test.test01.region_id
-          └─TableReader	0.01	root		data:Selection
-            └─Selection	0.01	cop[tikv]		eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), gt(cast(test.test01.registration_num, bigint(20) BINARY), 0), le(test.test01.stat_date, 20191202)
-              └─TableFullScan	10000.00	cop[tikv]	table:test01	keep order:false, stats:pseudo
+└─HashAgg	1.00	root		group by:Column#32, Column#33, Column#34, funcs:count(1)->Column#31
+  └─Projection	0.01	root		Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#34
+    └─IndexJoin	0.01	root		left outer join, inner:TableReader, outer key:Column#16, inner key:test.test02.id, equal cond:eq(Column#16, test.test02.id)
+      ├─Union(Build)	0.01	root		
+      │ ├─TableDual	0.00	root		rows:0
+      │ └─Projection	0.01	root		test.test01.stat_date, test.test01.show_date, test.test01.region_id
+      │   └─TableReader	0.01	root		data:Selection
+      │     └─Selection	0.01	cop[tikv]		eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), gt(cast(test.test01.registration_num, bigint(20) BINARY), 0), le(test.test01.stat_date, 20191202)
+      │       └─TableFullScan	10000.00	cop[tikv]	table:test01	keep order:false, stats:pseudo
+      └─TableReader(Probe)	1.00	root		data:TableRangeScan
+        └─TableRangeScan	1.00	cop[tikv]	table:b	range: decided by [Column#16], keep order:false, stats:pseudo
 drop table if exists t;
 create table t(a int, nb int not null, nc int not null);
 explain format = 'brief' select ifnull(a, 0) from t;
diff --git a/cmd/explaintest/r/imdbload.result b/cmd/explaintest/r/imdbload.result
index 1fd23fdc6491b..33294bc544a29 100644
--- a/cmd/explaintest/r/imdbload.result
+++ b/cmd/explaintest/r/imdbload.result
@@ -279,6 +279,15 @@ id	estRows	task	access object	operator info
 TableReader_7	804024.75	root		data:Selection_6
 └─Selection_6	804024.75	cop[tikv]		or(and(eq(imdbload.char_name.imdb_index, "I"), lt(imdbload.char_name.surname_pcode, "E436")), and(eq(imdbload.char_name.imdb_index, "L"), lt(imdbload.char_name.surname_pcode, "E436")))
   └─TableFullScan_5	4314864.00	cop[tikv]	table:char_name	keep order:false
+explain select * from char_name use index (itest2) where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'));
+id	estRows	task	access object	operator info
+IndexLookUp_7	2010061.87	root		
+├─IndexRangeScan_5(Build)	2010061.87	cop[tikv]	table:char_name, index:itest2(imdb_index, surname_pcode, name_pcode_nf)	range:["I" -inf,"I" "E436"), ["L" -inf,"L" "E436"), keep order:false
+└─TableRowIDScan_6(Probe)	2010061.87	cop[tikv]	table:char_name	keep order:false
+trace plan target = 'estimation' select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'));
+CE_trace
+[{"table_name":"char_name","type":"Column Stats-Range","expr":"((id >= -9223372036854775808 and id <= 9223372036854775807))","row_count":4314864},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'))","row_count":2010061},{"table_name":"char_name","type":"Index Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Table Stats-Expression-CNF","expr":"`or`(`and`(`eq`(imdbload.char_name.imdb_index, 'I'), `lt`(imdbload.char_name.surname_pcode, 'E436')), `and`(`eq`(imdbload.char_name.imdb_index, 'L'), `lt`(imdbload.char_name.surname_pcode, 'E436')))","row_count":804024}]
+
 explain select * from char_name where ((imdb_index = 'V') and (surname_pcode < 'L3416'));
 id	estRows	task	access object	operator info
 TableReader_7	1927106.39	root		data:Selection_6
diff --git a/cmd/explaintest/t/imdbload.test b/cmd/explaintest/t/imdbload.test
index 70e4af321dbdd..ba5f090d08e68 100644
--- a/cmd/explaintest/t/imdbload.test
+++ b/cmd/explaintest/t/imdbload.test
@@ -281,6 +281,8 @@ load stats 's/imdbload_stats/cast_info.json';
 
 -- Actual row count: 1
 explain select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'));
+explain select * from char_name use index (itest2) where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'));
+trace plan target = 'estimation' select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'));
 
 -- Actual row count: 0
 explain select * from char_name where ((imdb_index = 'V') and (surname_pcode < 'L3416'));
diff --git a/executor/prepared_test.go b/executor/prepared_test.go
index ab251b9e4853d..570d46ed49955 100644
--- a/executor/prepared_test.go
+++ b/executor/prepared_test.go
@@ -168,7 +168,7 @@ func TestPreparedNullParam(t *testing.T) {
 		ps := []*util.ProcessInfo{tkProcess}
 		tk.Session().SetSessionManager(&mockSessionManager1{PS: ps})
 		tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows(
-			"TableDual_5 8000.00 root  rows:0"))
+			"TableDual_5 0.00 root  rows:0"))
 	}
 }
 
diff --git a/expression/testdata/expression_suite_out.json b/expression/testdata/expression_suite_out.json
index 8b89ee1e4dc81..7047b62ba1156 100644
--- a/expression/testdata/expression_suite_out.json
+++ b/expression/testdata/expression_suite_out.json
@@ -197,8 +197,8 @@
       {
         "SQL": "explain format = 'brief' select * from t1 left join t2 on false",
         "Result": [
-          "HashJoin 80000000.00 root  CARTESIAN left outer join",
-          "├─TableDual(Build) 8000.00 root  rows:0",
+          "HashJoin 10000.00 root  CARTESIAN left outer join",
+          "├─TableDual(Build) 0.00 root  rows:0",
           "└─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "  └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ]
@@ -206,8 +206,8 @@
       {
         "SQL": "explain format = 'brief' select * from t1 right join t2 on false",
         "Result": [
-          "HashJoin 80000000.00 root  CARTESIAN right outer join",
-          "├─TableDual(Build) 8000.00 root  rows:0",
+          "HashJoin 10000.00 root  CARTESIAN right outer join",
+          "├─TableDual(Build) 0.00 root  rows:0",
           "└─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "  └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo"
         ]
@@ -215,8 +215,8 @@
       {
         "SQL": "explain format = 'brief' select * from t1 left join t2 on t1.a = 1 and t1.a = 2",
         "Result": [
-          "HashJoin 80000000.00 root  CARTESIAN left outer join",
-          "├─TableDual(Build) 8000.00 root  rows:0",
+          "HashJoin 10000.00 root  CARTESIAN left outer join",
+          "├─TableDual(Build) 0.00 root  rows:0",
           "└─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "  └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ]
@@ -224,18 +224,18 @@
       {
         "SQL": "explain format = 'brief' select * from t1 left join t2 on t1.a =1 where t1.a = 2",
         "Result": [
-          "HashJoin 80000.00 root  CARTESIAN left outer join",
-          "├─TableReader(Build) 10.00 root  data:Selection",
-          "│ └─Selection 10.00 cop[tikv]  eq(test.t1.a, 2)",
-          "│   └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
-          "└─TableDual(Probe) 8000.00 root  rows:0"
+          "HashJoin 10.00 root  CARTESIAN left outer join",
+          "├─TableDual(Build) 0.00 root  rows:0",
+          "└─TableReader(Probe) 10.00 root  data:Selection",
+          "  └─Selection 10.00 cop[tikv]  eq(test.t1.a, 2)",
+          "    └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ]
       },
       {
         "SQL": "explain format = 'brief' select * from t1 left join t2 on t2.a = 1 and t2.a = 2",
         "Result": [
-          "HashJoin 80000000.00 root  CARTESIAN left outer join",
-          "├─TableDual(Build) 8000.00 root  rows:0",
+          "HashJoin 10000.00 root  CARTESIAN left outer join",
+          "├─TableDual(Build) 0.00 root  rows:0",
           "└─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "  └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ]
diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go
index 719c7ce1b849c..6a3966af961bd 100644
--- a/planner/core/integration_test.go
+++ b/planner/core/integration_test.go
@@ -3361,7 +3361,7 @@ func TestIssue14481(t *testing.T) {
 	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t(a int default null, b int default null, c int default null)")
 	plan := tk.MustQuery("explain format = 'brief' select * from t where a = 1 and a = 2")
-	plan.Check(testkit.Rows("TableDual 8000.00 root  rows:0"))
+	plan.Check(testkit.Rows("TableDual 0.00 root  rows:0"))
 	tk.MustExec("drop table t")
 }
 
diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json
index c730b6d6cc83f..099c4a31efabf 100644
--- a/planner/core/testdata/integration_suite_out.json
+++ b/planner/core/testdata/integration_suite_out.json
@@ -5574,7 +5574,7 @@
         "Plan": [
           "IndexMerge 8.00 root  ",
           "├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false",
-          "├─Selection(Build) 0.80 cop[tikv]  1",
+          "├─Selection(Build) 1.00 cop[tikv]  1",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false",
           "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false"
         ],
@@ -5585,7 +5585,7 @@
         "Plan": [
           "IndexMerge 8.00 root  ",
           "├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false",
-          "├─Selection(Build) 0.80 cop[tikv]  1",
+          "├─Selection(Build) 1.00 cop[tikv]  1",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false",
           "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false"
         ],
@@ -5595,9 +5595,9 @@
         "SQL": "desc format='brief' select /*+ use_index_merge(t) */ * from t where (a=1 and length(a)=1) or (b=1 and length(b)=1)",
         "Plan": [
           "IndexMerge 8.00 root  ",
-          "├─Selection(Build) 0.80 cop[tikv]  1",
+          "├─Selection(Build) 1.00 cop[tikv]  1",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false",
-          "├─Selection(Build) 0.80 cop[tikv]  1",
+          "├─Selection(Build) 1.00 cop[tikv]  1",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false",
           "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false"
         ],
diff --git a/planner/core/testdata/partition_pruner_out.json b/planner/core/testdata/partition_pruner_out.json
index 7977fc780453d..db74202a1459b 100644
--- a/planner/core/testdata/partition_pruner_out.json
+++ b/planner/core/testdata/partition_pruner_out.json
@@ -1485,10 +1485,10 @@
         "SQL": "select * from t1 where a in (1,6) and (a=3 and b =3)",
         "Result": null,
         "Plan": [
-          "TableDual 8000.00 root  rows:0"
+          "TableDual 0.00 root  rows:0"
         ],
         "IndexPlan": [
-          "TableDual 8000.00 root  rows:0"
+          "TableDual 0.00 root  rows:0"
         ]
       },
       {
@@ -2488,10 +2488,10 @@
         "SQL": "select * from t1 where a in (1,6) and (a=3 and b =3)",
         "Result": null,
         "Plan": [
-          "TableDual 8000.00 root  rows:0"
+          "TableDual 0.00 root  rows:0"
         ],
         "IndexPlan": [
-          "TableDual 8000.00 root  rows:0"
+          "TableDual 0.00 root  rows:0"
         ]
       },
       {
diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json
index 8d78c7b1aa4ff..f9d4ede451753 100644
--- a/util/ranger/testdata/ranger_suite_out.json
+++ b/util/ranger/testdata/ranger_suite_out.json
@@ -123,7 +123,7 @@
       {
         "SQL": "select * from t where a = 1 and (b = 1 or b = 2) and b = 3 and c > 1;",
         "Plan": [
-          "TableDual_5 8000.00 root  rows:0"
+          "TableDual_5 0.00 root  rows:0"
         ],
         "Result": null
       },
@@ -169,7 +169,7 @@
       {
         "SQL": "select * from t where a = 1 and b is null and b = 1 and c > 1;",
         "Plan": [
-          "TableDual_5 8000.00 root  rows:0"
+          "TableDual_5 0.00 root  rows:0"
         ],
         "Result": null
       }