Repository: pingcap/tidb (Go)

Issue:
invalid memory address or nil pointer dereference
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)

<!-- a step by step guide for reproducing the bug. -->
```
use test;
drop table if exists t1, t2;
CREATE TABLE `t1` (
  `key1` int(11) NOT NULL,
  `key2` int(11) NOT NULL,
  KEY `i1` (`key1`),
  KEY `i2` (`key2`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
delete /*+ use_index_merge(t1)*/ FROM t1 WHERE key1=25 OR (key2 < SOME (SELECT /*+ use_index_merge(t1)*/ key2 FROM t1 WHERE key1<5 OR key2<8));
```

### 2. What did you expect to see? (Required)
```
mysql> delete /*+ use_index_merge(t1)*/ FROM t1 WHERE key1=25 OR (key2 < SOME (SELECT /*+ use_index_merge(t1)*/ key2 FROM t1 WHERE key1<5 OR key2<8));
Query OK, 0 rows affected, 1 warning (0.01 sec)
```

### 3. What did you see instead (Required)
```
mysql> delete /*+ use_index_merge(t1)*/ FROM t1 WHERE key1=25 OR (key2 < SOME (SELECT /*+ use_index_merge(t1)*/ key2 FROM t1 WHERE key1<5 OR key2<8));
ERROR 1105 (HY000): runtime error: invalid memory address or nil pointer dereference
```

### 4. What is your TiDB version? (Required)

<!-- Paste the output of SELECT tidb_version() -->
master v4.0.0-beta.2-2274-gdb62e341f



Discussion:
/assign
panic stack:

```
[stack="goroutine 429 [running]:
github.com/pingcap/tidb/server.(*clientConn).Run.func1(0x47b86e0, 0xc00f71d920, 0xc00f5bda00)
	/root/go/src/github.com/pingcap/tidb/server/conn.go:731 +0xd1
panic(0x4043d40, 0x5a9d110)
	/usr/local/go/src/runtime/panic.go:975 +0x4c6
github.com/pingcap/tidb/planner/core.(*DataSource).buildIndexMergePartialPath(0xc00054b680, 0xc010979920, 0x3, 0x3, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:564 +0x171
github.com/pingcap/tidb/planner/core.(*DataSource).generateIndexMergeOrPaths(0xc00054b680, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:427 +0xd16
github.com/pingcap/tidb/planner/core.(*DataSource).generateAndPruneIndexMergePath(0xc00054b680, 0xc00f6ab101, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:337 +0x53
github.com/pingcap/tidb/planner/core.(*DataSource).DeriveStats(0xc00054b680, 0x612f4f8, 0x0, 0x0, 0xc01094f4a0, 0x612f4f8, 0x0, 0x0, 0xc010955b60, 0x0, ...)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:324 +0xc16
github.com/pingcap/tidb/planner/core.(*baseLogicalPlan).recursiveDeriveStats(0xc00054b6a0, 0xc010955b60, 0x0, 0x2, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:125 +0x50c
github.com/pingcap/tidb/planner/core.(*baseLogicalPlan).recursiveDeriveStats(0xc0107f2f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:118 +0x29c
github.com/pingcap/tidb/planner/core.(*baseLogicalPlan).recursiveDeriveStats(0xc0108b3ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:118 +0x29c
github.com/pingcap/tidb/planner/core.(*baseLogicalPlan).recursiveDeriveStats(0xc010984020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/stats.go:118 +0x29c
github.com/pingcap/tidb/planner/core.physicalOptimize(0x47ea840, 0xc010984000, 0xc01094af80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/optimizer.go:208 +0x8f
github.com/pingcap/tidb/planner/core.DoOptimize(0x47b86e0, 0xc010954b40, 0x47ebea0, 0xc00f5bdb00, 0x20ae, 0x47ea840, 0xc010984000, 0x0, 0x0, 0x0, ...)
	/root/go/src/github.com/pingcap/tidb/planner/core/optimizer.go:139 +0x345
github.com/pingcap/tidb/planner/core.(*PlanBuilder).buildDelete(0xc010939200, 0x47b86e0, 0xc010954b40, 0xc01094d110, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/logical_plan_builder.go:4676 +0x1266
github.com/pingcap/tidb/planner/core.(*PlanBuilder).Build(0xc010939200, 0x47b86e0, 0xc010954b40, 0x47c8f80, 0xc01094d110, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/planner/core/planbuilder.go:603 +0xd79
github.com/pingcap/tidb/planner.optimize(0x47b86e0, 0xc010954b40, 0x47ebea0, 0xc00f5bdb00, 0x47c8f80, 0xc01094d110, 0x47e54a0, 0xc0104eedc0, 0x0, 0x0, ...)
	/root/go/src/github.com/pingcap/tidb/planner/optimize.go:233 +0x2c6
github.com/pingcap/tidb/planner.Optimize(0x47b86e0, 0xc010954b40, 0x47ebea0, 0xc00f5bdb00, 0x47c8f80, 0xc01094d110, 0x47e54a0, 0xc0104eedc0, 0x0, 0x0, ...)
	/root/go/src/github.com/pingcap/tidb/planner/optimize.go:123 +0xa3b
github.com/pingcap/tidb/executor.(*Compiler).Compile(0xc010468d40, 0x47b86e0, 0xc010954b40, 0x47cf8e0, 0xc01094d110, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/executor/compiler.go:62 +0x5db
github.com/pingcap/tidb/session.(*session).ExecuteStmt(0xc00f5bdb00, 0x47b86e0, 0xc0109549c0, 0x47cf8e0, 0xc01094d110, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/session/session.go:1516 +0x6e5
github.com/pingcap/tidb/server.(*TiDBContext).ExecuteStmt(0xc00f71dda0, 0x47b86e0, 0xc010954990, 0x47cf8e0, 0xc01094d110, 0x0, 0x0, 0x0, 0x0)
	/root/go/src/github.com/pingcap/tidb/server/driver_tidb.go:212 +0xac
github.com/pingcap/tidb/server.(*clientConn).handleStmt(0xc00f5bda00, 0x47b86e0, 0xc010954990, 0x47cf8e0, 0xc01094d110, 0x612f4f8, 0x0, 0x0, 0x1, 0x0, ...)
	/root/go/src/github.com/pingcap/tidb/server/conn.go:1607 +0x19e
github.com/pingcap/tidb/server.(*clientConn).handleQuery(0xc00f5bda00, 0x47b8620, 0xc010958380, 0xc0107f1e61, 0x8e, "]
```
panic is triggered by

```
	rc, err := ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID, indexAccessPaths[i].Ranges)
```

`indexAccessPaths[i].Index` is nil because the filter `key2 < SOME (SELECT /*+ use_index_merge(t1)*/ key2 FROM t1 WHERE key1<5 OR key2<8)` is collapsed into constant true.
The bug is introduced by https://github.com/pingcap/tidb/pull/22825, the estimation logic below does not consider possible table path at all.

```
 563     for i := 0; i < len(indexAccessPaths); i++ {
 564         rc, err := ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID,
 565         if err != nil {
 566             return nil, err
 567         }
 568         if rc < minEstRow {
 569             minEstRowIndex = i
 570             minEstRow = rc
 571         }
 572     }
```
>  ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID

If indexAccessPaths[i].Index equals nil, continue in the loop?
> > ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID
> 
> If indexAccessPaths[i].Index equals nil, continue in the loop?

No, we'd better:
1) handle the constant DNF item in `generateIndexMergeOrPaths`;
2) take table path into consideration in `buildIndexMergePartialPath` as well;
> > > ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID
> > 
> > 
> > If indexAccessPaths[i].Index equals nil, continue in the loop?
> 
> No, we'd better:
> 
> 1. handle the constant DNF item in `generateIndexMergeOrPaths`;
> 2. take table path into consideration in `buildIndexMergePartialPath` as well;

Could you provide some advise for me to fix that bug? I'm just a new hand and very interested in tidb.
> > > > ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID
> > > 
> > > 
> > > If indexAccessPaths[i].Index equals nil, continue in the loop?
> > 
> > 
> > No, we'd better:
> > 
> > 1. handle the constant DNF item in `generateIndexMergeOrPaths`;
> > 2. take table path into consideration in `buildIndexMergePartialPath` as well;
> 
> Could you provide some advise for me to fix that bug? I'm just a new hand and very interested in tidb.

Nice to hear that, but I suggest you starting from issues marked as "help-wanted", i.e, https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3Astatus%2Fhelp-wanted. For this issue, IMO it is pretty unfriendly for new contributors if we need not just a protective patch for panic.
> > > > > ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID
> > > > 
> > > > 
> > > > If indexAccessPaths[i].Index equals nil, continue in the loop?
> > > 
> > > 
> > > No, we'd better:
> > > 
> > > 1. handle the constant DNF item in `generateIndexMergeOrPaths`;
> > > 2. take table path into consideration in `buildIndexMergePartialPath` as well;
> > 
> > 
> > Could you provide some advise for me to fix that bug? I'm just a new hand and very interested in tidb.
> 
> Nice to hear that, but I suggest you starting from issues marked as "help-wanted", i.e, https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3Astatus%2Fhelp-wanted. For this issue, IMO it is pretty unfriendly for new contributors if we need not just a protective patch for panic.

Thanks for your advise :-). Seems i have misunderstanding on that issue.
## Please edit this comment or add a new comment to complete the following information

### Not a bug

1. Remove the 'type/bug' label
2. Add notes to indicate why it is not a bug

### Duplicate bug

1. Add the 'type/duplicate' label
2. Add the link to the original bug

### Bug

Note: Make Sure that 'component', and 'severity' labels are added
Example for how to fill out the template: https://github.com/pingcap/tidb/issues/20100

#### 1. Root Cause Analysis (RCA) (optional) 
<!-- Write down the reason why this bug occurs -->

#### 2. Symptom (optional)

<!-- What will the user see when this bug occurs. The error message may be in the terminal, log or monitoring -->

#### 3. All Trigger Conditions (optional)

<!-- All the user scenarios that may trigger this bug -->

#### 4. Workaround (optional)

#### 5. Affected versions
master
<!--
In the format of [start_version:end_version], multiple version ranges are
accepted. If the bug only affects the unreleased version, please input:
"unreleased". For example:

Notes:
  1. Do not use any white spaces in '[]'.
  2. The range in '[]' is a closed interval
  3. The version format is `v$Major.$Minor.$Patch`, the $Majoy and $Minor
     number in a version range should be the same. [v3.0.1:v3.1.2] is
     invalid because the $Minor number of the version range is different.

Example 1: [v3.0.1:v3.0.5], [v4.0.1:v4.0.5]
Example 2: unreleased
-->

#### 6. Fixed versions
master
<!--
The first released version that contains this fix in each minor version. If the bug's affected version has been released, the fixed version should be a detailed version number; If the bug doesn't affect any released version, the fixed version can be "master";  

Example 1: v3.0.13, v4.0.5
Example 2: master
-->



Patch:
diff --git a/planner/core/stats.go b/planner/core/stats.go
index c6c3e88442cd0..eeb5e86463536 100644
--- a/planner/core/stats.go
+++ b/planner/core/stats.go
@@ -506,8 +506,10 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 				logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err))
 				continue
 			}
-			if len(path.AccessConds) == 0 {
-				// If AccessConds is empty, we ignore the access path.
+			// If `AccessConds` is empty, we ignore the access path.
+			// If the path contains a full range, ignore it also. This can happen when `AccessConds` is constant true, and
+			// it comes from the result of a subquery, so it is not folded.
+			if len(path.AccessConds) == 0 || ranger.HasFullRange(path.Ranges) {
 				continue
 			}
 			// If we have point or empty range, just remove other possible paths.
@@ -531,8 +533,10 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 				continue
 			}
 			noIntervalRanges := ds.deriveIndexPathStats(path, conditions, true)
-			if len(path.AccessConds) == 0 {
-				// If AccessConds is empty, we ignore the access path.
+			// If `AccessConds` is empty, we ignore the access path.
+			// If the path contains a full range, ignore it also. This can happen when `AccessConds` is constant true, and
+			// it comes from the result of a subquery, so it is not folded.
+			if len(path.AccessConds) == 0 || ranger.HasFullRange(path.Ranges) {
 				continue
 			}
 			// If we have empty range, or point range on unique index, just remove other possible paths.
@@ -561,9 +565,9 @@ func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.Access
 	minEstRowIndex := 0
 	minEstRow := math.MaxFloat64
 	for i := 0; i < len(indexAccessPaths); i++ {
-		rc, err := ds.stats.HistColl.GetRowCountByIndexRanges(ds.ctx.GetSessionVars().StmtCtx, indexAccessPaths[i].Index.ID, indexAccessPaths[i].Ranges)
-		if err != nil {
-			return nil, err
+		rc := indexAccessPaths[i].CountAfterAccess
+		if len(indexAccessPaths[i].IndexFilters) > 0 {
+			rc = indexAccessPaths[i].CountAfterIndex
 		}
 		if rc < minEstRow {
 			minEstRowIndex = i
diff --git a/util/ranger/types.go b/util/ranger/types.go
index 20f7a1509a1ce..26548ac1815bb 100644
--- a/util/ranger/types.go
+++ b/util/ranger/types.go
@@ -122,6 +122,16 @@ func (ran *Range) IsFullRange() bool {
 	return true
 }
 
+// HasFullRange checks if any range in the slice is a full range.
+func HasFullRange(ranges []*Range) bool {
+	for _, ran := range ranges {
+		if ran.IsFullRange() {
+			return true
+		}
+	}
+	return false
+}
+
 // String implements the Stringer interface.
 func (ran *Range) String() string {
 	lowStrs := make([]string, 0, len(ran.LowVal))


Test patch:
diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go
index e14dd2f2f4332..2fe45c983a19f 100644
--- a/planner/core/integration_test.go
+++ b/planner/core/integration_test.go
@@ -2607,6 +2607,22 @@ func (s *testIntegrationSuite) TestReorderSimplifiedOuterJoins(c *C) {
 	}
 }
 
+func (s *testIntegrationSuite) TestIndexMergeConstantTrue(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t;")
+	tk.MustExec("create table t(a int primary key, b int not null, key(b))")
+	tk.MustExec("delete /*+ use_index_merge(t) */ FROM t WHERE a=1 OR (b < SOME (SELECT /*+ use_index_merge(t)*/ b FROM t WHERE a<2 OR b<2))")
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int not null, b int not null, key(a), key(b))")
+	tk.MustExec("delete /*+ use_index_merge(t) */ FROM t WHERE a=1 OR (b < SOME (SELECT /*+ use_index_merge(t)*/ b FROM t WHERE a<2 OR b<2))")
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int primary key, b int not null, c int, key(a), key(b,c))")
+	tk.MustExec("delete /*+ use_index_merge(t) */ FROM t WHERE a=1 OR (a<2 and b<2)")
+}
+
 func (s *testIntegrationSerialSuite) TestPushDownAggForMPP(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")