Repository: pingcap/tidb
File: pkg/planner/core/stats.go (Go)

Code before:
	// isUnsigned indicates whether the handle is unsigned.
	// Now we just cast the unsigned int to int and then store the int value inside the Datum.
	// We wrap the uint64/int64 with Datum here to keep us untouched with Datum's ineternal representation.
	if isUnsigned {
		minDatum = types.NewUintDatum(0)
		maxDatum = types.NewUintDatum(math.MaxUint64)
	} else {
		minDatum = types.NewIntDatum(math.MinInt64)
		maxDatum = types.NewIntDatum(math.MaxInt64)
	}
	for i := len(ranges) - 1; i >= 0; i-- {
		ran := ranges[i]
		if ran.HighVal[0].IsNull() {
			ranges = append(ranges[:i], ranges[i+1:]...)
			continue
		}
		if ran.LowVal[0].IsNull() || ran.LowVal[0].Kind() == types.KindMinNotNull {
			ran.LowVal[0].SetInt64(minDatum.GetInt64())
			ran.LowExclude = false
		}
		if ran.HighVal[0].Kind() == types.KindMaxValue {
			ran.HighVal[0].SetInt64(maxDatum.GetInt64())
		}
	}
}

func getGeneralAttributesFromPaths(paths []*util.AccessPath, totalRowCount float64) (float64, bool) {
	minSelectivity := 1.0
	indexForce := false
	for _, path := range paths {
		// For table path and index merge path, AccessPath.CountAfterIndex is not set and meaningless,
		// but we still consider their AccessPath.CountAfterAccess.
		if totalRowCount > 0 {
			if path.IsTablePath() || path.PartialIndexPaths != nil {
				minSelectivity = min(minSelectivity, path.CountAfterAccess/totalRowCount)
			} else {
				minSelectivity = min(minSelectivity, path.CountAfterIndex/totalRowCount)
			}
		}
		if !indexForce && path.Forced {
			indexForce = true
		}
	}
	return minSelectivity, indexForce
}

func getGroupNDVs(ds *logicalop.DataSource) []property.GroupNDV {
	colGroups := ds.AskedColumnGroup
	if len(ds.AskedColumnGroup) == 0 {
		return nil
	}

Review comment (suggestion):
When `isUnsigned` is true, the code creates unsigned datums but then uses `GetInt64()` and `SetInt64()` to transfer the values. For the maximum value case, `math.MaxUint64` stored in a `KindUint64` datum has an internal int64 representation of -1. Calling `GetInt64()` returns -1, and then `SetInt64(-1)` sets the datum as a signed -1 rather than unsigned MaxUint64. This changes the datum kind from `KindUint64` to `KindInt64` and loses the unsigned semantics. Use `SetUint64(maxDatum.GetUint64())` for unsigned handles to preserve the correct type and value.
```suggestion
			if isUnsigned {
				ran.LowVal[0].SetUint64(minDatum.GetUint64())
			} else {
				ran.LowVal[0].SetInt64(minDatum.GetInt64())
			}
			ran.LowExclude = false
		}
		if ran.HighVal[0].Kind() == types.KindMaxValue {
			if isUnsigned {
				ran.HighVal[0].SetUint64(maxDatum.GetUint64())
			} else {
				ran.HighVal[0].SetInt64(maxDatum.GetInt64())
			}
```

Code after:
func fixTiCIIndexRangesForIntHandle(ranges []*ranger.Range, isUnsigned bool) []*ranger.Range {
	var minDatum, maxDatum types.Datum
	// isUnsigned indicates whether the handle is unsigned.
	// Now we just cast the unsigned int to int and then store the int value inside the Datum.
	// We wrap the uint64/int64 with Datum here to keep us untouched with Datum's internal representation.
	if isUnsigned {
		minDatum = types.NewUintDatum(0)
		maxDatum = types.NewUintDatum(math.MaxUint64)
	} else {
		minDatum = types.NewIntDatum(math.MinInt64)
		maxDatum = types.NewIntDatum(math.MaxInt64)
	}
	for i := len(ranges) - 1; i >= 0; i-- {
		ran := ranges[i]
		// Remove the [NULL, NULL] which is invalid when generating int handle's kv.KeyRange.
		if ran.HighVal[0].IsNull() {
			ranges = slices.Delete(ranges, i, i+1)
			continue
		}
		// Convert the min and max value to the int min/max value.
		if ran.LowVal[0].IsNull() || ran.LowVal[0].Kind() == types.KindMinNotNull {
			ran.LowVal[0].SetInt64(minDatum.GetInt64())
			ran.LowExclude = false
		}
		if ran.HighVal[0].Kind() == types.KindMaxValue {
			ran.HighVal[0].SetInt64(maxDatum.GetInt64())
		}
	}
	return ranges
}

func getGeneralAttributesFromPaths(paths []*util.AccessPath, totalRowCount float64) (float64, bool) {
	minSelectivity := 1.0
	indexForce := false
	for _, path := range paths {
		// For table path and index merge path, AccessPath.CountAfterIndex is not set and meaningless,
		// but we still consider their AccessPath.CountAfterAccess.
		if totalRowCount > 0 {
			if path.IsTablePath() || path.PartialIndexPaths != nil {
				minSelectivity = min(minSelectivity, path.CountAfterAccess/totalRowCount)
			} else {
				minSelectivity = min(minSelectivity, path.CountAfterIndex/totalRowCount)
			}
		}
		if !indexForce && path.Forced {
			indexForce = true
		}
	}
	return minSelectivity, indexForce
}