Repository: apache/pinot (2025-07-24T14:18:33Z)

Issue:
CAST function calls with literal-only operands don't get evaluated during query planning in the MSQE
- Function calls with literal-only operands get evaluated in the broker during query planning / optimization itself instead of during query execution in the servers as an optimization. 
- This is done through the [PinotEvaluateLiteralRule](https://github.com/apache/pinot/blob/9a18becd1591340f653b1e6caf53ba069b1645e4/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java#L59).
- However, there's a bug that prevents `CAST` functions from being evaluated. The `CAST` operator in Calcite only has one operand (the value to be cast) and the type to be cast to is determined by the operator's return type. Pinot's `CAST` function implementation requires two arguments: the value to be cast and the target type.
- This discrepancy causes the `CAST` function to not be found in the rule implementation, and the function call is retained as is.
- This can cause a myriad of downstream issues. For instance, take a query like:
```
select AirlineID from airlineStats where ts > '2000-05-16T00:00:00' and ts < '2025-05-16T00:00:00';
```
- The filter becomes `AND(>($82, CAST(_UTF-8'2000-05-16T00:00:00'):TIMESTAMP(0) NOT NULL), <($82, CAST(_UTF-8'2025-05-16T00:00:00'):TIMESTAMP(0) NOT NULL))` where the casts are added by Calcites to convert the string literals to the appropriate type.
- Since the `CAST` doesn't get evaluated during query planning, it makes it down all the way to the leaf stage, where [PredicateComparisonRewriter](https://github.com/apache/pinot/blob/9a18becd1591340f653b1e6caf53ba069b1645e4/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java#L33) rewrites the filter expression to:
```
FilterAnd
  FilterExpression(predicate=[minus(ts,cast('2000-05-16T00:00:00','TIMESTAMP')) > '0'], operator=[RANGE])
  FilterExpression(predicate=[minus(ts,cast('2025-05-16T00:00:00','TIMESTAMP')) < '0'], operator=[RANGE])
```
- This is a big issue because it'll prevent the filter from using range indexes.


Discussion:

Patch:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
index 347031ec864d..c18ffed81294 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
@@ -38,43 +38,43 @@ private DataTypeConversionFunctions() {
 
   @ScalarFunction
   public static Object cast(Object value, String targetTypeLiteral) {
-    try {
-      Class<?> clazz = value.getClass();
-      // TODO: Support cast for MV
-      Preconditions.checkArgument(!clazz.isArray() | clazz == byte[].class, "%s must not be an array type", clazz);
-      PinotDataType sourceType = PinotDataType.getSingleValueType(clazz);
-      String transformed = targetTypeLiteral.toUpperCase();
-      PinotDataType targetDataType;
-      switch (transformed) {
-        case "BIGINT":
-          targetDataType = LONG;
-          break;
-        case "DECIMAL":
-          targetDataType = BIG_DECIMAL;
-          break;
-        case "INT":
-          targetDataType = INTEGER;
-          break;
-        case "VARBINARY":
-          targetDataType = BYTES;
-          break;
-        case "VARCHAR":
-          targetDataType = STRING;
-          break;
-        default:
+    Class<?> clazz = value.getClass();
+    // TODO: Support cast for MV
+    Preconditions.checkArgument(!clazz.isArray() | clazz == byte[].class, "%s must not be an array type", clazz);
+    PinotDataType sourceType = PinotDataType.getSingleValueType(clazz);
+    String transformed = targetTypeLiteral.toUpperCase();
+    PinotDataType targetDataType;
+    switch (transformed) {
+      case "BIGINT":
+        targetDataType = LONG;
+        break;
+      case "DECIMAL":
+        targetDataType = BIG_DECIMAL;
+        break;
+      case "INT":
+        targetDataType = INTEGER;
+        break;
+      case "VARBINARY":
+        targetDataType = BYTES;
+        break;
+      case "VARCHAR":
+        targetDataType = STRING;
+        break;
+      default:
+        try {
           targetDataType = PinotDataType.valueOf(transformed);
-          break;
-      }
-      if (sourceType == STRING && (targetDataType == INTEGER || targetDataType == LONG)) {
-        if (String.valueOf(value).contains(".")) {
-          // convert integers via double to avoid parse errors
-          return targetDataType.convert(DOUBLE.convert(value, sourceType), DOUBLE);
+        } catch (IllegalArgumentException e) {
+          throw new IllegalArgumentException("Unknown data type: " + targetTypeLiteral);
         }
+        break;
+    }
+    if (sourceType == STRING && (targetDataType == INTEGER || targetDataType == LONG)) {
+      if (String.valueOf(value).contains(".")) {
+        // convert integers via double to avoid parse errors
+        return targetDataType.convert(DOUBLE.convert(value, sourceType), DOUBLE);
       }
-      return targetDataType.convert(value, sourceType);
-    } catch (IllegalArgumentException e) {
-      throw new IllegalArgumentException("Unknown data type: " + targetTypeLiteral);
     }
+    return targetDataType.convert(value, sourceType);
   }
 
   /**
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
index ca634a04c2e8..fea69ab5d211 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
@@ -161,15 +161,14 @@ private static RexNode evaluateLiteralOnlyFunction(RexCall rexCall, RexBuilder r
     assert operands.stream().allMatch(
         operand -> operand instanceof RexLiteral || (operand instanceof RexCall && ((RexCall) operand).getOperands()
             .stream().allMatch(op -> op instanceof RexLiteral)));
+
     int numArguments = operands.size();
     ColumnDataType[] argumentTypes = new ColumnDataType[numArguments];
     Object[] arguments = new Object[numArguments];
     for (int i = 0; i < numArguments; i++) {
       RexNode rexNode = operands.get(i);
       RexLiteral rexLiteral;
-      if (rexNode instanceof RexCall && ((RexCall) rexNode).getOperator().getKind() == SqlKind.CAST) {
-        rexLiteral = (RexLiteral) ((RexCall) rexNode).getOperands().get(0);
-      } else if (rexNode instanceof RexLiteral) {
+      if (rexNode instanceof RexLiteral) {
         rexLiteral = (RexLiteral) rexNode;
       } else {
         // Function operands cannot be evaluated, skip
@@ -178,6 +177,14 @@ private static RexNode evaluateLiteralOnlyFunction(RexCall rexCall, RexBuilder r
       argumentTypes[i] = RelToPlanNodeConverter.convertToColumnDataType(rexLiteral.getType());
       arguments[i] = getLiteralValue(rexLiteral);
     }
+
+    if (rexCall.getKind() == SqlKind.CAST) {
+      // Handle separately because the CAST operator only has one operand (the value to be cast) and the type to be cast
+      // to is determined by the operator's return type. Pinot's CAST function implementation requires two arguments:
+      // the value to be cast and the target type.
+      argumentTypes = new ColumnDataType[]{argumentTypes[0], ColumnDataType.STRING};
+      arguments = new Object[]{arguments[0], RelToPlanNodeConverter.convertToColumnDataType(rexCall.getType()).name()};
+    }
     String canonicalName = FunctionRegistry.canonicalize(PinotRuleUtils.extractFunctionName(rexCall));
     FunctionInfo functionInfo = FunctionRegistry.lookupFunctionInfo(canonicalName, argumentTypes);
     if (functionInfo == null) {


Test patch:
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ErrorCodesIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ErrorCodesIntegrationTest.java
index 383a71fd1d47..069e3fa382d8 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ErrorCodesIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ErrorCodesIntegrationTest.java
@@ -138,7 +138,7 @@ public void testInvalidCasting()
       throws Exception {
     // ArrTime expects a numeric type
     testQueryException("SELECT COUNT(*) FROM mytable where ArrTime = 'potato'",
-        useMultiStageQueryEngine() ? QueryErrorCode.QUERY_EXECUTION : QueryErrorCode.QUERY_VALIDATION);
+        useMultiStageQueryEngine() ? QueryErrorCode.QUERY_PLANNING : QueryErrorCode.QUERY_VALIDATION);
   }
 
   @Test
diff --git a/pinot-query-planner/src/test/resources/queries/LiteralEvaluationPlans.json b/pinot-query-planner/src/test/resources/queries/LiteralEvaluationPlans.json
index f23e0662e077..e63b4aab8a68 100644
--- a/pinot-query-planner/src/test/resources/queries/LiteralEvaluationPlans.json
+++ b/pinot-query-planner/src/test/resources/queries/LiteralEvaluationPlans.json
@@ -45,7 +45,7 @@
         "sql": "EXPLAIN PLAN FOR SELECT timestampDiff(DAY, CAST(ts as TIMESTAMP), CAST(dateTrunc('MONTH', FROMDATETIME('1997-02-01 00:00:00', 'yyyy-MM-dd HH:mm:ss')) as TIMESTAMP)) FROM d",
         "output": [
           "Execution Plan",
-          "\nLogicalProject(EXPR$0=[TIMESTAMPDIFF(FLAG(DAY), CAST($7):TIMESTAMP(0) NOT NULL, CAST(854755200000:BIGINT):TIMESTAMP(0) NOT NULL)])",
+          "\nLogicalProject(EXPR$0=[TIMESTAMPDIFF(FLAG(DAY), CAST($7):TIMESTAMP(0) NOT NULL, 1997-02-01 00:00:00)])",
           "\n  PinotLogicalTableScan(table=[[default, d]])",
           "\n"
         ]
@@ -253,6 +253,36 @@
           "\n"
         ]
       },
+      {
+        "description": "filter with implicit cast",
+        "sql": "EXPLAIN PLAN FOR SELECT * FROM a WHERE col3 > '10.5'",
+        "output": [
+          "Execution Plan",
+          "\nLogicalFilter(condition=[>($2, 10)])",
+          "\n  PinotLogicalTableScan(table=[[default, a]])",
+          "\n"
+        ]
+      },
+      {
+        "description": "filter with explicit cast",
+        "sql": "EXPLAIN PLAN FOR SELECT * FROM a WHERE col3 > CAST('10.5' AS INT)",
+        "output": [
+          "Execution Plan",
+          "\nLogicalFilter(condition=[>($2, 10)])",
+          "\n  PinotLogicalTableScan(table=[[default, a]])",
+          "\n"
+        ]
+      },
+      {
+        "description": "filter with nested explicit casts",
+        "sql": "EXPLAIN PLAN FOR SELECT * FROM a WHERE col3 > CAST(CAST('10.5' AS LONG) AS INT)",
+        "output": [
+          "Execution Plan",
+          "\nLogicalFilter(condition=[>($2, 10)])",
+          "\n  PinotLogicalTableScan(table=[[default, a]])",
+          "\n"
+        ]
+      },
       {
         "description": "select non-exist literal function",
         "sql": "EXPLAIN PLAN FOR Select nonExistFun(1,2) FROM a",
diff --git a/pinot-query-planner/src/test/resources/queries/PhysicalOptimizerPlans.json b/pinot-query-planner/src/test/resources/queries/PhysicalOptimizerPlans.json
index ca7dc9730742..106d0e7f9a91 100644
--- a/pinot-query-planner/src/test/resources/queries/PhysicalOptimizerPlans.json
+++ b/pinot-query-planner/src/test/resources/queries/PhysicalOptimizerPlans.json
@@ -466,7 +466,7 @@
     "queries": [
       {
         "description": "Self semi-joins",
-        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, col2 FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = 'foo') AND col2 IN (SELECT col2 FROM a WHERE col3 = 'bar')",
+        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, col2 FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = '1') AND col2 IN (SELECT col2 FROM a WHERE col3 = '2')",
         "output": [
           "Execution Plan",
           "\nPhysicalExchange(exchangeStrategy=[SINGLETON_EXCHANGE])",
@@ -477,18 +477,18 @@
           "\n          PhysicalTableScan(table=[[default, a]])",
           "\n      PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n        PhysicalProject(col2=[$1])",
-          "\n          PhysicalFilter(condition=[=($2, CAST(_UTF-8'foo'):INTEGER NOT NULL)])",
+          "\n          PhysicalFilter(condition=[=($2, 1)])",
           "\n            PhysicalTableScan(table=[[default, a]])",
           "\n    PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n      PhysicalProject(col2=[$1])",
-          "\n        PhysicalFilter(condition=[=($2, CAST(_UTF-8'bar'):INTEGER NOT NULL)])",
+          "\n        PhysicalFilter(condition=[=($2, 2)])",
           "\n          PhysicalTableScan(table=[[default, a]])",
           "\n"
         ]
       },
       {
         "description": "Self semi and anti semi-joins",
-        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, col2 FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = 'foo') AND col2 NOT IN (SELECT col2 FROM a WHERE col3 = 'bar') AND col2 IN (SELECT col2 FROM a WHERE col3 = 'lorem')",
+        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, col2 FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = '1') AND col2 NOT IN (SELECT col2 FROM a WHERE col3 = '2') AND col2 IN (SELECT col2 FROM a WHERE col3 = '3')",
         "output": [
           "Execution Plan",
           "\nPhysicalExchange(exchangeStrategy=[SINGLETON_EXCHANGE])",
@@ -503,23 +503,23 @@
           "\n                  PhysicalTableScan(table=[[default, a]])",
           "\n              PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n                PhysicalProject(col2=[$1])",
-          "\n                  PhysicalFilter(condition=[=($2, CAST(_UTF-8'foo'):INTEGER NOT NULL)])",
+          "\n                  PhysicalFilter(condition=[=($2, 1)])",
           "\n                    PhysicalTableScan(table=[[default, a]])",
           "\n          PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n            PhysicalAggregate(group=[{0}], agg#0=[MIN($1)], aggType=[DIRECT])",
           "\n              PhysicalProject(col2=[$1], $f1=[true])",
-          "\n                PhysicalFilter(condition=[=($2, CAST(_UTF-8'bar'):INTEGER NOT NULL)])",
+          "\n                PhysicalFilter(condition=[=($2, 2)])",
           "\n                  PhysicalTableScan(table=[[default, a]])",
           "\n    PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n      PhysicalProject(col2=[$1])",
-          "\n        PhysicalFilter(condition=[=($2, CAST(_UTF-8'lorem'):INTEGER NOT NULL)])",
+          "\n        PhysicalFilter(condition=[=($2, 3)])",
           "\n          PhysicalTableScan(table=[[default, a]])",
           "\n"
         ]
       },
       {
         "description": "Self semi and anti semi-joins with aggregation in the end",
-        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, COUNT(*) FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = 'foo') AND col2 NOT IN (SELECT col2 FROM a WHERE col3 = 'bar') AND col2 IN (SELECT col2 FROM a WHERE col3 = 'lorem') GROUP BY col1",
+        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR SELECT col1, COUNT(*) FROM a WHERE col2 IN (SELECT col2 FROM a WHERE col3 = '1') AND col2 NOT IN (SELECT col2 FROM a WHERE col3 = '2') AND col2 IN (SELECT col2 FROM a WHERE col3 = '3') GROUP BY col1",
         "output": [
           "Execution Plan",
           "\nPhysicalExchange(exchangeStrategy=[SINGLETON_EXCHANGE])",
@@ -537,16 +537,16 @@
           "\n                        PhysicalTableScan(table=[[default, a]])",
           "\n                    PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n                      PhysicalProject(col2=[$1])",
-          "\n                        PhysicalFilter(condition=[=($2, CAST(_UTF-8'foo'):INTEGER NOT NULL)])",
+          "\n                        PhysicalFilter(condition=[=($2, 1)])",
           "\n                          PhysicalTableScan(table=[[default, a]])",
           "\n                PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n                  PhysicalAggregate(group=[{0}], agg#0=[MIN($1)], aggType=[DIRECT])",
           "\n                    PhysicalProject(col2=[$1], $f1=[true])",
-          "\n                      PhysicalFilter(condition=[=($2, CAST(_UTF-8'bar'):INTEGER NOT NULL)])",
+          "\n                      PhysicalFilter(condition=[=($2, 2)])",
           "\n                        PhysicalTableScan(table=[[default, a]])",
           "\n          PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n            PhysicalProject(col2=[$1])",
-          "\n              PhysicalFilter(condition=[=($2, CAST(_UTF-8'lorem'):INTEGER NOT NULL)])",
+          "\n              PhysicalFilter(condition=[=($2, 3)])",
           "\n                PhysicalTableScan(table=[[default, a]])",
           "\n"
         ]
@@ -613,7 +613,7 @@
     "queries": [
       {
         "description": "Union, distinct, etc. but still maximally identity exchange",
-        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR WITH tmp AS (SELECT col2 FROM a WHERE col1 = 'foo' UNION ALL SELECT col2 FROM a WHERE col3 = 'bar'), tmp2 AS (SELECT DISTINCT col2 FROM tmp) SELECT COUNT(*), col3 FROM a WHERE col2 IN (SELECT col2 FROM tmp2) GROUP BY col3",
+        "sql": "SET usePhysicalOptimizer=true; EXPLAIN PLAN FOR WITH tmp AS (SELECT col2 FROM a WHERE col1 = 'foo' UNION ALL SELECT col2 FROM a WHERE col3 = '1'), tmp2 AS (SELECT DISTINCT col2 FROM tmp) SELECT COUNT(*), col3 FROM a WHERE col2 IN (SELECT col2 FROM tmp2) GROUP BY col3",
         "output": [
           "Execution Plan",
           "\nPhysicalExchange(exchangeStrategy=[SINGLETON_EXCHANGE])",
@@ -633,7 +633,7 @@
           "\n                      PhysicalTableScan(table=[[default, a]])",
           "\n                PhysicalExchange(exchangeStrategy=[IDENTITY_EXCHANGE])",
           "\n                  PhysicalProject(col2=[$1])",
-          "\n                    PhysicalFilter(condition=[=($2, CAST(_UTF-8'bar'):INTEGER NOT NULL)])",
+          "\n                    PhysicalFilter(condition=[=($2, 1)])",
           "\n                      PhysicalTableScan(table=[[default, a]])",
           "\n"
         ]