Repository: apache/datafusion (2025-08-12T09:29:17Z)

Issue:
Redundant aggregation elimination regression
This passes in DataFusion 49.0.0

```
statement ok
CREATE TABLE tab1(col0 INTEGER, col1 INTEGER, col2 INTEGER)

statement ok
INSERT INTO tab1 VALUES(51,14,96)

statement ok
INSERT INTO tab1 VALUES(85,5,59)

statement ok
INSERT INTO tab1 VALUES(91,47,68)

query R
SELECT + CAST ( NULL AS INTEGER ) * + + AVG ( DISTINCT 4 ) + - SUM ( ALL + col1 ) AS col0
FROM tab1
----
NULL
```

the query fails on current main

```
1. query failed: DataFusion error: Execution error: avg(DISTINCT) aggregations are not available
[SQL] SELECT + CAST ( NULL AS INTEGER ) * + + AVG ( DISTINCT 4 ) + - SUM ( ALL + col1 ) AS col0
FROM tab1
```



Discussion:
first failing commit
- 876b504cb0770debda687c91a9ba7295291d26e2 (https://github.com/apache/datafusion/pull/17088)
The problem can be reproduced before https://github.com/apache/datafusion/commit/876b504cb0770debda687c91a9ba7295291d26e2 if the following rule
https://github.com/apache/datafusion/blob/9463cf6792469534a32830aefbba32218716fb15/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs#L1067-L1074

gets duplicated to cover `Plus` operator too.
Shorter repro

```
CREATE OR REPLACE TABLE tab1(col0 INTEGER, col1 INTEGER, col2 INTEGER);
INSERT INTO tab1 VALUES(51,14,96);
SELECT NULL * AVG(DISTINCT 4) + SUM(col1) AS col0 FROM tab1;
```

Plan before https://github.com/apache/datafusion/commit/876b504cb0770debda687c91a9ba7295291d26e2 has AVG eliminated, but SUM remains:

```
+---------------+-------------------------------------------------------------------------+
| plan_type     | plan                                                                    |
+---------------+-------------------------------------------------------------------------+
| logical_plan  | Projection: Float64(NULL) + CAST(sum(tab1.col1) AS Float64) AS col0     |
|               |   Aggregate: groupBy=[[]], aggr=[[sum(CAST(tab1.col1 AS Int64))]]       |
|               |     TableScan: tab1 projection=[col1]                                   |
| physical_plan | ProjectionExec: expr=[NULL + CAST(sum(tab1.col1)@0 AS Float64) as col0] |
|               |   AggregateExec: mode=Single, gby=[], aggr=[sum(tab1.col1)]             |
|               |     DataSourceExec: partitions=1, partition_sizes=[3]                   |
|               |                                                                         |
+---------------+-------------------------------------------------------------------------+
```



On `876b504cb0770debda687c91a9ba7295291d26e2^` with added rule for `Plus` the SUM gets removed, but AVG remains


```
+---------------+----------------------------------------------------------------------------------------+
| plan_type     | plan                                                                                   |
+---------------+----------------------------------------------------------------------------------------+
| logical_plan  | Projection: Float64(NULL) AS col0                                                      |
|               |   Aggregate: groupBy=[[]], aggr=[[avg(DISTINCT Float64(4)) AS avg(DISTINCT Int64(4))]] |
|               |     TableScan: tab1 projection=[]                                                      |
| physical_plan | ProjectionExec: expr=[NULL as col0]                                                    |
|               |   AggregateExec: mode=Single, gby=[], aggr=[avg(DISTINCT Int64(4))]                    |
|               |     DataSourceExec: partitions=1, partition_sizes=[3]                                  |
|               |                                                                                        |
+---------------+----------------------------------------------------------------------------------------+
```

Maybe there is some optimizer rule pruning unused aggregations that retains at least one aggregate function. When both become redundant, it doesn't know which one to prune?



https://github.com/apache/datafusion/blob/de8cbd0ab606bec6eb3ea082fa61f09aa16874eb/datafusion/optimizer/src/optimize_projections/mod.rs#L158
fix attempt

- https://github.com/apache/datafusion/pull/17139




Patch:
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index d68e6cd81272..ea21da29849e 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -3522,7 +3522,10 @@ impl Aggregate {
     ) -> Result<Self> {
         if group_expr.is_empty() && aggr_expr.is_empty() {
             return plan_err!(
-                "Aggregate requires at least one grouping or aggregate expression"
+                "Aggregate requires at least one grouping or aggregate expression. \
+                Aggregate without grouping expressions nor aggregate expressions is \
+                logically equivalent to, but less efficient than, VALUES producing \
+                single row. Please use VALUES instead."
             );
         }
         let group_expr_count = grouping_set_expr_count(&group_expr)?;
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs
index 7b7be82b70ca..08909f5f8667 100644
--- a/datafusion/optimizer/src/optimize_projections/mod.rs
+++ b/datafusion/optimizer/src/optimize_projections/mod.rs
@@ -26,12 +26,12 @@ use std::sync::Arc;
 
 use datafusion_common::{
     get_required_group_by_exprs_indices, internal_datafusion_err, internal_err, Column,
-    HashMap, JoinType, Result,
+    DFSchema, HashMap, JoinType, Result,
 };
 use datafusion_expr::expr::Alias;
 use datafusion_expr::{
-    logical_plan::LogicalPlan, Aggregate, Distinct, Expr, Projection, TableScan, Unnest,
-    Window,
+    logical_plan::LogicalPlan, Aggregate, Distinct, EmptyRelation, Expr, Projection,
+    TableScan, Unnest, Window,
 };
 
 use crate::optimize_projections::required_indices::RequiredIndices;
@@ -153,23 +153,16 @@ fn optimize_projections(
 
             // Only use the absolutely necessary aggregate expressions required
             // by the parent:
-            let mut new_aggr_expr = aggregate_reqs.get_at_indices(&aggregate.aggr_expr);
-
-            // Aggregations always need at least one aggregate expression.
-            // With a nested count, we don't require any column as input, but
-            // still need to create a correct aggregate, which may be optimized
-            // out later. As an example, consider the following query:
-            //
-            // SELECT count(*) FROM (SELECT count(*) FROM [...])
-            //
-            // which always returns 1.
-            if new_aggr_expr.is_empty()
-                && new_group_bys.is_empty()
-                && !aggregate.aggr_expr.is_empty()
-            {
-                // take the old, first aggregate expression
-                new_aggr_expr = aggregate.aggr_expr;
-                new_aggr_expr.resize_with(1, || unreachable!());
+            let new_aggr_expr = aggregate_reqs.get_at_indices(&aggregate.aggr_expr);
+
+            if new_group_bys.is_empty() && new_aggr_expr.is_empty() {
+                // Global aggregation with no aggregate functions always produces 1 row and no columns.
+                return Ok(Transformed::yes(LogicalPlan::EmptyRelation(
+                    EmptyRelation {
+                        produce_one_row: true,
+                        schema: Arc::new(DFSchema::empty()),
+                    },
+                )));
             }
 
             let all_exprs_iter = new_group_bys.iter().chain(new_aggr_expr.iter());
@@ -1146,9 +1139,7 @@ mod tests {
             plan,
             @r"
         Aggregate: groupBy=[[]], aggr=[[count(Int32(1))]]
-          Projection:
-            Aggregate: groupBy=[[]], aggr=[[count(Int32(1))]]
-              TableScan: ?table? projection=[]
+          EmptyRelation
         "
         )
     }


Test patch:
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 36a116154175..a38d6ef1ac20 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -2744,23 +2744,20 @@ async fn test_count_wildcard_on_where_exist() -> Result<()> {
 
     assert_snapshot!(
         pretty_format_batches(&sql_results).unwrap(),
-        @r###"
-    +---------------+---------------------------------------------------------+
-    | plan_type     | plan                                                    |
-    +---------------+---------------------------------------------------------+
-    | logical_plan  | LeftSemi Join:                                          |
-    |               |   TableScan: t1 projection=[a, b]                       |
-    |               |   SubqueryAlias: __correlated_sq_1                      |
-    |               |     Projection:                                         |
-    |               |       Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] |
-    |               |         TableScan: t2 projection=[]                     |
-    | physical_plan | NestedLoopJoinExec: join_type=RightSemi                 |
-    |               |   ProjectionExec: expr=[]                               |
-    |               |     PlaceholderRowExec                                  |
-    |               |   DataSourceExec: partitions=1, partition_sizes=[1]     |
-    |               |                                                         |
-    +---------------+---------------------------------------------------------+
-    "###
+        @r"
+    +---------------+-----------------------------------------------------+
+    | plan_type     | plan                                                |
+    +---------------+-----------------------------------------------------+
+    | logical_plan  | LeftSemi Join:                                      |
+    |               |   TableScan: t1 projection=[a, b]                   |
+    |               |   SubqueryAlias: __correlated_sq_1                  |
+    |               |     EmptyRelation                                   |
+    | physical_plan | NestedLoopJoinExec: join_type=RightSemi             |
+    |               |   PlaceholderRowExec                                |
+    |               |   DataSourceExec: partitions=1, partition_sizes=[1] |
+    |               |                                                     |
+    +---------------+-----------------------------------------------------+
+    "
     );
 
     let df_results = ctx
@@ -2783,23 +2780,20 @@ async fn test_count_wildcard_on_where_exist() -> Result<()> {
 
     assert_snapshot!(
         pretty_format_batches(&df_results).unwrap(),
-        @r###"
-    +---------------+---------------------------------------------------------------------+
-    | plan_type     | plan                                                                |
-    +---------------+---------------------------------------------------------------------+
-    | logical_plan  | LeftSemi Join:                                                      |
-    |               |   TableScan: t1 projection=[a, b]                                   |
-    |               |   SubqueryAlias: __correlated_sq_1                                  |
-    |               |     Projection:                                                     |
-    |               |       Aggregate: groupBy=[[]], aggr=[[count(Int64(1)) AS count(*)]] |
-    |               |         TableScan: t2 projection=[]                                 |
-    | physical_plan | NestedLoopJoinExec: join_type=RightSemi                             |
-    |               |   ProjectionExec: expr=[]                                           |
-    |               |     PlaceholderRowExec                                              |
-    |               |   DataSourceExec: partitions=1, partition_sizes=[1]                 |
-    |               |                                                                     |
-    +---------------+---------------------------------------------------------------------+
-    "###
+        @r"
+    +---------------+-----------------------------------------------------+
+    | plan_type     | plan                                                |
+    +---------------+-----------------------------------------------------+
+    | logical_plan  | LeftSemi Join:                                      |
+    |               |   TableScan: t1 projection=[a, b]                   |
+    |               |   SubqueryAlias: __correlated_sq_1                  |
+    |               |     EmptyRelation                                   |
+    | physical_plan | NestedLoopJoinExec: join_type=RightSemi             |
+    |               |   PlaceholderRowExec                                |
+    |               |   DataSourceExec: partitions=1, partition_sizes=[1] |
+    |               |                                                     |
+    +---------------+-----------------------------------------------------+
+    "
     );
 
     Ok(())
diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt
index 4d61b254f507..c32a5e6b33d6 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -429,14 +429,11 @@ logical_plan
 01)LeftSemi Join: 
 02)--TableScan: t1 projection=[a]
 03)--SubqueryAlias: __correlated_sq_1
-04)----Projection:
-05)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]
-06)--------TableScan: t2 projection=[]
+04)----EmptyRelation
 physical_plan
 01)NestedLoopJoinExec: join_type=LeftSemi
 02)--DataSourceExec: partitions=1, partition_sizes=[0]
-03)--ProjectionExec: expr=[]
-04)----PlaceholderRowExec
+03)--PlaceholderRowExec
 
 statement ok
 drop table t1;
diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt
index f57c50506893..21de2d49fba2 100644
--- a/datafusion/sqllogictest/test_files/explain_tree.slt
+++ b/datafusion/sqllogictest/test_files/explain_tree.slt
@@ -1263,14 +1263,11 @@ physical_plan
 04)│    join_type: LeftSemi    │              │
 05)└─────────────┬─────────────┘              │
 06)┌─────────────┴─────────────┐┌─────────────┴─────────────┐
-07)│       DataSourceExec      ││       ProjectionExec      │
+07)│       DataSourceExec      ││     PlaceholderRowExec    │
 08)│    --------------------   ││                           │
 09)│          files: 1         ││                           │
 10)│        format: csv        ││                           │
-11)└───────────────────────────┘└─────────────┬─────────────┘
-12)-----------------------------┌─────────────┴─────────────┐
-13)-----------------------------│     PlaceholderRowExec    │
-14)-----------------------------└───────────────────────────┘
+11)└───────────────────────────┘└───────────────────────────┘
 
 # Query with cross join.
 query TT
diff --git a/datafusion/sqllogictest/test_files/expr/date_part.slt b/datafusion/sqllogictest/test_files/expr/date_part.slt
index df17a8bca968..64f16f72421a 100644
--- a/datafusion/sqllogictest/test_files/expr/date_part.slt
+++ b/datafusion/sqllogictest/test_files/expr/date_part.slt
@@ -1089,4 +1089,4 @@ SELECT EXTRACT("isodow" FROM to_timestamp('2020-09-08T12:00:00+00:00'))
 query I
 SELECT EXTRACT('isodow' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
 ----
-1
\ No newline at end of file
+1
diff --git a/datafusion/sqllogictest/test_files/issue_17138.slt b/datafusion/sqllogictest/test_files/issue_17138.slt
new file mode 100644
index 000000000000..d7dcf8d4dbdc
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/issue_17138.slt
@@ -0,0 +1,36 @@
+statement ok
+CREATE TABLE tab1(col0 INTEGER, col1 INTEGER, col2 INTEGER)
+
+statement ok
+INSERT INTO tab1 VALUES(51,14,96)
+
+query R
+SELECT NULL * AVG(DISTINCT 4) + SUM(col1) AS col0 FROM tab1
+----
+NULL
+
+query TT
+EXPLAIN SELECT NULL * AVG(DISTINCT 4) + SUM(col1) AS col0 FROM tab1
+----
+logical_plan
+01)Projection: Float64(NULL) AS col0
+02)--EmptyRelation
+physical_plan
+01)ProjectionExec: expr=[NULL as col0]
+02)--PlaceholderRowExec
+
+# Similar, with a few more arithmetic operations
+query R
+SELECT + CAST ( NULL AS INTEGER ) * + + AVG ( DISTINCT 4 ) + - SUM ( ALL + col1 ) AS col0 FROM tab1
+----
+NULL
+
+query TT
+EXPLAIN SELECT + CAST ( NULL AS INTEGER ) * + + AVG ( DISTINCT 4 ) + - SUM ( ALL + col1 ) AS col0 FROM tab1
+----
+logical_plan
+01)Projection: Float64(NULL) AS col0
+02)--EmptyRelation
+physical_plan
+01)ProjectionExec: expr=[NULL as col0]
+02)--PlaceholderRowExec
diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/getbit.slt b/datafusion/sqllogictest/test_files/spark/bitwise/getbit.slt
index ca1b2724a8ce..7cfdfe825727 100644
--- a/datafusion/sqllogictest/test_files/spark/bitwise/getbit.slt
+++ b/datafusion/sqllogictest/test_files/spark/bitwise/getbit.slt
@@ -73,4 +73,3 @@ query I
 SELECT getbit(11, NULL);
 ----
 NULL
-
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index ed73eecda03e..e33271cf6de9 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -1453,9 +1453,7 @@ logical_plan
 01)LeftSemi Join: 
 02)--TableScan: t1 projection=[a]
 03)--SubqueryAlias: __correlated_sq_1
-04)----Projection:
-05)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]
-06)--------TableScan: t2 projection=[]
+04)----EmptyRelation
 
 statement count 0
 drop table t1;