Repository: ibis-project/ibis

Issue:
bug: wrong results when a group_by is followed by a filter
Note: I am not sure whether this one should stay here or go to the ibis_bigquery repo.

However, I think this one is quite serious because it leads very hard-to-detect bugs.

```
import ibis
import ibis_bigquery
from google.cloud import bigquery
import pandas as pd

my_table = 'my_table'

pdf = pd.DataFrame({'a': [1, 1, 1], 'b': [1, 2, 3]})

# Load client
client = bigquery.Client(project=my_project)

# Load data to BQ
client.delete_table(my_dataset + "." + my_table, not_found_ok=True)
job = client.load_table_from_dataframe(pdf, my_dataset + "." + my_table)

ibis.options.interactive = False

conn = ibis_bigquery.connect(
    project_id=my_project,
    dataset_id=my_dataset)

t = conn.table(my_table)

e1 = t.group_by(["a"]) \
    .aggregate([t.b.sum().name("b"),
                ])

e2 = e1.filter(e1.b == 6)

print(e2.execute())

print(ibis_bigquery.compile(e2))
```

In the sql, the filter is inside the group by which is wrong:

```
SELECT `a`, sum(`b`) AS `b`
FROM `my_project.my_dataset.my_table`
WHERE `b` = 6
GROUP BY 1

```
Now, if I just change the name of the aggregated attribute:

```

e1 = t.group_by(["a"]) \
    .aggregate([t.b.sum().name("sb"),
                ])

e2 = e1.filter(e1.sb == 6)

print(e2.execute())

print(ibis_bigquery.compile(e2))

```
the sql is now correct:

```
SELECT *
FROM (
  SELECT `a`, sum(`b`) AS `sb`
  FROM `my_project.my_dataset.my_table`
  GROUP BY 1
) t0
WHERE `sb` = 6
```


Discussion:
Seems like the problem exists also in SQLAlchemy backends, doesn't seem to be BigQuery specific.

```python
import ibis

con = ibis.sqlite.connect('/home/mgarcia/src/ibis/ci/ibis-testing-data/ibis_testing.db')

t = con.table('functional_alltypes')

e1 = (t.group_by(["int_col"])
       .aggregate([t.bigint_col.sum().name("bigint_col")]))

e2 = e1.filter(e1.bigint_col == 60)

print(e2.compile())
```
Returns:
```sql
SELECT t0.int_col, sum(t0.bigint_col) AS bigint_col 
FROM base.functional_alltypes AS t0 
WHERE t0.bigint_col = ?
GROUP BY t0.int_col
```
Bugs like this one are good motivating cases for why the internal data structures need to be simplified, and most of the expression analysis removed until we have a better design for it.

The problem here is that the expression analysis code is assuming that if something is referring to `bigint_col` it's the same `bigint_col` from the leaf table and therefore eligible for pushdown.

Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -620,7 +620,12 @@ def apply_filter(expr, predicates):
         # Potential fusion opportunity
         # GH1344: We can't sub in things with correlated subqueries
         simplified_predicates = [
-            sub_for(predicate, [(expr, op.table)])
+            # Originally this line tried substituting op.table in for expr, but
+            # that is too aggressive in the presence of filters that occur
+            # after aggregations.
+            #
+            # See https://github.com/ibis-project/ibis/pull/3341 for details
+            sub_for(predicate, [(op.table, expr)])
             if not has_reduction(predicate)
             else predicate
             for predicate in predicates


Test patch:
diff --git a/ibis/tests/sql/test_compiler.py b/ibis/tests/sql/test_compiler.py
--- a/ibis/tests/sql/test_compiler.py
+++ b/ibis/tests/sql/test_compiler.py
@@ -2038,20 +2038,23 @@ def test_self_join_filter_analysis_bug(self):
         expr, _ = self._case_filter_self_join_analysis_bug()
 
         expected = """\
-SELECT t0.`region`, t0.`total` - t1.`total` AS `diff`
-FROM (
+WITH t0 AS (
   SELECT `region`, `kind`, sum(`amount`) AS `total`
   FROM purchases
-  WHERE `kind` = 'foo'
   GROUP BY 1, 2
-) t0
+)
+SELECT t1.`region`, t1.`total` - t2.`total` AS `diff`
+FROM (
+  SELECT *
+  FROM t0
+  WHERE `kind` = 'foo'
+) t1
   INNER JOIN (
-    SELECT `region`, `kind`, sum(`amount`) AS `total`
-    FROM purchases
+    SELECT *
+    FROM t0
     WHERE `kind` = 'bar'
-    GROUP BY 1, 2
-  ) t1
-    ON t0.`region` = t1.`region`"""
+  ) t2
+    ON t1.`region` = t2.`region`"""
         self._compare_sql(expr, expected)
 
     def test_join_filtered_tables_no_pushdown(self):
diff --git a/ibis/tests/sql/test_sqlalchemy.py b/ibis/tests/sql/test_sqlalchemy.py
--- a/ibis/tests/sql/test_sqlalchemy.py
+++ b/ibis/tests/sql/test_sqlalchemy.py
@@ -691,11 +691,41 @@ def test_where_correlated_subquery_with_join(self):
 
         self._compare_sqla(expr, ex)
 
+    def test_filter_group_by_agg_with_same_name(self):
+        # GH 2907
+        t = ibis.table(
+            [("int_col", "int32"), ("bigint_col", "int64")], name="t"
+        )
+        expr = (
+            t.group_by("int_col")
+            .aggregate(bigint_col=lambda t: t.bigint_col.sum())
+            .filter(lambda t: t.bigint_col == 60)
+        )
+
+        t1 = sa.table(
+            "t", sa.column("int_col"), sa.column("bigint_col")
+        ).alias("t1")
+        t0 = (
+            sa.select(
+                [
+                    t1.c.int_col.label("int_col"),
+                    sa.func.sum(t1.c.bigint_col).label("bigint_col"),
+                ]
+            )
+            .group_by(t1.c.int_col)
+            .alias("t0")
+        )
+        ex = sa.select([t0]).where(t0.c.bigint_col == 60)
+
+        self._compare_sqla(expr, ex)
+
     def _compare_sqla(self, expr, sqla):
         context = AlchemyContext(compiler=AlchemyCompiler)
         result_sqla = AlchemyCompiler.to_sql(expr, context)
-        result = str(result_sqla.compile())
-        expected = str(sqla.compile())
+        result = str(
+            result_sqla.compile(compile_kwargs=dict(literal_binds=True))
+        )
+        expected = str(sqla.compile(compile_kwargs=dict(literal_binds=True)))
         assert result == expected
 
     def _to_sqla(self, table):