Repository: ibis-project/ibis

Issue:
bug: mutate + filter compiles to unintentionnal cross join
The following expression is being compiled to an SQL cross-join although my intention was only to filter the original table.
I am missusing the API or is this a bug ?

```
ibis-framework      1.3.0
Python 3.6.7
```

## Current behavior

```python
person = ibis.table([('person_id', 'int64'), ('birth_datetime', 'timestamp')], name='person')

mutated = (
    person
    .mutate((2021 - person.birth_datetime.year()).name('age'))
)
selected_person = (
    mutated
    .filter(mutated.age <= 40)
    [mutated.person_id, mutated.age]
)

print(ibis.postgres.compile(selected_person))
```
Compiles to the following query which is a cross join (`SELECT FROM [table a], [table b]`).

```sql
SELECT person_id,
       t0.age
FROM
  (SELECT t1.person_id AS person_id,
          t1.birth_datetime AS birth_datetime
   FROM person AS t1),
  (SELECT t1.person_id AS person_id,
          t1.birth_datetime AS birth_datetime,
          :param_1 - CAST(EXTRACT(YEAR
                                  FROM t1.birth_datetime) AS SMALLINT) AS age
   FROM person AS t1) AS t0
WHERE t0.age <= :param_2
```

## Expected behavior

My intention was to filter the original table, corresponding to the following query:

```sql
SELECT t0.person_id,
       t0.age
FROM
  (SELECT t1.person_id AS person_id,
          t1.birth_datetime AS birth_datetime,
          :param_1 - CAST(EXTRACT(YEAR
                                  FROM t1.birth_datetime) AS SMALLINT) AS age
   FROM person AS t1) AS t0
WHERE t0.age <= :param_2
```

Note that small modifications to the python code will indeed compile to the correct query:

```python
person = ibis.table([('person_id', 'int64'), ('birth_datetime', 'timestamp')], name='person')

mutated = (
    person
    # [person.person_id, person.birth_datetime]  # "pre selecting" the columns produces the correct query
    .mutate((2021 - person.birth_datetime.year()).name('age'))
)
selected_person = (
    mutated
    .filter(mutated.age <= 40)
    [mutated.person_id, mutated.age]  #  using ["person_id", "age"]  instead of the column expressions
                                                              # will also compile to the correct query
)

print(ibis.postgres.compile(selected_person))
```




Discussion:
This is definitely a bug! Thanks for posting.
@BenjaminHabert I looked into this yesterday and it's a fairly tricky bug to fix. However, we're still aiming to fix it for 3.0.0.

In the meantime, as you noticed you can workaround the issue by referring to the columns in the final projection in your example using strings.

Thanks again for the bug report!

Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -895,8 +895,7 @@ def try_fusion(self, root):
         root_selections = root.selections
         parent_op = self.parent.op()
         for val in resolved:
-            # XXX
-            lifted_val = substitute_parents(val)
+            lifted_val = substitute_parents(val, past_projection=False)
 
             # a * projection
             if isinstance(val, ir.TableExpr) and (
@@ -968,7 +967,10 @@ def _among_roots(self, node):
         return self.roots_shared(node) > 0
 
     def roots_shared(self, node):
-        return sum(root.is_ancestor(node) for root in self.roots)
+        return sum(
+            root.is_ancestor(node) or node.compatible_with(root)
+            for root in self.roots
+        )
 
     def shares_some_roots(self, expr):
         expr_roots = expr.op().root_tables()


Test patch:
diff --git a/ibis/tests/sql/test_compiler.py b/ibis/tests/sql/test_compiler.py
--- a/ibis/tests/sql/test_compiler.py
+++ b/ibis/tests/sql/test_compiler.py
@@ -898,7 +898,7 @@ def test_join_between_joins(self):
 
         result = Compiler.to_sql(projected)
         expected = """\
-SELECT t0.*, `value3`, `value4`
+SELECT t0.*, t1.`value3`, t1.`value4`
 FROM (
   SELECT t2.*, t3.`value2`
   FROM `first` t2
@@ -1147,25 +1147,26 @@ def test_fuse_projections(self):
         table2_filtered = table2[pred]
 
         f2 = (table2['foo'] * 2).name('qux')
-        f3 = (table['foo'] * 2).name('qux')
 
         table3 = table2.projection([table2, f2])
 
         # fusion works even if there's a filter
         table3_filtered = table2_filtered.projection([table2, f2])
 
-        expected = table[table, f1, f3]
-        expected2 = table[pred][table, f1, f3]
-
-        assert table3.equals(expected)
-        assert table3_filtered.equals(expected2)
-
-        ex_sql = """SELECT *, `foo` + `bar` AS `baz`, `foo` * 2 AS `qux`
-FROM tbl"""
+        ex_sql = """\
+SELECT *, `foo` * 2 AS `qux`
+FROM (
+  SELECT *, `foo` + `bar` AS `baz`
+  FROM tbl
+) t0"""
 
-        ex_sql2 = """SELECT *, `foo` + `bar` AS `baz`, `foo` * 2 AS `qux`
-FROM tbl
-WHERE `value` > 0"""
+        ex_sql2 = """\
+SELECT *, `foo` * 2 AS `qux`
+FROM (
+  SELECT *, `foo` + `bar` AS `baz`
+  FROM tbl
+  WHERE `value` > 0
+) t0"""
 
         table3_sql = Compiler.to_sql(table3)
         table3_filt_sql = Compiler.to_sql(table3_filtered)
@@ -1173,18 +1174,6 @@ def test_fuse_projections(self):
         assert table3_sql == ex_sql
         assert table3_filt_sql == ex_sql2
 
-        # Use the intermediate table refs
-        table3 = table2.projection([table2, f2])
-
-        # fusion works even if there's a filter
-        table3_filtered = table2_filtered.projection([table2, f2])
-
-        expected = table[table, f1, f3]
-        expected2 = table[pred][table, f1, f3]
-
-        assert table3.equals(expected)
-        assert table3_filtered.equals(expected2)
-
     def test_projection_filter_fuse(self):
         expr1, expr2, expr3 = self._case_projection_fuse_filter()
 
@@ -1226,7 +1215,7 @@ def test_bug_project_multiple_times(self):
     INNER JOIN tpch_region t4
       ON t3.`n_regionkey` = t4.`r_regionkey`
 )
-SELECT `c_name`, `r_name`, `n_name`
+SELECT t0.`c_name`, t0.`r_name`, t0.`n_name`
 FROM t0
   LEFT SEMI JOIN (
     SELECT *
@@ -1454,7 +1443,7 @@ def test_self_join_subquery_distinct_equal(self):
     INNER JOIN tpch_nation t3
       ON t2.`r_regionkey` = t3.`n_regionkey`
 )
-SELECT `r_name`, t1.`n_name`
+SELECT t0.`r_name`, t1.`n_name`
 FROM t0
   INNER JOIN t0 t1
     ON t0.`r_regionkey` = t1.`r_regionkey`"""
diff --git a/ibis/tests/sql/test_sqlalchemy.py b/ibis/tests/sql/test_sqlalchemy.py
--- a/ibis/tests/sql/test_sqlalchemy.py
+++ b/ibis/tests/sql/test_sqlalchemy.py
@@ -715,6 +715,27 @@ def test_filter_group_by_agg_with_same_name(self):
             .alias("t0")
         )
         ex = sa.select([t0]).where(t0.c.bigint_col == 60)
+        self._compare_sqla(expr, ex)
+
+    def test_mutate_filter_join_no_cross_join(self):
+        person = ibis.table(
+            [('person_id', 'int64'), ('birth_datetime', 'timestamp')],
+            name='person',
+        )
+        mutated = person.mutate(age=400)
+        expr = mutated.filter(mutated.age <= 40)[mutated.person_id]
+
+        person = sa.table(
+            "person", sa.column("person_id"), sa.column("birth_datetime")
+        ).alias("t1")
+        t0 = sa.select(
+            [
+                person.c.person_id,
+                person.c.birth_datetime,
+                sa.literal(400).label("age"),
+            ]
+        ).alias("t0")
+        ex = sa.select([t0.c.person_id]).where(t0.c.age <= 40)
 
         self._compare_sqla(expr, ex)