Repository: ibis-project/ibis

Issue:
bug: clickhouse and impala semi/anti join code is wrong
Impala and clickhouse are generating incorrect code in the tests from https://github.com/ibis-project/ibis/pull/3501.


Discussion:

Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -351,20 +351,6 @@ def _lift_TableColumn(self, expr, block=None):
 
         if isinstance(root, ops.Selection):
             can_lift = False
-            all_simple_columns = all(
-                isinstance(sel.op(), ops.TableColumn)
-                and sel.op().name == sel.get_name()
-                for sel in root.selections
-                if isinstance(sel, ir.ValueExpr)
-                if sel.has_name()
-            )
-
-            # If there are predicates or sort keys,
-            # then we cannot lift the root
-            no_predicates_or_sort_keys = not (
-                root.predicates or root.sort_keys
-            )
-
             for val in root.selections:
                 value_op = val.op()
                 if (
@@ -373,15 +359,6 @@ def _lift_TableColumn(self, expr, block=None):
                 ):
                     can_lift = True
                     lifted_root = self.lift(val)
-                elif (
-                    all_simple_columns
-                    and no_predicates_or_sort_keys
-                    and isinstance(val, ir.ValueExpr)
-                    and val.has_name()
-                    and node.name == val.get_name()
-                ):
-                    can_lift = True
-                    lifted_root = self.lift(value_op.table)
 
             if can_lift and not block:
                 lifted_node = ops.TableColumn(lifted_root, node.name)


Test patch:
diff --git a/ibis/backends/tests/test_join.py b/ibis/backends/tests/test_join.py
--- a/ibis/backends/tests/test_join.py
+++ b/ibis/backends/tests/test_join.py
@@ -133,17 +133,11 @@ def test_mutating_join(backend, con, batting, awards_players, how):
     [
         param(
             "semi",
-            marks=pytest.mark.notimpl(
-                ["clickhouse", "dask", "datafusion", "impala"]
-            ),
+            marks=pytest.mark.notimpl(["dask", "datafusion"]),
         ),
         param(
             "anti",
-            marks=pytest.mark.notimpl(
-                # clickhouse and impala are broken, the rest are not
-                # implemented
-                ["clickhouse", "dask", "datafusion", "impala"]
-            ),
+            marks=pytest.mark.notimpl(["dask", "datafusion"]),
         ),
     ],
 )
@@ -195,3 +189,16 @@ def test_join_then_filter_no_column_overlap(awards_players, batting):
     filters = [expr.RBI == 9]
     q = expr.filter(filters)
     q.execute()
+
+
+@pytest.mark.notimpl(["datafusion"])
+@pytest.mark.notyet(
+    ["pyspark"],
+    reason="pyspark doesn't support joining on differing column names",
+)
+def test_mutate_then_join_no_column_overlap(batting, awards_players):
+    left = batting.mutate(year=batting.yearID)
+    left = left["year", "RBI"]
+    right = awards_players
+    expr = left.join(right, left.year == right.yearID)
+    expr.execute()
diff --git a/ibis/tests/expr/test_analysis.py b/ibis/tests/expr/test_analysis.py
--- a/ibis/tests/expr/test_analysis.py
+++ b/ibis/tests/expr/test_analysis.py
@@ -41,25 +41,6 @@ def test_rewrite_join_projection_without_other_ops(con):
     assert not op.table.equals(ex_expr)
 
 
-def test_rewrite_past_projection(con):
-    table = con.table('test1')
-
-    # Rewrite past a projection
-    table3 = table[['c', 'f']]
-    expr = table3['c'] == 2
-
-    result = L.substitute_parents(expr)
-    expected = table['c'] == 2
-    assert_equal(result, expected)
-
-    # Unsafe to rewrite past projection
-    table5 = table[(table.f * 2).name('c'), table.f]
-    expr = table5['c'] == 2
-    result = L.substitute_parents(expr)
-    expected = expr
-    assert result.equals(expected)
-
-
 def test_multiple_join_deeper_reference():
     # Join predicates down the chain might reference one or more root
     # tables in the hierarchy.