Repository: ibis-project/ibis

Issue:
feat: Allow `isin()` to work across multiple base `Table`s
### Is your feature request related to a problem?

I want to do this:
```python
import ibis
import pandas as pd

df = pd.DataFrame(
    {
        "x": [0, 1, 2],
    }
)
t1 = ibis.memtable(df)
t2 = ibis.memtable(df) 
t1.x.isin(t2.x)
```

but results in

```python stacktrace
---------------------------------------------------------------------------
RelationError                             Traceback (most recent call last)
File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/IPython/core/formatters.py:972, in MimeBundleFormatter.__call__(self, obj, include, exclude)
    969     method = get_real_method(obj, self.print_method)
    971     if method is not None:
--> 972         return method(include=include, exclude=exclude)
    973     return None
    974 else:

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/rich/jupyter.py:48, in JupyterMixin._repr_mimebundle_(self, include, exclude, **kwargs)
     41 def _repr_mimebundle_(
     42     self: "ConsoleRenderable",
     43     include: Sequence[str],
     44     exclude: Sequence[str],
     45     **kwargs: Any,
     46 ) -> Dict[str, str]:
     47     console = get_console()
---> 48     segments = list(console.render(self, console.options))
     49     html = _render_segments(segments)
     50     text = console._render_buffer(segments)

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/rich/console.py:1306, in Console.render(self, renderable, options)
   1304 renderable = rich_cast(renderable)
   1305 if hasattr(renderable, "__rich_console__") and not isclass(renderable):
-> 1306     render_iterable = renderable.__rich_console__(self, _options)  # type: ignore[union-attr]
   1307 elif isinstance(renderable, str):
   1308     text_renderable = self.render_str(
   1309         renderable, highlight=_options.highlight, markup=_options.markup
   1310     )

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:544, in Column.__rich_console__(self, console, options)
    542 def __rich_console__(self, console, options):
    543     named = self.name(self.op().name)
--> 544     projection = named.as_table()
    545     return console.render(projection, options=options)

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:540, in Column.as_table(self)
    539 def as_table(self):
--> 540     return self.to_projection()

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:489, in Value.to_projection(self)
    487 roots = find_immediate_parent_tables(self.op())
    488 if len(roots) > 1:
--> 489     raise com.RelationError(
    490         f'Cannot convert {type(self)} expression '
    491         'involving multiple base table references '
    492         'to a projection'
    493     )
    494 table = roots[0].to_expr()
    495 return table.projection([self])

RelationError: Cannot convert <class 'ibis.expr.types.logical.BooleanColumn'> expression involving multiple base table references to a projection
---------------------------------------------------------------------------
RelationError                             Traceback (most recent call last)
File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/IPython/core/formatters.py:706, in PlainTextFormatter.__call__(self, obj)
    699 stream = StringIO()
    700 printer = pretty.RepresentationPrinter(stream, self.verbose,
    701     self.max_width, self.newline,
    702     max_seq_length=self.max_seq_length,
    703     singleton_pprinters=self.singleton_printers,
    704     type_pprinters=self.type_printers,
    705     deferred_pprinters=self.deferred_printers)
--> 706 printer.pretty(obj)
    707 printer.flush()
    708 return stream.getvalue()

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/IPython/lib/pretty.py:410, in RepresentationPrinter.pretty(self, obj)
    407                         return meth(obj, self, cycle)
    408                 if cls is not object \
    409                         and callable(cls.__dict__.get('__repr__')):
--> 410                     return _repr_pprint(obj, self, cycle)
    412     return _default_pprint(obj, self, cycle)
    413 finally:

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/IPython/lib/pretty.py:778, in _repr_pprint(obj, p, cycle)
    776 """A pprint that just redirects to the normal repr function."""
    777 # Find newlines and replace them with p.break_()
--> 778 output = repr(obj)
    779 lines = output.splitlines()
    780 with p.group():

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/core.py:45, in Expr.__repr__(self)
     43 with simple_console.capture() as capture:
     44     try:
---> 45         simple_console.print(self)
     46     except TranslationError as e:
     47         lines = [
     48             "Translation to backend failed",
     49             f"Error message: {e.args[0]}",
     50             "Expression repr follows:",
     51             self._repr(),
     52         ]

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/rich/console.py:1694, in Console.print(self, sep, end, style, justify, overflow, no_wrap, emoji, markup, highlight, width, height, crop, soft_wrap, new_line_start, *objects)
   1692 if style is None:
   1693     for renderable in renderables:
-> 1694         extend(render(renderable, render_options))
   1695 else:
   1696     for renderable in renderables:

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/rich/console.py:1306, in Console.render(self, renderable, options)
   1304 renderable = rich_cast(renderable)
   1305 if hasattr(renderable, "__rich_console__") and not isclass(renderable):
-> 1306     render_iterable = renderable.__rich_console__(self, _options)  # type: ignore[union-attr]
   1307 elif isinstance(renderable, str):
   1308     text_renderable = self.render_str(
   1309         renderable, highlight=_options.highlight, markup=_options.markup
   1310     )

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:544, in Column.__rich_console__(self, console, options)
    542 def __rich_console__(self, console, options):
    543     named = self.name(self.op().name)
--> 544     projection = named.as_table()
    545     return console.render(projection, options=options)

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:540, in Column.as_table(self)
    539 def as_table(self):
--> 540     return self.to_projection()

File ~/Library/Application Support/hatch/env/virtual/noatak-UM6-FHel/noatak/lib/python3.10/site-packages/ibis/expr/types/generic.py:489, in Value.to_projection(self)
    487 roots = find_immediate_parent_tables(self.op())
    488 if len(roots) > 1:
--> 489     raise com.RelationError(
    490         f'Cannot convert {type(self)} expression '
    491         'involving multiple base table references '
    492         'to a projection'
    493     )
    494 table = roots[0].to_expr()
    495 return table.projection([self])

RelationError: Cannot convert <class 'ibis.expr.types.logical.BooleanColumn'> expression involving multiple base table references to a projection
```

### Describe the solution you'd like

Should also work for `notin`. Briefly browsing over the basic operations at https://ibis-project.org/docs/4.0.0/api/expressions/generic/, I don't actually see any other operations in this broad category.

### What version of ibis are you running?

4.0.0

### What backend(s) are you using, if any?

Should work for all.

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Thanks for the report! This is a bit tricky, but I think it's doable. I'll poke around this week and see what's going on.

Resolving patch:
diff --git a/ibis/backends/clickhouse/compiler/values.py b/ibis/backends/clickhouse/compiler/values.py
--- a/ibis/backends/clickhouse/compiler/values.py
+++ b/ibis/backends/clickhouse/compiler/values.py
@@ -810,8 +810,8 @@ def _string_contains(op, **kw):
 
 
 def contains(op_string: Literal["IN", "NOT IN"]) -> str:
-    def translate(op, *, cache, **kw):
-        import ibis.expr.analysis as an
+    def tr(op, *, cache, **kw):
+        from ibis.backends.clickhouse.compiler import translate
 
         value = op.value
         options = op.options
@@ -827,28 +827,11 @@ def translate(op, *, cache, **kw):
             not isinstance(options, tuple)
             and options.output_shape is rlz.Shape.COLUMNAR
         ):
-            leaves = list(an.find_immediate_parent_tables(options))
-            nleaves = len(leaves)
-            if nleaves > 1:
-                raise NotImplementedError(
-                    "more than one leaf table in a NOT IN/IN query unsupported"
-                )
-            (leaf,) = leaves
-
-            shared_roots_count = sum(
-                an.shares_all_roots(value, child)
-                for child in an.find_immediate_parent_tables(options)
-            )
-            if shared_roots_count == nleaves:
-                from ibis.backends.clickhouse.compiler.relations import translate_rel
-
-                op = options.to_expr().as_table().op()
-                subquery = translate_rel(op, table=cache[leaf], **kw)
-                right_arg = f"({subquery})"
-            else:
-                raise NotImplementedError(
-                    "ClickHouse doesn't support correlated subqueries"
-                )
+            # this will fail to execute if there's a correlation, but it's too
+            # annoying to detect so we let it through to enable the
+            # uncorrelated use case (pandas-style `.isin`)
+            subquery = translate(options.to_expr().as_table().op(), {})
+            right_arg = f"({subquery})"
         else:
             right_arg = translate_val(options, cache=cache, **kw)
 
@@ -856,7 +839,7 @@ def translate(op, *, cache, **kw):
         # make sense to do so for Sequence operations
         return f"{left_arg} {op_string} {right_arg}"
 
-    return translate
+    return tr
 
 
 translate_val.register(ops.Contains)(contains("IN"))
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -159,6 +159,12 @@ def finder(node):
                 return g.halt, node
             else:
                 return g.proceed, None
+
+        # HACK: special case ops.Contains to only consider the needle's base
+        # table, since that's the only expression that matters for determining
+        # cardinality
+        elif isinstance(node, ops.Contains):
+            return [node.value], None
         else:
             return g.proceed, None
 
@@ -646,6 +652,8 @@ def _find_projections(node):
         return g.proceed, None
     elif isinstance(node, ops.TableNode):
         return g.halt, node
+    elif isinstance(node, ops.Contains):
+        return [node.value], None
     else:
         return g.proceed, None
 


Test patch:
diff --git a/ibis/backends/impala/tests/snapshots/test_sql/test_nested_join_multiple_ctes/out.sql b/ibis/backends/impala/tests/snapshots/test_sql/test_nested_join_multiple_ctes/out.sql
--- a/ibis/backends/impala/tests/snapshots/test_sql/test_nested_join_multiple_ctes/out.sql
+++ b/ibis/backends/impala/tests/snapshots/test_sql/test_nested_join_multiple_ctes/out.sql
@@ -1,25 +1,22 @@
 WITH t0 AS (
-  SELECT t3.`userid`, t3.`movieid`, t3.`rating`,
-         CAST(t3.`timestamp` AS timestamp) AS `datetime`
-  FROM ratings t3
+  SELECT t2.`userid`, t2.`movieid`, t2.`rating`,
+         CAST(t2.`timestamp` AS timestamp) AS `datetime`
+  FROM ratings t2
 ),
 t1 AS (
-  SELECT t0.*, t4.`title`
+  SELECT t0.*, t3.`title`
   FROM t0
-    INNER JOIN movies t4
-      ON t0.`movieid` = t4.`movieid`
+    INNER JOIN movies t3
+      ON t0.`movieid` = t3.`movieid`
 )
-SELECT t2.*
-FROM (
-  SELECT t1.*
-  FROM t1
-  WHERE (t1.`userid` = 118205) AND
-        (extract(t1.`datetime`, 'year') > 2001)
-) t2
-WHERE t2.`movieid` IN (
-  SELECT t3.`movieid`
+SELECT t1.*
+FROM t1
+WHERE (t1.`userid` = 118205) AND
+      (extract(t1.`datetime`, 'year') > 2001) AND
+      (t1.`movieid` IN (
+  SELECT t2.`movieid`
   FROM (
-    SELECT t4.`movieid`
+    SELECT t3.`movieid`
     FROM (
       SELECT t1.*
       FROM t1
@@ -27,6 +24,6 @@ WHERE t2.`movieid` IN (
             (extract(t1.`datetime`, 'year') > 2001) AND
             (t1.`userid` = 118205) AND
             (extract(t1.`datetime`, 'year') < 2009)
-    ) t4
-  ) t3
-)
\ No newline at end of file
+    ) t3
+  ) t2
+))
\ No newline at end of file
diff --git a/ibis/backends/tests/test_generic.py b/ibis/backends/tests/test_generic.py
--- a/ibis/backends/tests/test_generic.py
+++ b/ibis/backends/tests/test_generic.py
@@ -839,3 +839,46 @@ def test_typeof(backend, con):
     result = con.execute(expr)
 
     assert result is not None
+
+
+@pytest.mark.broken(["polars"], reason="incorrect answer")
+@pytest.mark.notimpl(["datafusion", "bigquery", "impala", "pyspark"])
+@pytest.mark.notyet(["dask", "mssql"], reason="not supported by the backend")
+def test_isin_uncorrelated(
+    backend, batting, awards_players, batting_df, awards_players_df
+):
+    expr = batting.select(
+        "playerID",
+        "yearID",
+        x=batting.yearID.isin(awards_players.yearID),
+    ).order_by(["playerID", "yearID"])
+    result = expr.execute().x
+    expected = (
+        batting_df.sort_values(["playerID", "yearID"])
+        .reset_index(drop=True)
+        .yearID.isin(awards_players_df.yearID)
+        .rename("x")
+    )
+    backend.assert_series_equal(result, expected)
+
+
+@pytest.mark.broken(["polars"], reason="incorrect answer")
+@pytest.mark.notimpl(["datafusion", "pyspark"])
+@pytest.mark.notyet(["dask"], reason="not supported by the backend")
+def test_isin_uncorrelated_filter(
+    backend, batting, awards_players, batting_df, awards_players_df
+):
+    expr = (
+        batting.select("playerID", "yearID")
+        .filter(batting.yearID.isin(awards_players.yearID))
+        .order_by(["playerID", "yearID"])
+    )
+    result = expr.execute()
+    expected = (
+        batting_df.loc[
+            batting_df.yearID.isin(awards_players_df.yearID), ["playerID", "yearID"]
+        ]
+        .sort_values(["playerID", "yearID"])
+        .reset_index(drop=True)
+    )
+    backend.assert_frame_equal(result, expected)