Repository: ibis-project/ibis

Issue:
bug: .isin(Expr) returns wrong results
### What happened?

Not sure exactly what conditions cause it to not work, but here is one example:

```python
import pandas as pd
import ibis
ibis.options.interactive = True

df = pd.DataFrame(
    {
        "x": [1, 2, 3, 4, 5],
    }
)
t = ibis.memtable(df)
good = t[t.x > 2].x
t.x.isin(good)
```
results in
```
┏━━━━━━━━━━━━━━━━┓
┃ Contains(x, x) ┃
┡━━━━━━━━━━━━━━━━┩
│ boolean        │
├────────────────┤
│ True           │
│ True           │
│ True           │
│ True           │
│ True           │
└────────────────┘
```
when the first two should be False

### What version of ibis are you using?

master

### What backend(s) are you using, if any?

duckdb

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Thanks for the report @NickCrews !

That's definitely a bug.

We seem to be dropping the `where` clause in the subselection:

```python
[ins] In [12]: expr = t.x.isin(good)

[ins] In [13]: ibis.show_sql(expr, dialect="duckdb")
SELECT
  t0.x IN (
    SELECT
      t0.x
    FROM _ibis_memtable_bzgyzryhtfcdoxi7m7v2p187i AS t0
  ) AS "Contains(x, x)"
FROM _ibis_memtable_bzgyzryhtfcdoxi7m7v2p187i AS t0
```

Resolving patch:
diff --git a/ibis/backends/base/sql/alchemy/registry.py b/ibis/backends/base/sql/alchemy/registry.py
--- a/ibis/backends/base/sql/alchemy/registry.py
+++ b/ibis/backends/base/sql/alchemy/registry.py
@@ -189,14 +189,15 @@ def _contains(func):
     def translate(t, op):
         left = t.translate(op.value)
 
-        if isinstance(op.options, tuple):
+        options = op.options
+        if isinstance(options, tuple):
             right = [t.translate(x) for x in op.options]
-        elif op.options.output_shape.is_columnar():
-            right = t.translate(op.options)
+        elif options.output_shape.is_columnar():
+            right = t.translate(ops.TableArrayView(options.to_expr().as_table()))
             if not isinstance(right, sa.sql.Selectable):
                 right = sa.select(right)
         else:
-            right = t.translate(op.options)
+            right = t.translate(options)
 
         return func(left, right)
 


Test patch:
diff --git a/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/snowflake/out.sql b/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/snowflake/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/snowflake/out.sql
@@ -0,0 +1,32 @@
+WITH t0 AS (
+  SELECT
+    t5.street AS street,
+    ROW_NUMBER() OVER (ORDER BY t5.street) - 1 AS key
+  FROM data AS t5
+), t1 AS (
+  SELECT
+    t0.key AS key
+  FROM t0
+), t2 AS (
+  SELECT
+    t0.street AS street,
+    t0.key AS key
+  FROM t0
+  JOIN t1
+    ON t0.key = t1.key
+), t3 AS (
+  SELECT
+    t2.street AS street,
+    ROW_NUMBER() OVER (ORDER BY t2.street) - 1 AS key
+  FROM t2
+), t4 AS (
+  SELECT
+    t3.key AS key
+  FROM t3
+)
+SELECT
+  t3.street,
+  t3.key
+FROM t3
+JOIN t4
+  ON t3.key = t4.key
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/bigquery/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/bigquery/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/bigquery/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.`x` IN (
+    SELECT
+      t1.`x`
+    FROM (
+      SELECT
+        t0.*
+      FROM t AS t0
+      WHERE
+        t0.`x` > 2
+    ) AS t1
+  ) AS `tmp`
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/clickhouse/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/clickhouse/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/clickhouse/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        *
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  )
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/duckdb/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/duckdb/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/duckdb/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > CAST(2 AS SMALLINT)
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/impala/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/impala/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/impala/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.`x` IN (
+    SELECT
+      t1.`x`
+    FROM (
+      SELECT
+        t0.*
+      FROM t AS t0
+      WHERE
+        t0.`x` > 2
+    ) AS t1
+  ) AS `Contains(x, x)`
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mssql/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mssql/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mssql/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mysql/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mysql/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/mysql/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS `Contains(x, x)`
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/postgres/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/postgres/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/postgres/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/snowflake/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/snowflake/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/snowflake/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/sqlite/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/sqlite/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/sqlite/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/snapshots/test_sql/test_isin_bug/trino/out.sql b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/trino/out.sql
new file mode 100644
--- /dev/null
+++ b/ibis/backends/tests/snapshots/test_sql/test_isin_bug/trino/out.sql
@@ -0,0 +1,13 @@
+SELECT
+  t0.x IN (
+    SELECT
+      t1.x
+    FROM (
+      SELECT
+        t0.x AS x
+      FROM t AS t0
+      WHERE
+        t0.x > 2
+    ) AS t1
+  ) AS "Contains(x, x)"
+FROM t AS t0
\ No newline at end of file
diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py
--- a/ibis/backends/tests/test_sql.py
+++ b/ibis/backends/tests/test_sql.py
@@ -126,3 +126,13 @@ def test_cte_refs_in_topo_order(backend, snapshot):
 
     sql = str(ibis.to_sql(mr3, dialect=backend.name()))
     snapshot.assert_match(sql, "out.sql")
+
+
+@pytest.mark.never(
+    ["pandas", "dask", "datafusion", "polars", "pyspark"], reason="not SQL"
+)
+def test_isin_bug(con, snapshot):
+    t = ibis.table(dict(x="int"), name="t")
+    good = t[t.x > 2].x
+    expr = t.x.isin(good)
+    snapshot.assert_match(str(ibis.to_sql(expr, dialect=con.name)), "out.sql")
diff --git a/ibis/tests/sql/snapshots/test_sqlalchemy/test_where_uncorrelated_subquery/out.sql b/ibis/tests/sql/snapshots/test_sqlalchemy/test_where_uncorrelated_subquery/out.sql
--- a/ibis/tests/sql/snapshots/test_sqlalchemy/test_where_uncorrelated_subquery/out.sql
+++ b/ibis/tests/sql/snapshots/test_sqlalchemy/test_where_uncorrelated_subquery/out.sql
@@ -7,6 +7,6 @@ FROM foo AS t0
 WHERE
   t0.job IN (
     SELECT
-      bar.job
-    FROM bar
+      t1.job
+    FROM bar AS t1
   )
\ No newline at end of file