Repository: ibis-project/ibis

Issue:
Error when trying to join tables with Pandas backend
Hello, I'm trying to perform an outer join on two tables with the Pandas backend and am unable to do so due to a signature error. Here is an example code of what I'm trying to do:

```
import ibis
import pandas as pd
from ibis.backends.pandas import udf
from ibis.expr.api import NumericValue
import ibis.expr.datatypes as dt

df = pd.DataFrame({"test": [1, 2, 3], "name": ["a", "b", "c"]})
df_2 = pd.DataFrame({"test_2": [1, 5, 6], "name_2": ["d", "e", "f"]})

conn = ibis.pandas.connect({"df": df, "df_2": df_2})

ibis_table_1 = conn.table("df")
ibis_table_2 = conn.table("df_2")

joined = ibis_table_1.outer_join(
    ibis_table_2, predicates=ibis_table_1["test"] == ibis_table_2["test_2"])
joined = joined.materialize()
result = joined.execute()
print(result)
```

The error I get from the above is:

```
Traceback (most recent call last):
  File "pandas_test.py", line 22, in <module>
    result = joined.execute()
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\expr\types.py", line 273, in execute
    return self._find_backend().execute(
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\__init__.py", line 121, in execute
    return execute_and_reset(query, params=params, **kwargs)
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\core.py", line 487, in execute_and_reset
    result = execute(
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\multipledispatch\dispatcher.py", line 278, in __call__
    return func(*args, **kwargs)
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\trace.py", line 138, in traced_func
    return func(*args, **kwargs)
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\core.py", line 432, in main_execute
    return execute_with_scope(
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\core.py", line 218, in execute_with_scope
    result = execute_until_in_scope(
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\trace.py", line 138, in traced_func
    return func(*args, **kwargs)
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\ibis\backends\pandas\core.py", line 359, in execute_until_in_scope
    result = execute_node(
  File "C:\Users\tanaimn\.conda\envs\ibis_test\lib\site-packages\multipledispatch\dispatcher.py", line 273, in __call__
    raise NotImplementedError(
NotImplementedError: Could not find signature for execute_node: <MaterializedJoin, DataFrame>
```

I've tried this on both Ibis 1.4.0 and 2.0.0 and face the same error using both. However, I do notice that it seems like joins have already been implemented for the Pandas backend so is this a mistake of implementation on my end and/or is there a workaround? Many thanks in advance for any help!


Discussion:
This is an honest-to-goodness bug. Thanks for the report @tengznaim! I'll have a PR up shortly to fix this.

Resolving patch:
diff --git a/ibis/backends/dask/execution/join.py b/ibis/backends/dask/execution/join.py
--- a/ibis/backends/dask/execution/join.py
+++ b/ibis/backends/dask/execution/join.py
@@ -15,6 +15,11 @@
 from ..execution import constants
 
 
+@execute_node.register(ops.MaterializedJoin, dd.DataFrame)
+def execute_materialized_join(op, df, **kwargs):
+    return df
+
+
 @execute_node.register(
     ops.AsOfJoin, dd.DataFrame, dd.DataFrame, (Timedelta, type(None))
 )
@@ -69,9 +74,9 @@ def execute_cross_join(op, left, right, **kwargs):
     return result
 
 
-# TODO - execute_materialized_join - #2553
+# TODO - execute_join - #2553
 @execute_node.register(ops.Join, dd.DataFrame, dd.DataFrame)
-def execute_materialized_join(op, left, right, **kwargs):
+def execute_join(op, left, right, **kwargs):
     op_type = type(op)
 
     try:
diff --git a/ibis/backends/pandas/execution/join.py b/ibis/backends/pandas/execution/join.py
--- a/ibis/backends/pandas/execution/join.py
+++ b/ibis/backends/pandas/execution/join.py
@@ -55,7 +55,7 @@ def execute_cross_join(op, left, right, **kwargs):
 
 
 @execute_node.register(ops.Join, pd.DataFrame, pd.DataFrame)
-def execute_materialized_join(op, left, right, **kwargs):
+def execute_join(op, left, right, **kwargs):
     op_type = type(op)
 
     try:
@@ -94,6 +94,11 @@ def execute_materialized_join(op, left, right, **kwargs):
     return df
 
 
+@execute_node.register(ops.MaterializedJoin, pd.DataFrame)
+def execute_materialized_join(op, df, **kwargs):
+    return df
+
+
 @execute_node.register(
     ops.AsOfJoin, pd.DataFrame, pd.DataFrame, (pd.Timedelta, type(None))
 )


Test patch:
diff --git a/ibis/backends/dask/tests/execution/test_join.py b/ibis/backends/dask/tests/execution/test_join.py
--- a/ibis/backends/dask/tests/execution/test_join.py
+++ b/ibis/backends/dask/tests/execution/test_join.py
@@ -237,7 +237,7 @@ def test_multi_join_with_post_expression_filter(how, left, df1):
     )
 
 
-@pytest.mark.xfail(reason="TODO - execute_materialized_join - #2553")
+@pytest.mark.xfail(reason="TODO - execute_join - #2553")
 @join_type
 def test_join_with_non_trivial_key(how, left, right, df1, df2):
     # also test that the order of operands in the predicate doesn't matter
@@ -261,7 +261,7 @@ def test_join_with_non_trivial_key(how, left, right, df1, df2):
     )
 
 
-@pytest.mark.xfail(reason="TODO - execute_materialized_join - #2553")
+@pytest.mark.xfail(reason="TODO - execute_join - #2553")
 @join_type
 def test_join_with_non_trivial_key_project_table(how, left, right, df1, df2):
     # also test that the order of operands in the predicate doesn't matter
@@ -496,3 +496,37 @@ def test_select_on_unambiguous_asof_join(func, npartitions):
         result.compute(scheduler='single-threaded'),
         expected.compute(scheduler='single-threaded'),
     )
+
+
+def test_materialized_join(npartitions):
+    df = dd.from_pandas(
+        pd.DataFrame({"test": [1, 2, 3], "name": ["a", "b", "c"]}),
+        npartitions=npartitions,
+    )
+    df_2 = dd.from_pandas(
+        pd.DataFrame({"test_2": [1, 5, 6], "name_2": ["d", "e", "f"]}),
+        npartitions=npartitions,
+    )
+
+    conn = ibis.dask.connect({"df": df, "df_2": df_2})
+
+    ibis_table_1 = conn.table("df")
+    ibis_table_2 = conn.table("df_2")
+
+    joined = ibis_table_1.outer_join(
+        ibis_table_2,
+        predicates=ibis_table_1["test"] == ibis_table_2["test_2"],
+    )
+    joined = joined.materialize()
+    result = joined.compile()
+    expected = dd.merge(
+        df,
+        df_2,
+        left_on="test",
+        right_on="test_2",
+        how="outer",
+    )
+    tm.assert_frame_equal(
+        result.compute(scheduler='single-threaded'),
+        expected.compute(scheduler='single-threaded'),
+    )
diff --git a/ibis/backends/pandas/tests/execution/test_join.py b/ibis/backends/pandas/tests/execution/test_join.py
--- a/ibis/backends/pandas/tests/execution/test_join.py
+++ b/ibis/backends/pandas/tests/execution/test_join.py
@@ -413,3 +413,28 @@ def test_select_on_unambiguous_asof_join(func):
     expr = func(join)
     result = expr.execute()
     tm.assert_frame_equal(result, expected)
+
+
+def test_materialized_join():
+    df = pd.DataFrame({"test": [1, 2, 3], "name": ["a", "b", "c"]})
+    df_2 = pd.DataFrame({"test_2": [1, 5, 6], "name_2": ["d", "e", "f"]})
+
+    conn = ibis.pandas.connect({"df": df, "df_2": df_2})
+
+    ibis_table_1 = conn.table("df")
+    ibis_table_2 = conn.table("df_2")
+
+    joined = ibis_table_1.outer_join(
+        ibis_table_2,
+        predicates=ibis_table_1["test"] == ibis_table_2["test_2"],
+    )
+    joined = joined.materialize()
+    result = joined.execute()
+    expected = pd.merge(
+        df,
+        df_2,
+        left_on="test",
+        right_on="test_2",
+        how="outer",
+    )
+    tm.assert_frame_equal(result, expected)