Repository: ibis-project/ibis

Issue:
feat(pandas): support `memtable` in pandas backend
### What happened?

While working on #5466 I discovered that `ibis.memtable` does not work with the pandas or dask backends:

```python
>>> import ibis
>>> expr = ibis.memtable({"column": [0, 1]})
>>> ibis.get_backend(expr)
<ibis.backends.duckdb.Backend object at 0x1288d7e90>
>>> expr.execute()
   column
0       0
1       1
>>> ibis.set_backend("pandas")
>>> expr.execute()
Traceback (most recent call last):
  Cell In[6], line 2
    expr.execute()
  File ~/code/ibis/ibis/expr/types/core.py:300 in execute
    return self._find_backend(use_default=True).execute(
  File ~/code/ibis/ibis/backends/pandas/__init__.py:240 in execute
    return execute_and_reset(node, params=params, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:489 in execute_and_reset
    result = execute(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:435 in main_execute
    return execute_with_scope(
  File ~/code/ibis/ibis/backends/pandas/core.py:222 in execute_with_scope
    result = execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:354 in execute_until_in_scope
    result = execute_node(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/dispatch.py:32 in raise_unknown_table_node
    raise com.UnboundExpressionError(
UnboundExpressionError: Node of type 'PandasInMemoryTable' has no data bound to it. You probably tried to execute an expression without a data source.
```

dask has a similar problem, probably because it delegates the operation to pandas.

I have also tried with sqlite and polars which work as expected. For backends that do not support the operation we get a more explicit exception, for instance for datafusion:

<details>

```python
Traceback (most recent call last):
  Cell In[13], line 1
    expr.execute()
  File ~/code/ibis/ibis/expr/types/core.py:300 in execute
    return self._find_backend(use_default=True).execute(
  File ~/code/ibis/ibis/backends/datafusion/__init__.py:254 in execute
    output = self.to_pyarrow(expr, params=params, limit=limit, **kwargs)
  File ~/code/ibis/ibis/backends/base/__init__.py:266 in to_pyarrow
    self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs)
  File ~/code/ibis/ibis/backends/datafusion/__init__.py:244 in to_pyarrow_batches
    frame = self._get_frame(expr, params, limit, **kwargs)
  File ~/code/ibis/ibis/backends/datafusion/__init__.py:213 in _get_frame
    return self.compile(expr, params, **kwargs)
  File ~/code/ibis/ibis/backends/datafusion/__init__.py:272 in compile
    return translate(expr.op())
  File ~/mambaforge/envs/ibisdev/lib/python3.11/functools.py:909 in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
  File ~/code/ibis/ibis/backends/datafusion/compiler.py:24 in operation
    raise com.OperationNotDefinedError(f'No translation rule for {type(op)}')
OperationNotDefinedError: No translation rule for <class 'ibis.backends.pandas.client.PandasInMemoryTable'>
```

</details>


### What version of ibis are you using?

ibis master

### What backend(s) are you using, if any?

pandas
dask

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Yeah, the pandas backend doesn't support `memtable` currently. I agree the error message could be better here.

Do you have a particular need for the pandas backend, or are you more exploring the API?
No, I was just trying to inspect how the default backend config impacts some of the ibis top level function to document it #5466 and I would have expected that pandas would naturally work as a memtable backend. Raising an informative exception would do.
/take

Resolving patch:
diff --git a/ibis/backends/pandas/execution/generic.py b/ibis/backends/pandas/execution/generic.py
--- a/ibis/backends/pandas/execution/generic.py
+++ b/ibis/backends/pandas/execution/generic.py
@@ -1442,6 +1442,11 @@ def execute_zero_if_null_series(op, data, **kwargs):
     return data.replace({np.nan: zero, None: zero, pd.NA: zero})
 
 
+@execute_node.register(ops.InMemoryTable)
+def execute_in_memory_table(op, **kwargs):
+    return op.data.to_frame()
+
+
 @execute_node.register(
     ops.ZeroIfNull,
     (type(None), type(pd.NA), numbers.Real, np.integer, np.floating),


Test patch:
diff --git a/ibis/backends/tests/test_array.py b/ibis/backends/tests/test_array.py
--- a/ibis/backends/tests/test_array.py
+++ b/ibis/backends/tests/test_array.py
@@ -473,11 +473,8 @@ def test_array_slice(con, start, stop):
 )
 @pytest.mark.notimpl(
     ["dask", "pandas"],
-    raises=com.UnboundExpressionError,
-    reason=(
-        "Node of type 'InMemoryTable' has no data bound to it. "
-        "You probably tried to execute an expression without a data source."
-    ),
+    raises=com.OperationNotDefinedError,
+    reason="Operation 'ArrayMap' is not implemented for this backend",
 )
 @pytest.mark.notimpl(
     ["sqlite", "mysql"],
@@ -508,11 +505,8 @@ def test_array_map(backend, con):
 )
 @pytest.mark.notimpl(
     ["dask", "pandas"],
-    raises=com.UnboundExpressionError,
-    reason=(
-        "Node of type 'InMemoryTable' has no data bound to it. "
-        "You probably tried to execute an expression without a data source."
-    ),
+    raises=com.OperationNotDefinedError,
+    reason="Operation 'ArrayMap' is not implemented for this backend'",
 )
 @pytest.mark.notimpl(
     ["sqlite", "mysql"],
diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py
--- a/ibis/backends/tests/test_client.py
+++ b/ibis/backends/tests/test_client.py
@@ -719,13 +719,13 @@ def test_deprecated_path_argument(backend_name, tmp_path):
         ),
     ],
 )
-@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
+@pytest.mark.notimpl(["datafusion"])
 def test_in_memory_table(backend, con, expr, expected):
     result = con.execute(expr)
     backend.assert_frame_equal(result, expected)
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
+@pytest.mark.notimpl(["datafusion"])
 def test_filter_memory_table(backend, con):
     t = ibis.memtable([(1, 2), (3, 4), (5, 6)], columns=["x", "y"])
     expr = t.filter(t.x > 1)
@@ -734,7 +734,7 @@ def test_filter_memory_table(backend, con):
     backend.assert_frame_equal(result, expected)
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
+@pytest.mark.notimpl(["datafusion"])
 def test_agg_memory_table(con):
     t = ibis.memtable([(1, 2), (3, 4), (5, 6)], columns=["x", "y"])
     expr = t.x.count()
diff --git a/ibis/backends/tests/test_export.py b/ibis/backends/tests/test_export.py
--- a/ibis/backends/tests/test_export.py
+++ b/ibis/backends/tests/test_export.py
@@ -151,7 +151,7 @@ def test_to_pyarrow_batches_borked_types(batting):
     assert len(batch) == 42
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "impala", "pandas", "pyspark"])
+@pytest.mark.notimpl(["dask", "datafusion", "impala", "pyspark"])
 def test_to_pyarrow_memtable(con):
     expr = ibis.memtable({"x": [1, 2, 3]})
     table = con.to_pyarrow(expr)
@@ -159,7 +159,7 @@ def test_to_pyarrow_memtable(con):
     assert len(table) == 3
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "impala", "pandas", "pyspark"])
+@pytest.mark.notimpl(["dask", "datafusion", "impala", "pyspark"])
 def test_to_pyarrow_batches_memtable(con):
     expr = ibis.memtable({"x": [1, 2, 3]})
     n = 0
diff --git a/ibis/backends/tests/test_generic.py b/ibis/backends/tests/test_generic.py
--- a/ibis/backends/tests/test_generic.py
+++ b/ibis/backends/tests/test_generic.py
@@ -916,7 +916,6 @@ def test_literal_na(con, dtype):
 
 
 @pytest.mark.notimpl(["datafusion"], raises=com.OperationNotDefinedError)
-@pytest.mark.notimpl(["dask", "pandas"], raises=com.UnboundExpressionError)
 def test_memtable_bool_column(backend, con, monkeypatch):
     monkeypatch.setattr(ibis.options, "default_backend", con)
 
@@ -925,7 +924,6 @@ def test_memtable_bool_column(backend, con, monkeypatch):
 
 
 @pytest.mark.notimpl(["datafusion"], raises=com.OperationNotDefinedError)
-@pytest.mark.notimpl(["dask", "pandas"], raises=com.UnboundExpressionError)
 @pytest.mark.broken(
     ["druid"],
     raises=AssertionError,
diff --git a/ibis/backends/tests/test_join.py b/ibis/backends/tests/test_join.py
--- a/ibis/backends/tests/test_join.py
+++ b/ibis/backends/tests/test_join.py
@@ -199,7 +199,7 @@ def test_semi_join_topk(batting, awards_players):
     assert not expr.limit(5).execute().empty
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "druid", "pandas"])
+@pytest.mark.notimpl(["dask", "datafusion", "druid"])
 @pytest.mark.broken(
     ["duckdb"],
     raises=exc.IbisTypeError,
@@ -214,7 +214,7 @@ def test_join_with_pandas(batting, awards_players):
     assert df.yearID.nunique() == 7
 
 
-@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
+@pytest.mark.notimpl(["dask", "datafusion"])
 def test_join_with_pandas_non_null_typed_columns(batting, awards_players):
     batting_filt = batting[lambda t: t.yearID < 1900][["yearID"]]
     awards_players_filt = awards_players[lambda t: t.yearID < 1900][