Repository: ibis-project/ibis

Issue:
bug: ProgrammingError / CatalogException when calling to_parquet on an ibis.memtable instance
### What happened?

Follow-up on the recently merged #5457:

```python
>>> import ibis
>>> expr = ibis.memtable({"column": [0, 1, 2, 3, 4]})
>>> expr.to_parquet("file.parquet")
Traceback (most recent call last):
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:1900 in _execute_context
    self.dialect.do_execute(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/default.py:736 in do_execute
    cursor.execute(statement, parameters)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/duckdb_engine/__init__.py:128 in execute
    self.__c.execute(statement, parameters)
CatalogException: Catalog Error: Table with name _ibis_memtable1 does not exist!
Did you mean "pg_tables"?
LINE 2: FROM _ibis_memtable1 AS t0
             ^

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  Cell In[7], line 1
    expr.to_parquet("file.parquet")
  File ~/code/ibis/ibis/expr/types/core.py:431 in to_parquet
    self._find_backend(use_default=True).to_parquet(
  File ~/code/ibis/ibis/backends/base/__init__.py:406 in to_parquet
    batch_reader = expr.to_pyarrow_batches(params=params, limit=limit)
  File ~/code/ibis/ibis/expr/types/core.py:363 in to_pyarrow_batches
    return self._find_backend(use_default=True).to_pyarrow_batches(
  File ~/code/ibis/ibis/backends/duckdb/__init__.py:508 in to_pyarrow_batches
    cursor = self.con.connect().execute(sql)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:1380 in execute
    return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/sql/elements.py:334 in _execute_on_connection
    return connection._execute_clauseelement(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:1572 in _execute_clauseelement
    ret = self._execute_context(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:1943 in _execute_context
    self._handle_dbapi_exception(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:2124 in _handle_dbapi_exception
    util.raise_(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/util/compat.py:211 in raise_
    raise exception
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/base.py:1900 in _execute_context
    self.dialect.do_execute(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/sqlalchemy/engine/default.py:736 in do_execute
    cursor.execute(statement, parameters)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/duckdb_engine/__init__.py:128 in execute
    self.__c.execute(statement, parameters)
ProgrammingError: (duckdb.CatalogException) Catalog Error: Table with name _ibis_memtable1 does not exist!
Did you mean "pg_tables"?
LINE 2: FROM _ibis_memtable1 AS t0
             ^
[SQL: SELECT t0."column" 
FROM _ibis_memtable1 AS t0]
(Background on this error at: https://sqlalche.me/e/14/f405)
```

### What version of ibis are you using?

ibis master

### What backend(s) are you using, if any?

DuckDB

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Note: I also tried with `ibis.set_backend("polars")` and `ibis.set_backend("sqlite")` and it works in both cases, so as the traceback hints, this is a DuckDB specific problem.
Thanks for the excellent reproducible example. I've diagnosed the issue, fix incoming.

Resolving patch:
diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py
--- a/ibis/backends/duckdb/__init__.py
+++ b/ibis/backends/duckdb/__init__.py
@@ -502,6 +502,7 @@ def to_pyarrow_batches(
 
         from ibis.backends.duckdb.pyarrow import IbisRecordBatchReader
 
+        self._register_in_memory_tables(expr)
         query_ast = self.compiler.to_ast_ensure_limit(expr, limit, params=params)
         sql = query_ast.compile()
 
@@ -519,6 +520,8 @@ def to_pyarrow(
         **_: Any,
     ) -> pa.Table:
         pa = self._import_pyarrow()
+
+        self._register_in_memory_tables(expr)
         query_ast = self.compiler.to_ast_ensure_limit(expr, limit, params=params)
         sql = query_ast.compile()
 


Test patch:
diff --git a/ibis/backends/tests/test_export.py b/ibis/backends/tests/test_export.py
--- a/ibis/backends/tests/test_export.py
+++ b/ibis/backends/tests/test_export.py
@@ -4,6 +4,8 @@
 import pytest
 from pytest import param
 
+import ibis
+
 pa = pytest.importorskip("pyarrow")
 
 limit = [
@@ -147,6 +149,24 @@ def test_to_pyarrow_batches_borked_types(batting):
     assert len(batch) == 42
 
 
+@pytest.mark.notimpl(["dask", "datafusion", "impala", "pandas", "pyspark"])
+def test_to_pyarrow_memtable(con):
+    expr = ibis.memtable({"x": [1, 2, 3]})
+    table = con.to_pyarrow(expr)
+    assert isinstance(table, pa.Table)
+    assert len(table) == 3
+
+
+@pytest.mark.notimpl(["dask", "datafusion", "impala", "pandas", "pyspark"])
+def test_to_pyarrow_batches_memtable(con):
+    expr = ibis.memtable({"x": [1, 2, 3]})
+    n = 0
+    for batch in con.to_pyarrow_batches(expr):
+        assert isinstance(batch, pa.RecordBatch)
+        n += len(batch)
+    assert n == 3
+
+
 def test_no_pyarrow_message(awards_players, monkeypatch):
     monkeypatch.setitem(sys.modules, "pyarrow", None)
     with pytest.raises(ModuleNotFoundError, match="requires `pyarrow` but"):