Repository: ibis-project/ibis

Issue:
feat: automatically set `temp_directory` for `ibis.read` use cases
### Is your feature request related to a problem?

My feature request is not related to a problem.

### Describe the solution you'd like

I'd like to be able to use `ibis.read` without running out of RAM if possible.

### What version of ibis are you running?

`master`

### What backend(s) are you using, if any?

DuckDB.

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:

Resolving patch:
diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py
--- a/ibis/backends/duckdb/__init__.py
+++ b/ibis/backends/duckdb/__init__.py
@@ -159,6 +159,7 @@ def do_connect(
         database: str | Path = ":memory:",
         path: str | Path = None,
         read_only: bool = False,
+        temp_directory: Path | str | None = None,
         **config: Any,
     ) -> None:
         """Create an Ibis client connected to a DuckDB database.
@@ -169,6 +170,9 @@ def do_connect(
             Path to a duckdb database.
         read_only
             Whether the database is read-only.
+        temp_directory
+            Directory to use for spilling to disk. Only set by default for
+            in-memory connections.
         config
             DuckDB configuration parameters. See the [DuckDB configuration
             documentation](https://duckdb.org/docs/sql/configuration) for
@@ -187,6 +191,16 @@ def do_connect(
             database = path
         if not (in_memory := database == ":memory:"):
             database = Path(database).absolute()
+        else:
+            if temp_directory is None:
+                temp_directory = (
+                    Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+                    / "duckdb"
+                )
+
+        if temp_directory is not None:
+            config["temp_directory"] = str(temp_directory)
+
         super().do_connect(
             sa.create_engine(
                 f"duckdb:///{database}",


Test patch:
diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py
--- a/ibis/backends/duckdb/tests/test_register.py
+++ b/ibis/backends/duckdb/tests/test_register.py
@@ -3,6 +3,7 @@
 import gzip
 import os
 import re
+import tempfile
 from pathlib import Path
 
 import pytest
@@ -187,3 +188,28 @@ def test_read_invalid(data_directory, basename):
     msg = f"^Unrecognized file type or extension: {re.escape(str(path))}"
     with pytest.raises(ValueError, match=msg):
         ibis.read(path)
+
+
+def test_temp_directory(tmp_path):
+    query = "SELECT value FROM duckdb_settings() WHERE name = 'temp_directory'"
+
+    # 1. in-memory + no temp_directory specified
+    con = ibis.duckdb.connect()
+    [(value,)] = con.con.execute(query).fetchall()
+    assert value  # we don't care what the specific value is
+
+    temp_directory = Path(tempfile.gettempdir()) / "duckdb"
+
+    # 2. in-memory + temp_directory specified
+    con = ibis.duckdb.connect(temp_directory=temp_directory)
+    [(value,)] = con.con.execute(query).fetchall()
+    assert value == str(temp_directory)
+
+    # 3. on-disk + no temp_directory specified
+    # untested, duckdb sets the temp_directory to something implementation
+    # defined
+
+    # 4. on-disk + temp_directory specified
+    con = ibis.duckdb.connect(tmp_path / "test2.ddb", temp_directory=temp_directory)
+    [(value,)] = con.con.execute(query).fetchall()
+    assert value == str(temp_directory)