Repository: ibis-project/ibis

Issue:
bug: inserting a dataframe into a duckdb table is very slow
### What happened?

Inserting dataframes into a duckdb table has a weird behavior: for large dataframes, the RAM usage increases greatly until OOM errors. For small dataframes (like in the example below) the writing throughput is close to 5 krows/s whereas using duckdb directly leads to a throughput of 5,000 krows/s.

The ibis script, we only insert dataframe with 1k rows because 100k rows took forever:
```python
import os
import ibis
import pandas as pd
import numpy as np
import psutil
from time import perf_counter

n_rows = int(1e3)
df = pd.DataFrame({
    "a": np.arange(n_rows),
    "b": np.arange(n_rows),
    "c": np.arange(n_rows),
})

print(f"df.memory_usage: {df.memory_usage().sum() / 1e6:.3f} MB")

con = ibis.duckdb.connect("duckdb.db")
table_name = "t"

p = psutil.Process()
print(f"mem_usage: {p.memory_info().rss/1e9:.3f} GB")

chunk = ibis.memtable(df)
for idx in range(100):
    inserted_rows = (idx+1) * df.shape[0]
    tic = perf_counter()
    con.insert(table_name, chunk)
    toc = perf_counter()
    wal_size = os.stat("duckdb.db.wal").st_size
    db_size = os.stat("duckdb.db").st_size
    print(
        f"inserted_rows: {inserted_rows}, mem_usage: {p.memory_info().rss/1e9:.3f} GB, "
        f"wal_size: {wal_size/1e6:.3f} MB, db_size: {db_size/1e6:.3f} MB, throughput: {n_rows / (toc - tic) / 1e3:.1f} krows/s"
    )
```

The duckdb script, we insert dataframe with 100k rows:
```python
import os
import ibis
import pandas as pd
import numpy as np
import psutil
from time import perf_counter
import duckdb

n_rows = int(1e5)
df = pd.DataFrame({
    "a": np.arange(n_rows),
    "b": np.arange(n_rows),
    "c": np.arange(n_rows),
})

print(f"df.memory_usage: {df.memory_usage().sum() / 1e6:.3f} MB")

con = duckdb.connect("direct_duckdb.db")
con.sql("CREATE TABLE t AS SELECT * FROM df")

p = psutil.Process()
print(f"mem_usage: {p.memory_info().rss/1e9:.3f} GB", flush=True)

chunk = ibis.memtable(df)
for idx in range(100):
    inserted_rows = (idx+1) * df.shape[0]
    tic = perf_counter()
    con.sql("INSERT INTO t SELECT * FROM df")
    toc = perf_counter()
    wal_size = os.stat("direct_duckdb.db.wal").st_size
    db_size = os.stat("direct_duckdb.db").st_size
    print(
        f"inserted_rows: {inserted_rows}, mem_usage: {p.memory_info().rss/1e9:.3f} GB, "
        f"wal_size: {wal_size/1e6:.3f} MB, db_size: {db_size/1e6:.3f} MB, throughput: {n_rows / (toc - tic) / 1e3:.1f} krows/s"
    )
```

cc @ogrisel that led this benchmark.

### What version of ibis are you using?

4.1.0

### What backend(s) are you using, if any?

duckdb version: 0.7.0

### Relevant log output

```sh
// with ibis
inserted_rows: 1000, mem_usage: 0.277 GB, wal_size: 2.485 MB, db_size: 2.109 MB, throughput: 5.3 krows/s
inserted_rows: 2000, mem_usage: 0.277 GB, wal_size: 2.509 MB, db_size: 2.109 MB, throughput: 6.2 krows/s
inserted_rows: 3000, mem_usage: 0.277 GB, wal_size: 2.533 MB, db_size: 2.109 MB, throughput: 5.1 krows/s

// with duckdb
inserted_rows: 100000, mem_usage: 0.203 GB, wal_size: 4.807 MB, db_size: 0.012 MB, throughput: 3567.0 krows/s
inserted_rows: 200000, mem_usage: 0.206 GB, wal_size: 7.210 MB, db_size: 0.012 MB, throughput: 5329.9 krows/s
inserted_rows: 300000, mem_usage: 0.209 GB, wal_size: 9.614 MB, db_size: 0.012 MB, throughput: 5453.4 krows/s
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:

Resolving patch:
diff --git a/ibis/backends/base/sql/alchemy/__init__.py b/ibis/backends/base/sql/alchemy/__init__.py
--- a/ibis/backends/base/sql/alchemy/__init__.py
+++ b/ibis/backends/base/sql/alchemy/__init__.py
@@ -11,6 +11,7 @@
 import sqlalchemy as sa
 
 import ibis
+import ibis.common.exceptions as exc
 import ibis.expr.operations as ops
 import ibis.expr.schema as sch
 import ibis.expr.types as ir
@@ -486,6 +487,19 @@ def table(
         sqla_table = self._get_sqla_table(name, database=database, schema=schema)
         return self._sqla_table_to_expr(sqla_table)
 
+    def _insert_dataframe(
+        self, table_name: str, df: pd.DataFrame, overwrite: bool
+    ) -> None:
+        if not self.inspector.has_table(table_name):
+            raise exc.IbisError(f"Cannot insert into non-existent table {table_name}")
+        df.to_sql(
+            table_name,
+            self.con,
+            index=False,
+            if_exists='replace' if overwrite else 'append',
+            schema=self._current_schema,
+        )
+
     def insert(
         self,
         table_name: str,
@@ -533,13 +547,7 @@ def insert(
             obj = in_mem_table.data.to_frame()
 
         if isinstance(obj, pd.DataFrame):
-            obj.to_sql(
-                table_name,
-                self.con,
-                index=False,
-                if_exists='replace' if overwrite else 'append',
-                schema=self._current_schema,
-            )
+            self._insert_dataframe(table_name, obj, overwrite=overwrite)
         elif isinstance(obj, ir.Table):
             to_table_expr = self.table(table_name)
             to_table_schema = to_table_expr.schema()
diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py
--- a/ibis/backends/duckdb/__init__.py
+++ b/ibis/backends/duckdb/__init__.py
@@ -704,3 +704,22 @@ def drop_view(
 
         with self.begin() as con:
             con.execute(view)
+
+    def _insert_dataframe(
+        self, table_name: str, df: pd.DataFrame, overwrite: bool
+    ) -> None:
+        columns = list(df.columns)
+        t = sa.table(table_name, *map(sa.column, columns))
+
+        quote = self.con.dialect.identifier_preparer.quote
+        table_name = quote(table_name)
+
+        # the table name df here matters, and *must* match the input variable's
+        # name because duckdb will look up this name in the outer scope of the
+        # insert call and pull in that variable's data to scan
+        source = sa.table("df", *map(sa.column, columns))
+
+        with self.begin() as con:
+            if overwrite:
+                con.execute(t.delete())
+            con.execute(t.insert().from_select(columns, sa.select(source)))


Test patch:
diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py
--- a/ibis/backends/tests/test_client.py
+++ b/ibis/backends/tests/test_client.py
@@ -399,8 +399,10 @@ def _emp(a, b, c, d):
 def test_insert_from_memtable(alchemy_con):
     df = pd.DataFrame({"x": range(3)})
     table_name = "memtable_test"
-    alchemy_con.insert(table_name, ibis.memtable(df))
-    alchemy_con.insert(table_name, ibis.memtable(df))
+    mt = ibis.memtable(df)
+    alchemy_con.create_table(table_name, schema=mt.schema())
+    alchemy_con.insert(table_name, mt)
+    alchemy_con.insert(table_name, mt)
 
     try:
         table = alchemy_con.tables[table_name]
diff --git a/ibis/tests/benchmarks/test_benchmarks.py b/ibis/tests/benchmarks/test_benchmarks.py
--- a/ibis/tests/benchmarks/test_benchmarks.py
+++ b/ibis/tests/benchmarks/test_benchmarks.py
@@ -3,6 +3,8 @@
 import inspect
 import itertools
 import string
+import tempfile
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -706,3 +708,19 @@ def test_repr_join(benchmark, customers, orders, orders_items, products):
     )
     op = expr.op()
     benchmark(repr, op)
+
+
+@pytest.mark.parametrize("overwrite", [True, False], ids=["overwrite", "no_overwrite"])
+def test_insert_duckdb(benchmark, overwrite):
+    pytest.importorskip("duckdb")
+    pytest.importorskip("duckdb_engine")
+
+    n_rows = int(1e4)
+    table_name = "t"
+    schema = ibis.schema(dict(a="int64", b="int64", c="int64"))
+    t = ibis.memtable(dict.fromkeys(list("abc"), range(n_rows)), schema=schema)
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as d:
+        con = ibis.duckdb.connect(Path(d, "test_insert.ddb"))
+        con.create_table(table_name, schema=schema)
+        benchmark(con.insert, table_name, t, overwrite=overwrite)