Repository: ibis-project/ibis

Issue:
bug(duckdb): re-registered view doesn't reflect changed schema
I initially register a `csv` file using `con.register`

```python
[ins] In [81]: movielens = con.register("/home/gil/data/movielens/ratings.csv", table_name="movielens")

[ins] In [82]: movielens
Out[82]: 
AlchemyTable: movielens
  column0 int32
  imdb_id int32
  rating  int32
```

This _looks_ correct but `imdb_id` is actually a string column and I haven't told DuckDB to sample enough rows to detect that.  But if I use `ALL_VARCHAR=True` to force all columns to load as strings, the returned table has the same schema as before:

```python
[ins] In [83]: movielens = con.register(
          ...:     "/home/gil/data/movielens/ratings.csv", table_name="movielens", ALL_VARCHAR=True
          ...: )

[ins] In [84]: movielens
Out[84]: 
AlchemyTable: movielens
  column0 int32
  imdb_id int32
  rating  int32
```

But if I register and change the underlying table name, the new schema is reflected:

```python
[ins] In [85]: movielens = con.register(
          ...:     "/home/gil/data/movielens/ratings.csv", table_name="_movielens", ALL_VARCHAR=True
          ...: )

[ins] In [86]: movielens
Out[86]: 
AlchemyTable: _movielens
  column0 string
  imdb_id string
  rating  string
```

The underlying call is `CREATE OR REPLACE VIEW` -- the view _is_ being replaced in `DuckDB`

```python
[ins] In [93]: con.con.execute("DESCRIBE movielens").fetchall()
Out[93]: 
[('column0', 'VARCHAR', 'YES', None, None, None),
 ('imdb_id', 'VARCHAR', 'YES', None, None, None),
 ('rating', 'VARCHAR', 'YES', None, None, None)]
```

but something in duckdb-engine or ibis seems to be caching the alchemy table schema.


Discussion:
IIRC having traced this problem before, this caching happens at the SQLAlchemy level in the `MetaData` object.

Resolving patch:
diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py
--- a/ibis/backends/duckdb/__init__.py
+++ b/ibis/backends/duckdb/__init__.py
@@ -222,7 +222,7 @@ def register(
                 cursor.cursor.c.register(table_name, dataset)
         elif isinstance(source, (str, Path)):
             sql, table_name = _generate_view_code(
-                source, table_name=table_name, **kwargs
+                str(source), table_name=table_name, **kwargs
             )
             self.con.execute(sql)
         else:
@@ -230,6 +230,21 @@ def register(
                 table_name = next(_gen_table_names)
             self.con.execute("register", (table_name, source))
 
+        _table = self.table(table_name)
+        with warnings.catch_warnings():
+            # don't fail or warn if duckdb-engine fails to discover types
+            # mostly (tinyint)
+            warnings.filterwarnings(
+                "ignore",
+                message="Did not recognize type",
+                category=sa.exc.SAWarning,
+            )
+            # We don't rely on index reflection, ignore this warning
+            warnings.filterwarnings(
+                "ignore",
+                message="duckdb-engine doesn't yet support reflection on indices",
+            )
+            self.inspector.reflect_table(_table.op().sqla_table, _table.columns)
         return self.table(table_name)
 
     def to_pyarrow_batches(


Test patch:
diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py
--- a/ibis/backends/duckdb/tests/test_register.py
+++ b/ibis/backends/duckdb/tests/test_register.py
@@ -1,4 +1,5 @@
 import contextlib
+import csv
 import gzip
 import os
 from pathlib import Path
@@ -141,3 +142,28 @@ def test_register_pyarrow_tables():
 def test_csv_register_kwargs(kwargs, expected_snippet):
     view_str, _ = _generate_view_code("bork.csv", **kwargs)
     assert expected_snippet in view_str
+
+
+def test_csv_reregister_schema(tmp_path):
+    con = ibis.duckdb.connect()
+
+    foo = tmp_path / "foo.csv"
+    with open(foo, "w", newline="") as csvfile:
+        foowriter = csv.writer(
+            csvfile,
+            delimiter=",",
+        )
+        foowriter.writerow(["cola", "colb", "colc"])
+        foowriter.writerow([0, 1, 2])
+        foowriter.writerow([1, 5, 6])
+        foowriter.writerow([2, 3.0, "bar"])
+
+    # For a full file scan, expect correct schema based on final row
+    foo_table = con.register(foo)
+    exp_schema = ibis.schema(dict(cola="int32", colb="float64", colc="string"))
+    assert foo_table.schema() == exp_schema
+
+    # If file scan is limited to first two rows, should be all int32
+    foo_table = con.register(foo, SAMPLE_SIZE=2)
+    exp_schema = ibis.schema(dict(cola="int32", colb="int32", colc="int32"))
+    assert foo_table.schema() == exp_schema