Repository: ibis-project/ibis

Issue:
bug: Numpy-generated random choices appear as NULLs
### What happened?

A Pandas data frame created by using `numpy.random.choice()`, when registered as a table in DuckDB backend, appears as all NULLs.

```python
import ibis
import numpy as np
import pandas as pd
import random

data = pd.DataFrame(
    {
        # works OK
        # "np_rand": np.random.choice(["foo", "bar"], p=[0.7, 0.3], size=10),
        # "core_rand": random.choices(["foo", "bar"], weights=[0.7, 0.3], k=10),

        "np_rand": [np.random.choice(["foo", "bar"], p=[0.7, 0.3]) for _ in range(5)],  # will be None in Ibis
        "core_rand": [random.choices(["foo", "bar"], weights=[0.7, 0.3])[0] for _ in range(5)],  # OK in Ibis
    }
)

if __name__ == "__main__":
    print("----- DF data -----")
    print(data)

    print()
    print("Numpy-generated column ends up as NULLs.")
    print("----- Ibis table data -----")
    conn = ibis.duckdb.connect()
    conn.register(data, table_name="data")
    t = conn.table("data")
    print(t.execute())

```

### What version of ibis are you using?

5.0.0

### What backend(s) are you using, if any?

DuckDB

### Relevant log output

```sh
----- DF data -----
  np_rand core_rand
0     foo       foo
1     foo       foo
2     bar       bar
3     foo       foo
4     foo       foo

Numpy-generated column ends up as NULLs.
----- Ibis table data -----
  np_rand core_rand
0    None       foo
1    None       foo
2    None       bar
3    None       foo
4    None       foo
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
This looks like a bug in DuckDB:

```
In [18]: import duckdb, numpy as np, pandas as pd

In [19]: df = pd.DataFrame({"x": list(map(np.str_, range(3))), "y": list(range(3))})

In [20]: con.execute("select * from df").fetchall()
Out[20]: [(None, 0), (None, 1), (None, 2)]
```
@dimitri-b Thanks for the report. Until the next release of duckdb you'll need to make sure that numpy arrays going into pandas have `object` dtype and not `str_` or any of the other non-object numpy string-like types.
I will see if there's a way we can workaround this in the meantime.

Resolving patch:
diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py
--- a/ibis/backends/duckdb/__init__.py
+++ b/ibis/backends/duckdb/__init__.py
@@ -437,9 +437,36 @@ def read_in_memory(
         ir.Table
             The just-registered table
         """
+
+        def _clean_up_string_columns(
+            t: pa.Table | pd.DataFrame,
+        ) -> pa.Table | pd.DataFrame:
+            """Work around an upstream duckdb type inference bug.
+
+            https://github.com/ibis-project/ibis/pull/5880
+
+            DuckDB doesn't infer the type of `numpy.str_` objects and considers
+            them NULL when scanning a pandas DataFrame.
+            """
+            import pandas as pd
+
+            if not isinstance(t, pd.DataFrame):
+                return t
+
+            from ibis.backends.pandas.client import sch
+
+            schema = sch.infer(t)
+            return t.assign(
+                **{
+                    name: col.map(str, na_action="ignore")
+                    for name, col in t.items()
+                    if schema[name].is_string()
+                }
+            )
+
         table_name = table_name or f"ibis_read_in_memory_{next(pd_n)}"
         with self.begin() as con:
-            con.connection.register(table_name, dataframe)
+            con.connection.register(table_name, _clean_up_string_columns(dataframe))
 
         return self.table(table_name)
 
@@ -741,8 +768,7 @@ def _register_in_memory_table(self, op: ops.InMemoryTable) -> None:
 
         name = op.name
         table = op.data.to_pyarrow(schema)
-        with self.begin() as con:
-            con.connection.register(name, table)
+        self.read_in_memory(table, table_name=name)
 
     def _get_sqla_table(
         self, name: str, schema: str | None = None, **kwargs: Any


Test patch:
diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py
--- a/ibis/backends/duckdb/tests/test_register.py
+++ b/ibis/backends/duckdb/tests/test_register.py
@@ -3,7 +3,9 @@
 import tempfile
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
+import pandas.testing as tm
 import pyarrow as pa
 import pytest
 
@@ -213,3 +215,9 @@ def test_s3_403_fallback(con, httpserver, monkeypatch):
     # pyarrow, which indicates the fallback worked
     with pytest.raises(pa.lib.ArrowInvalid):
         con.read_parquet(httpserver.url_for("/myfile"))
+
+
+def test_register_numpy_str(con):
+    data = pd.DataFrame({"a": [np.str_("xyz"), None]})
+    result = con.read_in_memory(data)
+    tm.assert_frame_equal(result.execute(), data)