Repository: ibis-project/ibis

Issue:
bug: polars backend chokes on Categorical
### What happened?

Expected `ibis.polars.read_parquet` to succeed with a categorical dtype in the parquet file. Instead, we see a `KeyError: Categorical` exception.

It looks like the reverse direction is fine, but I believe `ibis.backends.polars.datatypes.py` needs the `dt.Category: pl.Categorical,` pair added into `_to_polars_types`

### What version of ibis are you using?

4.0.0 (still exists on main)

### What backend(s) are you using, if any?

polars

### Relevant log output

```sh
KeyError                                  Traceback (most recent call last)
Cell In[3], line 1
----> 1 ibis.polars.read_parquet('data.parquet')

File ~/.conda/envs/modin/lib/python3.10/site-packages/ibis/backends/polars/__init__.py:211, in Backend.read_parquet(self, path, table_name, **kwargs)
    209 table_name = table_name or f"ibis_read_parquet_{next(pa_n)}"
    210 self._tables[table_name] = pl.scan_parquet(path, **kwargs)
--> 211 return self.table(table_name)

File ~/.conda/envs/modin/lib/python3.10/site-packages/ibis/backends/polars/__init__.py:61, in Backend.table(self, name, _schema)
     60 def table(self, name: str, _schema: sch.Schema = None) -> ir.Table:
---> 61     schema = sch.infer(self._tables[name])
     62     return self.table_class(name, schema, self).to_expr()

File ~/.conda/envs/modin/lib/python3.10/site-packages/multipledispatch/dispatcher.py:278, in Dispatcher.__call__(self, *args, **kwargs)
    276     self._cache[types] = func
    277 try:
--> 278     return func(*args, **kwargs)
    280 except MDNotImplementedError:
    281     funcs = self.dispatch_iter(*types)

File ~/.conda/envs/modin/lib/python3.10/site-packages/ibis/backends/polars/datatypes.py:104, in from_polars_schema(df)
    102 @sch.infer.register(pl.LazyFrame)
    103 def from_polars_schema(df: pl.LazyFrame) -> sch.Schema:
...
     75     """Convert polars dtype to the ibis counterpart."""
---> 76     klass = _to_ibis_dtypes[typ]
     77     return klass()

KeyError: Categorical
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
This should probably map to ibis's `String` type, unless polars supports categorical types that aren't strings in which case we should choose the appropriate type.

Ibis's Categorical is slated for removal. It doesn't provide much value and just takes up space in the codebase.
@dakoop Thanks for the report! Slated for v5.0.0.

Resolving patch:
diff --git a/ibis/backends/polars/datatypes.py b/ibis/backends/polars/datatypes.py
--- a/ibis/backends/polars/datatypes.py
+++ b/ibis/backends/polars/datatypes.py
@@ -10,7 +10,6 @@
 _to_polars_types = {
     dt.Boolean: pl.Boolean,
     dt.Null: pl.Null,
-    dt.Array: pl.List,
     dt.String: pl.Utf8,
     dt.Binary: pl.Binary,
     dt.Date: pl.Date,
@@ -28,6 +27,7 @@
 }
 
 _to_ibis_dtypes = {v: k for k, v in _to_polars_types.items()}
+_to_ibis_dtypes[pl.Categorical] = dt.String
 
 
 @functools.singledispatch
@@ -64,7 +64,7 @@ def from_ibis_struct(dtype):
 
 
 @to_polars_type.register(dt.Category)
-def from_ibis_category(dtype):
+def from_ibis_category(_):
     return pl.Categorical
 
 
@@ -82,8 +82,12 @@ def to_ibis_dtype(typ):
 
 @to_ibis_dtype.register(pl.Datetime)
 def from_polars_datetime(typ):
-    # TODO(kszucs): handle timezone?
-    return dt.Timestamp()
+    return dt.Timestamp(timezone=typ.tz)
+
+
+@to_ibis_dtype.register(pl.Duration)
+def from_polars_duration(typ):
+    return dt.Interval(unit=typ.tu)
 
 
 @to_ibis_dtype.register(pl.List)
@@ -98,10 +102,6 @@ def from_polars_struct(typ):
     )
 
 
-# Can't register here since polars datatypes are classes
-# @dt.dtype.register(pl.DataType)
-
-
 @sch.infer.register(pl.LazyFrame)
 def from_polars_schema(df: pl.LazyFrame) -> sch.Schema:
     fields = [(name, to_ibis_dtype(typ)) for name, typ in df.schema.items()]


Test patch:
diff --git a/ibis/backends/polars/tests/test_datatypes.py b/ibis/backends/polars/tests/test_datatypes.py
new file mode 100644
--- /dev/null
+++ b/ibis/backends/polars/tests/test_datatypes.py
@@ -0,0 +1,58 @@
+import polars as pl
+import pytest
+from pytest import param
+
+import ibis.expr.datatypes as dt
+from ibis.backends.polars.datatypes import to_ibis_dtype, to_polars_type
+
+
+@pytest.mark.parametrize(
+    ("ibis_dtype", "polars_type"),
+    [
+        param(dt.bool, pl.Boolean, id="bool"),
+        param(dt.null, pl.Null, id="null"),
+        param(dt.Array(dt.string), pl.List(pl.Utf8), id="array_string"),
+        param(dt.string, pl.Utf8, id="string"),
+        param(dt.binary, pl.Binary, id="binary"),
+        param(dt.date, pl.Date, id="date"),
+        param(dt.time, pl.Time, id="time"),
+        param(dt.int8, pl.Int8, id="int8"),
+        param(dt.int16, pl.Int16, id="int16"),
+        param(dt.int32, pl.Int32, id="int32"),
+        param(dt.int64, pl.Int64, id="int64"),
+        param(dt.uint8, pl.UInt8, id="uint8"),
+        param(dt.uint16, pl.UInt16, id="uint16"),
+        param(dt.uint32, pl.UInt32, id="uint32"),
+        param(dt.uint64, pl.UInt64, id="uint64"),
+        param(dt.float32, pl.Float32, id="float32"),
+        param(dt.float64, pl.Float64, id="float64"),
+        param(dt.timestamp, pl.Datetime("ns", time_zone=None), id="timestamp"),
+        param(
+            dt.Timestamp("UTC"), pl.Datetime("ns", time_zone="UTC"), id="timestamp_tz"
+        ),
+        param(dt.Interval(unit="ms"), pl.Duration("ms"), id="interval_ms"),
+        param(dt.Interval(unit="us"), pl.Duration("us"), id="interval_us"),
+        param(dt.Interval(unit="ns"), pl.Duration("ns"), id="interval_ns"),
+        param(
+            dt.Struct(
+                dict(a=dt.string, b=dt.Array(dt.Array(dt.Struct(dict(c=dt.float64)))))
+            ),
+            pl.Struct(
+                [
+                    pl.Field("a", pl.Utf8),
+                    pl.Field(
+                        "b", pl.List(pl.List(pl.Struct([pl.Field("c", pl.Float64)])))
+                    ),
+                ]
+            ),
+            id="struct",
+        ),
+    ],
+)
+def test_to_from_ibis_type(ibis_dtype, polars_type):
+    assert to_polars_type(ibis_dtype) == polars_type
+    assert to_ibis_dtype(polars_type) == ibis_dtype
+
+
+def test_categorical():
+    assert to_ibis_dtype(pl.Categorical()) == dt.string