Repository: narwhals-dev/narwhals

Issue:
[Bug]: Series.to_dummies raises for PyArrow when nulls are present
### Describe the bug

Calling `to_dummies` on a Series with nulls present raises an error

### Steps or code to reproduce the bug

```python
# ruff: noqa

# todo:
# - Series.rename, as alias for Series.alias. it's OK

import pyarrow as pa
import narwhals as nw

df = nw.from_native(pa.table({'a': ['x', 'y', None]}), eager_only=True)
print(df['a'].to_dummies().to_native())
```

### Expected results

```
pyarrow.Table
a_null: uint8
a_x: uint8
a_y: uint8
----
a_null: [[0,0,1]]
a_x: [[1,0,0]]
a_y: [[0,1,0]]
```

### Actual results

```
Traceback (most recent call last):
  File "/home/marcogorelli/polars-api-compat-dev/t.py", line 11, in <module>
    print(df['a'].to_dummies().to_native())
          ^^^^^^^^^^^^^^^^^^^^
  File "/home/marcogorelli/polars-api-compat-dev/narwhals/series.py", line 2289, in to_dummies
    self._compliant_series.to_dummies(separator=separator, drop_first=drop_first),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/marcogorelli/polars-api-compat-dev/narwhals/_arrow/series.py", line 661, in to_dummies
    columns[da.indices, np.arange(len(da))] = 1
    ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid in
dices
```

### Please run narwhals.show_version() and enter the output below.

```shell
1.8.2
```


### Relevant log output

_No response_

Discussion:
MarcoGorelli: > we still get
> 
> ```python
> In [15]: nw.from_native(pd.Series([1,2,3, None]), series_only=True).to_dummies().to_native()
> Out[15]:
>    _null  _1.0  _2.0  _3.0
> 0      0     1     0     0
> 1      0     0     1     0
> 2      0     0     0     1
> 3      1     0     0     0
> ```

sorry, that's right - the unique values in `pd.Series([1,2,3, None])` are indeed 1.0, 2.0, 3.0

Patch:
diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py
index 85ac3ec1..d0c2bf98 100644
--- a/narwhals/_arrow/series.py
+++ b/narwhals/_arrow/series.py
@@ -752,17 +752,32 @@ class ArrowSeries:
         from narwhals._arrow.dataframe import ArrowDataFrame
 
         series = self._native_series
-        da = series.dictionary_encode().combine_chunks()
+        name = self._name
+        da = series.dictionary_encode(null_encoding="encode").combine_chunks()
 
-        columns = np.zeros((len(da.dictionary), len(da)), np.uint8)
+        columns = np.zeros((len(da.dictionary), len(da)), np.int8)
         columns[da.indices, np.arange(len(da))] = 1
-        names = [f"{self._name}{separator}{v}" for v in da.dictionary]
+        null_col_pa, null_col_pl = f"{name}{separator}None", f"{name}{separator}null"
+        cols = [
+            {null_col_pa: null_col_pl}.get(
+                f"{name}{separator}{v}", f"{name}{separator}{v}"
+            )
+            for v in da.dictionary
+        ]
 
+        output_order = (
+            [
+                null_col_pl,
+                *sorted([c for c in cols if c != null_col_pl])[int(drop_first) :],
+            ]
+            if null_col_pl in cols
+            else sorted(cols)[int(drop_first) :]
+        )
         return ArrowDataFrame(
-            pa.Table.from_arrays(columns, names=names),
+            pa.Table.from_arrays(columns, names=cols),
             backend_version=self._backend_version,
             dtypes=self._dtypes,
-        ).select(*sorted(names)[int(drop_first) :])
+        ).select(*output_order)
 
     def quantile(
         self: Self,
diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py
index 4abc7cce..1bee0079 100644
--- a/narwhals/_pandas_like/series.py
+++ b/narwhals/_pandas_like/series.py
@@ -14,6 +14,7 @@ from narwhals._pandas_like.utils import int_dtype_mapper
 from narwhals._pandas_like.utils import narwhals_to_native_dtype
 from narwhals._pandas_like.utils import native_series_from_iterable
 from narwhals._pandas_like.utils import native_to_narwhals_dtype
+from narwhals._pandas_like.utils import select_columns_by_name
 from narwhals._pandas_like.utils import set_axis
 from narwhals._pandas_like.utils import to_datetime
 from narwhals._pandas_like.utils import validate_column_comparand
@@ -707,13 +708,28 @@ class PandasLikeSeries:
         plx = self.__native_namespace__()
         series = self._native_series
         name = str(self._name) if self._name else ""
+
+        null_col_pl = f"{name}{separator}null"
+
+        has_nulls = series.isna().any()
+        result = plx.get_dummies(
+            series,
+            prefix=name,
+            prefix_sep=separator,
+            drop_first=drop_first,
+            # Adds a null column at the end, depending on whether or not there are any.
+            dummy_na=has_nulls,
+            dtype="int8",
+        )
+        if has_nulls:
+            *cols, null_col_pd = list(result.columns)
+            output_order = [null_col_pd, *cols]
+            result = select_columns_by_name(
+                result, output_order, self._backend_version, self._implementation
+            ).rename(columns={null_col_pd: null_col_pl}, copy=False)
+
         return PandasLikeDataFrame(
-            plx.get_dummies(
-                series,
-                prefix=name,
-                prefix_sep=separator,
-                drop_first=drop_first,
-            ).astype(int),
+            result,
             implementation=self._implementation,
             backend_version=self._backend_version,
             dtypes=self._dtypes,
diff --git a/narwhals/_polars/series.py b/narwhals/_polars/series.py
index 3bc684d2..02b75520 100644
--- a/narwhals/_polars/series.py
+++ b/narwhals/_polars/series.py
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
     from types import ModuleType
 
     import numpy as np
+    import polars as pl
     from typing_extensions import Self
 
     from narwhals._polars.dataframe import PolarsDataFrame
@@ -27,7 +28,7 @@ class PolarsSeries:
     def __init__(
         self, series: Any, *, backend_version: tuple[int, ...], dtypes: DTypes
     ) -> None:
-        self._native_series = series
+        self._native_series: pl.Series = series
         self._backend_version = backend_version
         self._implementation = Implementation.POLARS
         self._dtypes = dtypes
@@ -85,7 +86,7 @@ class PolarsSeries:
 
     @property
     def name(self) -> str:
-        return self._native_series.name  # type: ignore[no-any-return]
+        return self._native_series.name
 
     @property
     def dtype(self: Self) -> DType:
@@ -103,7 +104,7 @@ class PolarsSeries:
     def cast(self, dtype: DType) -> Self:
         ser = self._native_series
         dtype = narwhals_to_native_dtype(dtype, self._dtypes)
-        return self._from_native_series(ser.cast(dtype))
+        return self._from_native_series(ser.cast(dtype))  # type: ignore[arg-type]
 
     def replace_strict(
         self, old: Sequence[Any], new: Sequence[Any], *, return_dtype: DType | None
@@ -203,16 +204,23 @@ class PolarsSeries:
     def to_dummies(
         self: Self, *, separator: str = "_", drop_first: bool = False
     ) -> PolarsDataFrame:
+        import polars as pl  # ignore-banned-import
+
         from narwhals._polars.dataframe import PolarsDataFrame
 
         if self._backend_version < (0, 20, 15):
+            has_nulls = self._native_series.is_null().any()
             result = self._native_series.to_dummies(separator=separator)
-            result = result.select(result.columns[int(drop_first) :])
+            output_columns = result.columns
+            if drop_first:
+                _ = output_columns.pop(int(has_nulls))
+
+            result = result.select(output_columns)
         else:
             result = self._native_series.to_dummies(
                 separator=separator, drop_first=drop_first
             )
-
+        result = result.with_columns(pl.all().cast(pl.Int8))
         return PolarsDataFrame(
             result, backend_version=self._backend_version, dtypes=self._dtypes
         )
diff --git a/narwhals/series.py b/narwhals/series.py
index 93295a84..0b5dfcc1 100644
--- a/narwhals/series.py
+++ b/narwhals/series.py
@@ -2550,7 +2550,7 @@ class Series:
             ┌─────┬─────┬─────┐
             │ a_1 ┆ a_2 ┆ a_3 │
             │ --- ┆ --- ┆ --- │
-            │ u8  ┆ u8  ┆ u8  │
+            │ i8  ┆ i8  ┆ i8  │
             ╞═════╪═════╪═════╡
             │ 1   ┆ 0   ┆ 0   │
             │ 0   ┆ 1   ┆ 0   │
@@ -2561,7 +2561,7 @@ class Series:
             ┌─────┬─────┐
             │ a_2 ┆ a_3 │
             │ --- ┆ --- │
-            │ u8  ┆ u8  │
+            │ i8  ┆ i8  │
             ╞═════╪═════╡
             │ 0   ┆ 0   │
             │ 1   ┆ 0   │


Test patch:
diff --git a/tests/series_only/to_dummy_test.py b/tests/series_only/to_dummy_test.py
index 10d6e971..ce25dd65 100644
--- a/tests/series_only/to_dummy_test.py
+++ b/tests/series_only/to_dummy_test.py
@@ -6,14 +6,15 @@ import narwhals.stable.v1 as nw
 from tests.utils import ConstructorEager
 from tests.utils import assert_equal_data
 
-data = [1, 2, 3]
+data = ["x", "y", "z"]
+data_na = ["x", "y", None]
 
 
 @pytest.mark.parametrize("sep", ["_", "-"])
 def test_to_dummies(constructor_eager: ConstructorEager, sep: str) -> None:
     s = nw.from_native(constructor_eager({"a": data}), eager_only=True)["a"].alias("a")
     result = s.to_dummies(separator=sep)
-    expected = {f"a{sep}1": [1, 0, 0], f"a{sep}2": [0, 1, 0], f"a{sep}3": [0, 0, 1]}
+    expected = {f"a{sep}x": [1, 0, 0], f"a{sep}y": [0, 1, 0], f"a{sep}z": [0, 0, 1]}
 
     assert_equal_data(result, expected)
 
@@ -22,6 +23,31 @@ def test_to_dummies(constructor_eager: ConstructorEager, sep: str) -> None:
 def test_to_dummies_drop_first(constructor_eager: ConstructorEager, sep: str) -> None:
     s = nw.from_native(constructor_eager({"a": data}), eager_only=True)["a"].alias("a")
     result = s.to_dummies(drop_first=True, separator=sep)
-    expected = {f"a{sep}2": [0, 1, 0], f"a{sep}3": [0, 0, 1]}
+    expected = {f"a{sep}y": [0, 1, 0], f"a{sep}z": [0, 0, 1]}
 
     assert_equal_data(result, expected)
+
+
+@pytest.mark.parametrize("sep", ["_", "-"])
+def test_to_dummies_with_nulls(constructor_eager: ConstructorEager, sep: str) -> None:
+    if "pandas_nullable_constructor" not in str(constructor_eager):
+        pytest.skip()
+    s = nw.from_native(constructor_eager({"a": data_na}), eager_only=True)["a"].alias("a")
+    result = s.to_dummies(separator=sep)
+    expected = {f"a{sep}null": [0, 0, 1], f"a{sep}x": [1, 0, 0], f"a{sep}y": [0, 1, 0]}
+
+    assert_equal_data(result, expected)
+
+
+@pytest.mark.parametrize("sep", ["_", "-"])
+def test_to_dummies_drop_first_na(
+    request: pytest.FixtureRequest, constructor_eager: ConstructorEager, sep: str
+) -> None:
+    if "cudf" in str(constructor_eager):
+        request.applymarker(pytest.mark.xfail)
+    s = nw.from_native(constructor_eager({"a": data_na}), eager_only=True)["a"].alias("a")
+    result = s.to_dummies(drop_first=True, separator=sep)
+    expected = {f"a{sep}null": [0, 0, 1], f"a{sep}y": [0, 1, 0]}
+
+    assert_equal_data(result, expected)
+    assert result.schema == {f"a{sep}null": nw.Int8, f"a{sep}y": nw.Int8}