Repository: narwhals-dev/narwhals
Pull request #570 (2024-07-21 17:25:17)

feat: pyarrow series descriptive methods
## What type of PR is this? (check all applicable)

- [ ] 💾 Refactor
- [x] ✨ Feature
- [ ] 🐛 Bug Fix
- [ ] 🔧 Optimization
- [ ] 📝 Documentation
- [ ] ✅ Test
- [ ] 🐳 Other

## Related issues 

- Related issue #372 

## Checklist

- [x] Code follows style guide (ruff)
- [x] Tests added 
- [x] Documented the changes

## If you have comments or can explain your changes, please do so below.

Majority (all?) of series descriptive methods - performances could need some love?!


Patch:
diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py
index 73111bff..075aaea6 100644
--- a/narwhals/_arrow/expr.py
+++ b/narwhals/_arrow/expr.py
@@ -233,6 +233,18 @@ class ArrowExpr:
     def fill_null(self: Self, value: Any) -> Self:
         return reuse_series_implementation(self, "fill_null", value=value)
 
+    def is_duplicated(self: Self) -> Self:
+        return reuse_series_implementation(self, "is_duplicated")
+
+    def is_unique(self: Self) -> Self:
+        return reuse_series_implementation(self, "is_unique")
+
+    def is_first_distinct(self: Self) -> Self:
+        return reuse_series_implementation(self, "is_first_distinct")
+
+    def is_last_distinct(self: Self) -> Self:
+        return reuse_series_implementation(self, "is_last_distinct")
+
     @property
     def dt(self: Self) -> ArrowExprDateTimeNamespace:
         return ArrowExprDateTimeNamespace(self)
diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py
index 9c1aa7d3..f6828a30 100644
--- a/narwhals/_arrow/series.py
+++ b/narwhals/_arrow/series.py
@@ -12,12 +12,15 @@ from narwhals._arrow.utils import reverse_translate_dtype
 from narwhals._arrow.utils import translate_dtype
 from narwhals._arrow.utils import validate_column_comparand
 from narwhals.dependencies import get_numpy
+from narwhals.dependencies import get_pandas
 from narwhals.dependencies import get_pyarrow
 from narwhals.dependencies import get_pyarrow_compute
+from narwhals.utils import generate_unique_token
 
 if TYPE_CHECKING:
     from typing_extensions import Self
 
+    from narwhals._arrow.dataframe import ArrowDataFrame
     from narwhals._arrow.namespace import ArrowNamespace
     from narwhals.dtypes import DType
 
@@ -375,6 +378,68 @@ class ArrowSeries:
 
         return self._from_native_series(pc.fill_null(ser, pa.scalar(value, dtype)))
 
+    def to_frame(self: Self) -> ArrowDataFrame:
+        from narwhals._arrow.dataframe import ArrowDataFrame
+
+        pa = get_pyarrow()
+        df = pa.Table.from_arrays([self._native_series], names=[self.name])
+        return ArrowDataFrame(df, backend_version=self._backend_version)
+
+    def to_pandas(self: Self) -> Any:
+        pd = get_pandas()
+        return pd.Series(self._native_series, name=self.name)
+
+    def is_duplicated(self: Self) -> ArrowSeries:
+        return self.to_frame().is_duplicated().alias(self.name)
+
+    def is_unique(self: Self) -> ArrowSeries:
+        return self.to_frame().is_unique().alias(self.name)
+
+    def is_first_distinct(self: Self) -> Self:
+        np = get_numpy()
+        pa = get_pyarrow()
+        pc = get_pyarrow_compute()
+
+        row_number = pa.array(np.arange(len(self)))
+        col_token = generate_unique_token(n_bytes=8, columns=[self.name])
+        first_distinct_index = (
+            pa.Table.from_arrays([self._native_series], names=[self.name])
+            .append_column(col_token, row_number)
+            .group_by(self.name)
+            .aggregate([(col_token, "min")])
+            .column(f"{col_token}_min")
+        )
+
+        return self._from_native_series(pc.is_in(row_number, first_distinct_index))
+
+    def is_last_distinct(self: Self) -> Self:
+        np = get_numpy()
+        pa = get_pyarrow()
+        pc = get_pyarrow_compute()
+
+        row_number = pa.array(np.arange(len(self)))
+        col_token = generate_unique_token(n_bytes=8, columns=[self.name])
+        last_distinct_index = (
+            pa.Table.from_arrays([self._native_series], names=[self.name])
+            .append_column(col_token, row_number)
+            .group_by(self.name)
+            .aggregate([(col_token, "max")])
+            .column(f"{col_token}_max")
+        )
+
+        return self._from_native_series(pc.is_in(row_number, last_distinct_index))
+
+    def is_sorted(self: Self, *, descending: bool = False) -> bool:
+        if not isinstance(descending, bool):
+            msg = f"argument 'descending' should be boolean, found {type(descending)}"
+            raise TypeError(msg)
+        pc = get_pyarrow_compute()
+        ser = self._native_series
+        if descending:
+            return pc.all(pc.greater_equal(ser[:-1], ser[1:])).as_py()  # type: ignore[no-any-return]
+        else:
+            return pc.all(pc.less_equal(ser[:-1], ser[1:])).as_py()  # type: ignore[no-any-return]
+
     @property
     def shape(self) -> tuple[int]:
         return (len(self._native_series),)
diff --git a/utils/check_backend_completeness.py b/utils/check_backend_completeness.py
index dacd7ead..b83766bc 100644
--- a/utils/check_backend_completeness.py
+++ b/utils/check_backend_completeness.py
@@ -17,17 +17,10 @@ MISSING = [
     "Series.drop_nulls",
     "Series.from_iterable",
     "Series.is_between",
-    "Series.is_duplicated",
-    "Series.is_first_distinct",
-    "Series.is_last_distinct",
-    "Series.is_sorted",
-    "Series.is_unique",
     "Series.quantile",
     "Series.round",
     "Series.shift",
     "Series.sort",
-    "Series.to_frame",
-    "Series.to_pandas",
     "Series.unique",
     "Series.value_counts",
 ]


Test patch:
diff --git a/tests/expr_and_series/is_duplicated_test.py b/tests/expr_and_series/is_duplicated_test.py
index 55d5f482..f4f8e305 100644
--- a/tests/expr_and_series/is_duplicated_test.py
+++ b/tests/expr_and_series/is_duplicated_test.py
@@ -1,6 +1,6 @@
 from typing import Any
 
-import pytest
+import numpy as np
 
 import narwhals.stable.v1 as nw
 from tests.utils import compare_dicts
@@ -11,9 +11,7 @@ data = {
 }
 
 
-def test_is_duplicated(request: Any, constructor: Any) -> None:
-    if "pyarrow_table" in str(constructor):
-        request.applymarker(pytest.mark.xfail)
+def test_is_duplicated_expr(constructor: Any) -> None:
     df = nw.from_native(constructor(data), eager_only=True)
     result = df.select(nw.all().is_duplicated())
     expected = {
@@ -21,3 +19,10 @@ def test_is_duplicated(request: Any, constructor: Any) -> None:
         "b": [False, False, False],
     }
     compare_dicts(result, expected)
+
+
+def test_is_duplicated_series(constructor_series: Any) -> None:
+    series = nw.from_native(constructor_series(data["a"]), series_only=True)
+    result = series.is_duplicated()
+    expected = np.array([True, True, False])
+    assert (result.to_numpy() == expected).all()
diff --git a/tests/expr_and_series/is_first_distinct_test.py b/tests/expr_and_series/is_first_distinct_test.py
index 71e5aa1e..db9be47e 100644
--- a/tests/expr_and_series/is_first_distinct_test.py
+++ b/tests/expr_and_series/is_first_distinct_test.py
@@ -1,6 +1,6 @@
 from typing import Any
 
-import pytest
+import numpy as np
 
 import narwhals.stable.v1 as nw
 from tests.utils import compare_dicts
@@ -11,10 +11,7 @@ data = {
 }
 
 
-def test_is_first_distinct(request: Any, constructor: Any) -> None:
-    if "pyarrow_table" in str(constructor):
-        request.applymarker(pytest.mark.xfail)
-
+def test_is_first_distinct_expr(constructor: Any) -> None:
     df = nw.from_native(constructor(data), eager_only=True)
     result = df.select(nw.all().is_first_distinct())
     expected = {
@@ -22,3 +19,10 @@ def test_is_first_distinct(request: Any, constructor: Any) -> None:
         "b": [True, True, True, False, False],
     }
     compare_dicts(result, expected)
+
+
+def test_is_first_distinct_series(constructor_series: Any) -> None:
+    series = nw.from_native(constructor_series(data["a"]), series_only=True)
+    result = series.is_first_distinct()
+    expected = np.array([True, False, True, True, False])
+    assert (result.to_numpy() == expected).all()
diff --git a/tests/expr_and_series/is_last_distinct_test.py b/tests/expr_and_series/is_last_distinct_test.py
index 36b4b7bc..7abfefaf 100644
--- a/tests/expr_and_series/is_last_distinct_test.py
+++ b/tests/expr_and_series/is_last_distinct_test.py
@@ -1,6 +1,6 @@
 from typing import Any
 
-import pytest
+import numpy as np
 
 import narwhals.stable.v1 as nw
 from tests.utils import compare_dicts
@@ -11,9 +11,7 @@ data = {
 }
 
 
-def test_is_last_distinct(request: Any, constructor: Any) -> None:
-    if "pyarrow_table" in str(constructor):
-        request.applymarker(pytest.mark.xfail)
+def test_is_last_distinct_expr(constructor: Any) -> None:
     df = nw.from_native(constructor(data), eager_only=True)
     result = df.select(nw.all().is_last_distinct())
     expected = {
@@ -21,3 +19,10 @@ def test_is_last_distinct(request: Any, constructor: Any) -> None:
         "b": [False, False, True, True, True],
     }
     compare_dicts(result, expected)
+
+
+def test_is_last_distinct_series(constructor_series: Any) -> None:
+    series = nw.from_native(constructor_series(data["a"]), series_only=True)
+    result = series.is_last_distinct()
+    expected = np.array([False, True, False, True, True])
+    assert (result.to_numpy() == expected).all()
diff --git a/tests/expr_and_series/is_unique_test.py b/tests/expr_and_series/is_unique_test.py
index 1d547bae..584a53aa 100644
--- a/tests/expr_and_series/is_unique_test.py
+++ b/tests/expr_and_series/is_unique_test.py
@@ -1,6 +1,6 @@
 from typing import Any
 
-import pytest
+import numpy as np
 
 import narwhals.stable.v1 as nw
 from tests.utils import compare_dicts
@@ -11,9 +11,7 @@ data = {
 }
 
 
-def test_is_unique(request: Any, constructor: Any) -> None:
-    if "pyarrow_table" in str(constructor):
-        request.applymarker(pytest.mark.xfail)
+def test_is_unique_expr(constructor: Any) -> None:
     df = nw.from_native(constructor(data), eager_only=True)
     result = df.select(nw.all().is_unique())
     expected = {
@@ -21,3 +19,10 @@ def test_is_unique(request: Any, constructor: Any) -> None:
         "b": [True, True, True],
     }
     compare_dicts(result, expected)
+
+
+def test_is_unique_series(constructor_series: Any) -> None:
+    series = nw.from_native(constructor_series(data["a"]), series_only=True)
+    result = series.is_unique()
+    expected = np.array([False, False, True])
+    assert (result.to_numpy() == expected).all()
diff --git a/tests/series_only/is_sorted_test.py b/tests/series_only/is_sorted_test.py
new file mode 100644
index 00000000..7861796d
--- /dev/null
+++ b/tests/series_only/is_sorted_test.py
@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+import narwhals.stable.v1 as nw
+
+data = [1, 3, 2]
+data_dups = [4, 4, 6]
+data_sorted = [7.0, 8, 9]
+
+
+@pytest.mark.parametrize(
+    ("input_data", "descending", "expected"),
+    [(data, False, False), (data_sorted, False, True), (data_sorted, True, False)],
+)
+def test_is_sorted(
+    constructor_series: Any,
+    input_data: str,
+    descending: bool,  # noqa: FBT001
+    expected: bool,  # noqa: FBT001
+) -> None:
+    series = nw.from_native(constructor_series(input_data), series_only=True)
+    result = series.is_sorted(descending=descending)
+    assert result == expected
+
+
+def test_is_sorted_invalid(constructor_series: Any) -> None:
+    series = nw.from_native(constructor_series(data_sorted), series_only=True)
+
+    with pytest.raises(TypeError):
+        series.is_sorted(descending="invalid_type")  # type: ignore[arg-type]
diff --git a/tests/series_only/test_common.py b/tests/series_only/test_common.py
index 5f53c6ad..bb5996e9 100644
--- a/tests/series_only/test_common.py
+++ b/tests/series_only/test_common.py
@@ -93,11 +93,11 @@ def test_boolean_reductions(request: Any, constructor: Any) -> None:
 def test_convert(request: Any, constructor_series: Any) -> None:
     if any(
         cname in str(constructor_series)
-        for cname in ("pyarrow_series", "pandas_series_nullable", "pandas_series_pyarrow")
+        for cname in ("pandas_series_nullable", "pandas_series_pyarrow")
     ):
         request.applymarker(pytest.mark.xfail)
 
-    series = nw.from_native(constructor_series(data).rename("a"), series_only=True)
+    series = nw.from_native(constructor_series(data), series_only=True).alias("a")
 
     result = series.to_numpy()
     assert_array_equal(result, np.array([1, 3, 2]))
@@ -114,46 +114,6 @@ def test_to_numpy() -> None:
     assert nw_series.shape == (3,)
 
 
-def test_is_duplicated(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(data_dups), series_only=True)
-    result = series.is_duplicated()
-    expected = np.array([True, True, False])
-    assert (result.to_numpy() == expected).all()
-
-
-def test_is_unique(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(data_dups), series_only=True)
-    result = series.is_unique()
-    expected = np.array([False, False, True])
-    assert (result.to_numpy() == expected).all()
-
-
-def test_is_first_distinct(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(data_dups), series_only=True)
-    result = series.is_first_distinct()
-    expected = np.array([True, False, True])
-    assert (result.to_numpy() == expected).all()
-
-
-def test_is_last_distinct(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(data_dups), series_only=True)
-    result = series.is_last_distinct()
-    expected = np.array([False, True, True])
-    assert (result.to_numpy() == expected).all()
-
-
 def test_value_counts(request: Any, constructor_series: Any) -> None:
     if "pyarrow_series" in str(constructor_series):
         request.applymarker(pytest.mark.xfail)
@@ -177,35 +137,6 @@ def test_value_counts(request: Any, constructor_series: Any) -> None:
     assert (a[a[:, 0].argsort()] == expected).all()
 
 
-@pytest.mark.parametrize(
-    ("input_data", "descending", "expected"),
-    [(data, False, False), (data_sorted, False, True), (data_sorted, True, False)],
-)
-def test_is_sorted(
-    request: Any,
-    constructor_series: Any,
-    input_data: str,
-    descending: bool,  # noqa: FBT001
-    expected: bool,  # noqa: FBT001
-) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(input_data), series_only=True)
-    result = series.is_sorted(descending=descending)
-    assert result == expected
-
-
-def test_is_sorted_invalid(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
-    series = nw.from_native(constructor_series(data_sorted), series_only=True)
-
-    with pytest.raises(TypeError):
-        series.is_sorted(descending="invalid_type")  # type: ignore[arg-type]
-
-
 @pytest.mark.parametrize(
     ("interpolation", "expected"),
     [
diff --git a/tests/series_only/to_frame_test.py b/tests/series_only/to_frame_test.py
index fd4a9907..be26998f 100644
--- a/tests/series_only/to_frame_test.py
+++ b/tests/series_only/to_frame_test.py
@@ -1,16 +1,11 @@
 from typing import Any
 
-import pytest
-
 import narwhals.stable.v1 as nw
 from tests.utils import compare_dicts
 
 data = [1, 2, 3]
 
 
-def test_to_frame(request: Any, constructor_series: Any) -> None:
-    if "pyarrow_series" in str(constructor_series):
-        request.applymarker(pytest.mark.xfail)
-
+def test_to_frame(constructor_series: Any) -> None:
     df = nw.from_native(constructor_series(data), series_only=True).alias("").to_frame()
     compare_dicts(df, {"": [1, 2, 3]})