Repository: narwhals-dev/narwhals
Pull request #542 (2024-07-16 20:31:37)

feat: add Series.drop nulls for pyarrow
<!--
# Thanks for contributing a pull request! 
## Please make sure you see our contribution guidelines: https://github.com/narwhals-dev/narwhals/blob/main/CONTRIBUTING.md
-->

## What type of PR is this? (check all applicable)

- [ ] 💾 Refactor
- [x] ✨ Feature
- [ ] 🐛 Bug Fix
- [ ] 🔧 Optimization
- [ ] 📝 Documentation
- [x] ✅ Test
- [ ] 🐳 Other

## Related issues 

- Related issue # 
- Closes #

## Checklist

- [ ] Code follows style guide (ruff)
- [x] Tests added 
- [ ] Documented the changes

## If you have comments or can explain your changes, please do so below.



mfonekpo: Yes I can. I opted for this because of error I was getting. And when I did
this, I wasn't getting the error again. Let me revert to it.

On Tue, 16 Jul 2024, 21:44 Marco Edward Gorelli, ***@***.***>
wrote:

> ***@***.**** commented on this pull request.
> ------------------------------
>
> In tests/series/drop_nulls_test.py
> <https://github.com/narwhals-dev/narwhals/pull/542#discussion_r1680043888>
> :
>
> > +    result_a = df.select(nw.col("A").drop_nulls())
> +    result_b = df.select(nw.col("B").drop_nulls())
> +    result_c = df.select(nw.col("C").drop_nulls())
> +    result_d = df.select(nw.col("D").drop_nulls())
>
> can you do all of these in the same select statement? so
>
> result = df.select(
>     nw.col('a').drop_nulls(),
>     nw.col('b').drop_nulls(),
> )
>
> etc...
>
> and then just make a single dict for expected
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/narwhals-dev/narwhals/pull/542#pullrequestreview-2181274052>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AOA4GJDWOVBC4P3PB6X7H4LZMWAZDAVCNFSM6AAAAABK7JXXQOVHI2DSMVQWIX3LMV43YUDVNRWFEZLROVSXG5CSMV3GSZLXHMZDCOBRGI3TIMBVGI>
> .
> You are receiving this because you authored the thread.Message ID:
> ***@***.***>
>

MarcoGorelli: > I opted for this because of error I was getting. And when I did
this, I wasn't getting the error again

Open source tip : always explain this kind of thing upfront. It's fine here, but if you do it in other projects, you risk losing their trust
mfonekpo: Thank you Marco for the tip. I will always bear that in mind.

On Tue, Jul 16, 2024 at 11:20 PM Marco Edward Gorelli <
***@***.***> wrote:

> I opted for this because of error I was getting. And when I did
> this, I wasn't getting the error again
>
> Open source tip : always explain this kind of thing upfront. It's fine
> here, but if you do it in other projects, you risk losing their trust
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/narwhals-dev/narwhals/pull/542#issuecomment-2231916481>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AOA4GJAUP5Q32S2IO3FYW5DZMWMCPAVCNFSM6AAAAABK7JXXQOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDEMZRHEYTMNBYGE>
> .
> You are receiving this because you authored the thread.Message ID:
> ***@***.***>
>


Patch:
diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py
index 0f0d5c3c..d0082a31 100644
--- a/narwhals/_arrow/expr.py
+++ b/narwhals/_arrow/expr.py
@@ -189,6 +189,9 @@ class ArrowExpr:
     def sum(self) -> Self:
         return reuse_series_implementation(self, "sum", returns_scalar=True)
 
+    def drop_nulls(self) -> Self:
+        return reuse_series_implementation(self, "drop_nulls")
+
     def alias(self, name: str) -> Self:
         # Define this one manually, so that we can
         # override `output_names` and not increase depth
diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py
index f6828a30..8bed4189 100644
--- a/narwhals/_arrow/series.py
+++ b/narwhals/_arrow/series.py
@@ -203,6 +203,10 @@ class ArrowSeries:
         pc = get_pyarrow_compute()
         return pc.sum(self._native_series)  # type: ignore[no-any-return]
 
+    def drop_nulls(self) -> ArrowSeries:
+        pc = get_pyarrow_compute()
+        return self._from_native_series(pc.drop_null(self._native_series))
+
     def std(self, ddof: int = 1) -> int:
         pc = get_pyarrow_compute()
         return pc.stddev(self._native_series, ddof=ddof)  # type: ignore[no-any-return]
diff --git a/utils/check_backend_completeness.py b/utils/check_backend_completeness.py
index b83766bc..d51ecf64 100644
--- a/utils/check_backend_completeness.py
+++ b/utils/check_backend_completeness.py
@@ -14,7 +14,6 @@ from narwhals._arrow.dataframe import ArrowDataFrame
 MISSING = [
     "DataFrame.iter_rows",
     "DataFrame.pipe",
-    "Series.drop_nulls",
     "Series.from_iterable",
     "Series.is_between",
     "Series.quantile",


Test patch:
diff --git a/tests/expr_and_series/drop_nulls_test.py b/tests/expr_and_series/drop_nulls_test.py
new file mode 100644
index 00000000..ef109e01
--- /dev/null
+++ b/tests/expr_and_series/drop_nulls_test.py
@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import Any
+
+import narwhals as nw
+from tests.utils import compare_dicts
+
+
+def test_drop_nulls(constructor: Any) -> None:
+    data = {
+        "A": [1, 2, None, 4],
+        "B": [5, 6, 7, 8],
+        "C": [None, None, None, None],
+        "D": [9, 10, 11, 12],
+    }
+
+    df = nw.from_native(constructor(data), eager_only=True)
+
+    result_a = df.select(nw.col("A").drop_nulls())
+    result_b = df.select(df["B"].drop_nulls())
+    result_c = df.select(nw.col("C").drop_nulls())
+    result_d = df.select(nw.col("D").drop_nulls())
+    expected_a = {"A": [1.0, 2.0, 4.0]}
+    expected_b = {"B": [5, 6, 7, 8]}
+    expected_c = {"C": []}  # type: ignore[var-annotated]
+    expected_d = {"D": [9, 10, 11, 12]}
+
+    compare_dicts(result_a, expected_a)
+    compare_dicts(result_b, expected_b)
+    compare_dicts(result_c, expected_c)
+    compare_dicts(result_d, expected_d)
diff --git a/tests/frame/test_common.py b/tests/frame/test_common.py
index bf7f619f..9030f99c 100644
--- a/tests/frame/test_common.py
+++ b/tests/frame/test_common.py
@@ -159,21 +159,6 @@ def test_expr_na(constructor_with_lazy: Any) -> None:
     compare_dicts(result_nna, expected)
 
 
-def test_drop_nulls(request: Any, constructor_with_lazy: Any) -> None:
-    if "pyarrow_table" in str(constructor_with_lazy):
-        request.applymarker(pytest.mark.xfail)
-
-    df = nw.from_native(constructor_with_lazy(data_na)).lazy()
-
-    result = df.select(nw.col("a").drop_nulls())
-    expected = {"a": [3, 2]}
-    compare_dicts(result, expected)
-
-    result = df.select(df.collect()["a"].drop_nulls())
-    expected = {"a": [3, 2]}
-    compare_dicts(result, expected)
-
-
 @pytest.mark.parametrize(
     ("drop", "left"),
     [