Repository: narwhals-dev/narwhals
Pull request #1712 (2025-01-03 11:44:45)

fix: always pass Column to pandas.spark.functions
<!--
# Thanks for contributing a pull request! 
## Please make sure you see our contribution guidelines: https://github.com/narwhals-dev/narwhals/blob/main/CONTRIBUTING.md
-->

## What type of PR is this? (check all applicable)

- [ ] 💾 Refactor
- [ ] ✨ Feature
- [x] 🐛 Bug Fix
- [ ] 🔧 Optimization
- [ ] 📝 Documentation
- [ ] ✅ Test
- [ ] 🐳 Other

## Related issues


## Checklist

- [x] Code follows style guide (ruff)
- [x] Tests added
- [x] Documented the changes

## If you have comments or can explain your changes, please do so below

I guess it was not picked up in the CI because `numpy` version is > 2.0.
Should we have an extra test session with `numpy<2.0` and pyspark? 


Patch:
diff --git a/docs/api-reference/expr_dt.md b/docs/api-reference/expr_dt.md
index 604ac4ab..15cbe5ef 100644
--- a/docs/api-reference/expr_dt.md
+++ b/docs/api-reference/expr_dt.md
@@ -23,6 +23,7 @@
         - total_nanoseconds
         - total_seconds
         - to_string
+        - weekday
         - year
       show_source: false
       show_bases: false
diff --git a/docs/api-reference/series_dt.md b/docs/api-reference/series_dt.md
index 23d4817c..6d93e2ea 100644
--- a/docs/api-reference/series_dt.md
+++ b/docs/api-reference/series_dt.md
@@ -23,6 +23,7 @@
         - total_nanoseconds
         - total_seconds
         - to_string
+        - weekday
         - year
       show_source: false
       show_bases: false
diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py
index 15afcba1..ac0c0737 100644
--- a/narwhals/_arrow/expr.py
+++ b/narwhals/_arrow/expr.py
@@ -658,6 +658,11 @@ class ArrowExprDateTimeNamespace:
             self._compliant_expr, "dt", "ordinal_day"
         )
 
+    def weekday(self: Self) -> ArrowExpr:
+        return reuse_series_namespace_implementation(
+            self._compliant_expr, "dt", "weekday"
+        )
+
     def total_minutes(self: Self) -> ArrowExpr:
         return reuse_series_namespace_implementation(
             self._compliant_expr, "dt", "total_minutes"
diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py
index 239ab5c1..c50942ba 100644
--- a/narwhals/_arrow/series.py
+++ b/narwhals/_arrow/series.py
@@ -1303,6 +1303,13 @@ class ArrowSeriesDateTimeNamespace:
             pc.day_of_year(self._compliant_series._native_series)
         )
 
+    def weekday(self: Self) -> ArrowSeries:
+        import pyarrow.compute as pc
+
+        return self._compliant_series._from_native_series(
+            pc.day_of_week(self._compliant_series._native_series, count_from_zero=False)
+        )
+
     def total_minutes(self: Self) -> ArrowSeries:
         import pyarrow as pa
         import pyarrow.compute as pc
diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py
index 8da2d422..c64401a4 100644
--- a/narwhals/_dask/expr.py
+++ b/narwhals/_dask/expr.py
@@ -1212,6 +1212,13 @@ class DaskExprDateTimeNamespace:
             returns_scalar=self._compliant_expr._returns_scalar,
         )
 
+    def weekday(self) -> DaskExpr:
+        return self._compliant_expr._from_call(
+            lambda _input: _input.dt.weekday + 1,  # Dask is 0-6
+            "weekday",
+            returns_scalar=self._compliant_expr._returns_scalar,
+        )
+
     def to_string(self, format: str) -> DaskExpr:  # noqa: A002
         return self._compliant_expr._from_call(
             lambda _input, format: _input.dt.strftime(format.replace("%.f", ".%f")),
diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py
index 07543011..3e4682c9 100644
--- a/narwhals/_pandas_like/expr.py
+++ b/narwhals/_pandas_like/expr.py
@@ -835,6 +835,11 @@ class PandasLikeExprDateTimeNamespace:
             self._compliant_expr, "dt", "ordinal_day"
         )
 
+    def weekday(self) -> PandasLikeExpr:
+        return reuse_series_namespace_implementation(
+            self._compliant_expr, "dt", "weekday"
+        )
+
     def total_minutes(self) -> PandasLikeExpr:
         return reuse_series_namespace_implementation(
             self._compliant_expr, "dt", "total_minutes"
diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py
index c9f3d006..49fa2a18 100644
--- a/narwhals/_pandas_like/series.py
+++ b/narwhals/_pandas_like/series.py
@@ -1298,6 +1298,14 @@ class PandasLikeSeriesDateTimeNamespace:
             )
         )
 
+    def weekday(self) -> PandasLikeSeries:
+        return (
+            self._compliant_series._from_native_series(
+                self._compliant_series._native_series.dt.weekday,
+            )
+            + 1  # Pandas is 0-6 while Polars is 1-7
+        )
+
     def _get_total_seconds(self) -> Any:
         if hasattr(self._compliant_series._native_series.dt, "total_seconds"):
             return self._compliant_series._native_series.dt.total_seconds()
diff --git a/narwhals/_spark_like/utils.py b/narwhals/_spark_like/utils.py
index 12892ca0..a3c77033 100644
--- a/narwhals/_spark_like/utils.py
+++ b/narwhals/_spark_like/utils.py
@@ -121,7 +121,7 @@ def maybe_evaluate(df: SparkLikeLazyFrame, obj: Any) -> Any:
 
 
 def _std(
-    _input: Column,
+    _input: Column | str,
     ddof: int,
     backend_version: tuple[int, ...],
     np_version: tuple[int, ...],
@@ -136,12 +136,14 @@ def _std(
         return F.stddev_samp(_input) * F.sqrt((n_rows - 1) / (n_rows - ddof))
 
     from pyspark.pandas.spark.functions import stddev
+    from pyspark.sql import functions as F  # noqa: N812
 
-    return stddev(_input, ddof=ddof)
+    input_col = F.col(_input) if isinstance(_input, str) else _input
+    return stddev(input_col, ddof=ddof)
 
 
 def _var(
-    _input: Column,
+    _input: Column | str,
     ddof: int,
     backend_version: tuple[int, ...],
     np_version: tuple[int, ...],
@@ -156,5 +158,7 @@ def _var(
         return F.var_samp(_input) * (n_rows - 1) / (n_rows - ddof)
 
     from pyspark.pandas.spark.functions import var
+    from pyspark.sql import functions as F  # noqa: N812
 
-    return var(_input, ddof=ddof)
+    input_col = F.col(_input) if isinstance(_input, str) else _input
+    return var(input_col, ddof=ddof)
diff --git a/narwhals/expr.py b/narwhals/expr.py
index ba0896db..d135ea70 100644
--- a/narwhals/expr.py
+++ b/narwhals/expr.py
@@ -5203,6 +5203,59 @@ class ExprDateTimeNamespace(Generic[ExprT]):
             lambda plx: self._expr._to_compliant_expr(plx).dt.ordinal_day()
         )
 
+    def weekday(self: Self) -> ExprT:
+        """Extract the week day from the underlying Date representation.
+
+        Returns:
+            Returns the ISO weekday number where monday = 1 and sunday = 7
+
+
+        Examples:
+            >>> from datetime import datetime
+            >>> import pandas as pd
+            >>> import polars as pl
+            >>> import pyarrow as pa
+            >>> import narwhals as nw
+            >>> from narwhals.typing import IntoFrameT
+            >>> data = {"a": [datetime(2020, 1, 1), datetime(2020, 8, 3)]}
+            >>> df_pd = pd.DataFrame(data)
+            >>> df_pl = pl.DataFrame(data)
+            >>> df_pa = pa.table(data)
+
+            We define a dataframe-agnostic function:
+
+            >>> def agnostic_weekday(df_native: IntoFrameT) -> IntoFrameT:
+            ...     df = nw.from_native(df_native)
+            ...     return df.with_columns(a_weekday=nw.col("a").dt.weekday()).to_native()
+
+            We can then pass either pandas, Polars, PyArrow, and other supported libraries to `agnostic_weekday`:
+
+            >>> agnostic_weekday(df_pd)
+                       a  a_weekday
+            0 2020-01-01          3
+            1 2020-08-03          1
+            >>> agnostic_weekday(df_pl)
+            shape: (2, 2)
+            ┌─────────────────────┬───────────┐
+            │ a                   ┆ a_weekday │
+            │ ---                 ┆ ---       │
+            │ datetime[μs]        ┆ i8        │
+            ╞═════════════════════╪═══════════╡
+            │ 2020-01-01 00:00:00 ┆ 3         │
+            │ 2020-08-03 00:00:00 ┆ 1         │
+            └─────────────────────┴───────────┘
+            >>> agnostic_weekday(df_pa)
+            pyarrow.Table
+            a: timestamp[us]
+            a_weekday: int64
+            ----
+            a: [[2020-01-01 00:00:00.000000,2020-08-03 00:00:00.000000]]
+            a_weekday: [[3,1]]
+        """
+        return self._expr.__class__(
+            lambda plx: self._expr._to_compliant_expr(plx).dt.weekday()
+        )
+
     def total_minutes(self: Self) -> ExprT:
         """Get total minutes.
 
diff --git a/narwhals/series.py b/narwhals/series.py
index 4203cdb7..a6f4736a 100644
--- a/narwhals/series.py
+++ b/narwhals/series.py
@@ -6250,6 +6250,58 @@ class SeriesDateTimeNamespace(Generic[SeriesT]):
             self._narwhals_series._compliant_series.dt.ordinal_day()
         )
 
+    def weekday(self: Self) -> SeriesT:
+        """Extract the week day in a datetime series.
+
+        Returns:
+            A new Series containing the week day for each datetime value.
+            Returns the ISO weekday number where monday = 1 and sunday = 7
+
+
+        Examples:
+            >>> from datetime import datetime
+            >>> import pandas as pd
+            >>> import polars as pl
+            >>> import pyarrow as pa
+            >>> import narwhals as nw
+            >>> from narwhals.typing import IntoSeriesT
+            >>> data = [datetime(2020, 1, 1), datetime(2020, 8, 3)]
+            >>> s_pd = pd.Series(data)
+            >>> s_pl = pl.Series(data)
+            >>> s_pa = pa.chunked_array([data])
+
+            We define a library agnostic function:
+
+            >>> def agnostic_weekday(s_native: IntoSeriesT) -> IntoSeriesT:
+            ...     s = nw.from_native(s_native, series_only=True)
+            ...     return s.dt.weekday().to_native()
+
+            We can then pass either pandas, Polars, PyArrow, and other supported libraries to `agnostic_weekday`:
+
+            >>> agnostic_weekday(s_pd)
+            0    3
+            1    1
+            dtype: int32
+            >>> agnostic_weekday(s_pl)  # doctest: +NORMALIZE_WHITESPACE
+            shape: (2,)
+            Series: '' [i8]
+            [
+               3
+               1
+            ]
+            >>> agnostic_weekday(s_pa)  # doctest: +ELLIPSIS
+            <pyarrow.lib.ChunkedArray object at ...>
+            [
+              [
+                3,
+                1
+              ]
+            ]
+        """
+        return self._narwhals_series._from_compliant_series(
+            self._narwhals_series._compliant_series.dt.weekday()
+        )
+
     def total_minutes(self: Self) -> SeriesT:
         """Get total minutes.
 
diff --git a/noxfile.py b/noxfile.py
index 49bb6053..87825be3 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -14,12 +14,12 @@ PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
 
 def run_common(session: Session, coverage_threshold: float) -> None:
-    if session.python == "3.12":
-        session.install("-e .[dev,extra,dask,modin]")
-    elif session.python != "3.8":
-        session.install("-e .[dev,extra,dask,modin,pyspark,ibis]")
+    if session.python == "3.8":
+        session.install("-e .[dev,core]")
+    elif session.python == "3.12":
+        session.install("-e .[dev,core,extra,dask,modin]")
     else:
-        session.install("-e .[dev]")
+        session.install("-e .[dev,core,extra,dask,modin,pyspark,ibis]")
 
     session.run(
         "pytest",


Test patch:
diff --git a/tests/expr_and_series/dt/datetime_attributes_test.py b/tests/expr_and_series/dt/datetime_attributes_test.py
index 0235920e..ad5f8dc3 100644
--- a/tests/expr_and_series/dt/datetime_attributes_test.py
+++ b/tests/expr_and_series/dt/datetime_attributes_test.py
@@ -32,6 +32,7 @@ data = {
         ("microsecond", [49000, 715000]),
         ("nanosecond", [49000000, 715000000]),
         ("ordinal_day", [60, 2]),
+        ("weekday", [1, 4]),
     ],
 )
 def test_datetime_attributes(
@@ -68,6 +69,7 @@ def test_datetime_attributes(
         ("microsecond", [49000, 715000]),
         ("nanosecond", [49000000, 715000000]),
         ("ordinal_day", [60, 2]),
+        ("weekday", [1, 4]),
     ],
 )
 def test_datetime_attributes_series(