Repository: narwhals-dev/narwhals
Pull request #1145 (2024-10-06 19:35:15)

feat: allow `format=None` in `str.to_datetime`
## What type of PR is this? (check all applicable)

- [ ] 💾 Refactor
- [x] ✨ Feature
- [ ] 🐛 Bug Fix
- [ ] 🔧 Optimization
- [ ] 📝 Documentation
- [ ] ✅ Test
- [ ] 🐳 Other

## Checklist

- [x] Code follows style guide (ruff)
- [x] Tests added 
- [x] Documented the changes

## If you have comments or can explain your changes, please do so below.

~~Opening as draft since~~ (Ok I misscliked) pyarrow is unsupported and may need some discussion for how to achieve it.

For context, this is the snippet in plotly that requires casting to datetime: [process_dataframe_timeline](https://github.com/plotly/plotly.py/blob/7181c4dbdd86400b2b984434e25128507aa8ea69/packages/python/plotly/plotly/express/_core.py#L1780)

FBruzzesi: > just a thought - it looks like [here](https://stackoverflow.com/a/76376483/4451315) there's a pure-python solution to inferring a format
> 
> * if it's pandas-like or polars, use `to_datetime` with `format=None`
> * else, take the first non-null element, get the format using the solution above, and try to parse using that?

Shall we consider doing that in narwhals for pyarrow backend and null format? I think it can be a quite common use case for anything related with timeseries.
MarcoGorelli: yeah sure!

Patch:
diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py
index 6d1001c1..c70425ef 100644
--- a/narwhals/_arrow/expr.py
+++ b/narwhals/_arrow/expr.py
@@ -540,7 +540,7 @@ class ArrowExprStringNamespace:
             self._expr, "str", "slice", offset, length
         )
 
-    def to_datetime(self, format: str | None = None) -> ArrowExpr:  # noqa: A002
+    def to_datetime(self: Self, format: str | None) -> ArrowExpr:  # noqa: A002
         return reuse_series_namespace_implementation(
             self._expr,
             "str",
diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py
index 183cf37b..50702349 100644
--- a/narwhals/_arrow/series.py
+++ b/narwhals/_arrow/series.py
@@ -1033,9 +1033,13 @@ class ArrowSeriesStringNamespace:
             ),
         )
 
-    def to_datetime(self: Self, format: str | None = None) -> ArrowSeries:  # noqa: A002
+    def to_datetime(self: Self, format: str | None) -> ArrowSeries:  # noqa: A002
         import pyarrow.compute as pc  # ignore-banned-import()
 
+        if format is None:
+            msg = "`format` is required for pyarrow backend."
+            raise ValueError(msg)
+
         return self._arrow_series._from_native_series(
             pc.strptime(self._arrow_series._native_series, format=format, unit="us")
         )
diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py
index d8d86692..10b95bc8 100644
--- a/narwhals/_dask/expr.py
+++ b/narwhals/_dask/expr.py
@@ -811,7 +811,7 @@ class DaskExprStringNamespace:
             returns_scalar=False,
         )
 
-    def to_datetime(self, format: str | None = None) -> DaskExpr:  # noqa: A002
+    def to_datetime(self: Self, format: str | None) -> DaskExpr:  # noqa: A002
         import dask.dataframe as dd  # ignore-banned-import()
 
         return self._expr._from_call(
diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py
index 4e301144..2ebadbe1 100644
--- a/narwhals/_pandas_like/expr.py
+++ b/narwhals/_pandas_like/expr.py
@@ -486,7 +486,7 @@ class PandasLikeExprStringNamespace:
             self._expr, "str", "slice", offset, length
         )
 
-    def to_datetime(self, format: str | None = None) -> PandasLikeExpr:  # noqa: A002
+    def to_datetime(self: Self, format: str | None) -> PandasLikeExpr:  # noqa: A002
         return reuse_series_namespace_implementation(
             self._expr,
             "str",
diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py
index 74e1c492..9cca6640 100644
--- a/narwhals/_pandas_like/series.py
+++ b/narwhals/_pandas_like/series.py
@@ -757,7 +757,7 @@ class PandasLikeSeriesStringNamespace:
             self._pandas_series._native_series.str.slice(start=offset, stop=stop),
         )
 
-    def to_datetime(self, format: str | None = None) -> PandasLikeSeries:  # noqa: A002
+    def to_datetime(self: Self, format: str | None) -> PandasLikeSeries:  # noqa: A002
         return self._pandas_series._from_native_series(
             to_datetime(self._pandas_series._implementation)(
                 self._pandas_series._native_series, format=format
diff --git a/narwhals/expr.py b/narwhals/expr.py
index 59e1ff76..8446d81c 100644
--- a/narwhals/expr.py
+++ b/narwhals/expr.py
@@ -2498,7 +2498,7 @@ class ExprStringNamespace:
         """
         return self._expr.__class__(lambda plx: self._expr._call(plx).str.slice(-n))
 
-    def to_datetime(self, format: str) -> Expr:  # noqa: A002
+    def to_datetime(self: Self, format: str | None = None) -> Expr:  # noqa: A002
         """
         Convert to Datetime dtype.
 
@@ -2508,10 +2508,13 @@ class ExprStringNamespace:
             in pandas, with no ability to set any other one. The ability to
             set the time unit in pandas, if the version permits, will arrive.
 
+        Warning:
+            As different backends auto-infer format in different ways, if `format=None`
+            there is no guarantee that the result will be equal.
+
         Arguments:
-            format: Format to parse strings with. Must be passed, as different
-                    dataframe libraries have different ways of auto-inferring
-                    formats.
+            format: Format to use for conversion. If set to None (default), the format is
+                inferred from the data.
 
         Examples:
             >>> import pandas as pd
diff --git a/narwhals/series.py b/narwhals/series.py
index bb970906..1753598c 100644
--- a/narwhals/series.py
+++ b/narwhals/series.py
@@ -3083,7 +3083,7 @@ class SeriesStringNamespace:
             self._narwhals_series._compliant_series.str.to_lowercase()
         )
 
-    def to_datetime(self, format: str) -> Series:  # noqa: A002
+    def to_datetime(self: Self, format: str | None = None) -> Series:  # noqa: A002
         """
         Parse Series with strings to a Series with Datetime dtype.
 
@@ -3093,10 +3093,13 @@ class SeriesStringNamespace:
             in pandas, with no ability to set any other one. The ability to
             set the time unit in pandas, if the version permits, will arrive.
 
+        Warning:
+            As different backends auto-infer format in different ways, if `format=None`
+            there is no guarantee that the result will be equal.
+
         Arguments:
-            format: Format to parse strings with. Must be passed, as different
-                    dataframe libraries have different ways of auto-inferring
-                    formats.
+            format: Format to use for conversion. If set to None (default), the format is
+                inferred from the data.
 
         Examples:
             >>> import pandas as pd
diff --git a/utils/check_api_reference.py b/utils/check_api_reference.py
index 60e968a8..7bc59042 100644
--- a/utils/check_api_reference.py
+++ b/utils/check_api_reference.py
@@ -121,6 +121,31 @@ if extra := set(documented).difference(series_methods):
     print(extra)  # noqa: T201
     ret = 1
 
+# Series.{cat, dt, str} methods
+for namespace in NAMESPACES.difference({"name"}):
+    series_methods = [
+        i
+        for i in getattr(
+            nw.from_native(pl.Series(), series_only=True), namespace
+        ).__dir__()
+        if not i[0].isupper() and i[0] != "_"
+    ]
+    with open(f"docs/api-reference/series_{namespace}.md") as fd:
+        content = fd.read()
+    documented = [
+        remove_prefix(i, "        - ")
+        for i in content.splitlines()
+        if i.startswith("        - ") and not i.startswith("        - _")
+    ]
+    if missing := set(series_methods).difference(documented):
+        print(f"Series.{namespace}: not documented")  # noqa: T201
+        print(missing)  # noqa: T201
+        ret = 1
+    if extra := set(documented).difference(series_methods):
+        print(f"Series.{namespace}: outdated")  # noqa: T201
+        print(extra)  # noqa: T201
+        ret = 1
+
 # Expr methods
 expr_methods = [
     i for i in nw.Expr(lambda: 0).__dir__() if not i[0].isupper() and i[0] != "_"
@@ -141,6 +166,29 @@ if extra := set(documented).difference(expr_methods):
     print(extra)  # noqa: T201
     ret = 1
 
+# Expr.{cat, dt, name, str} methods
+for namespace in NAMESPACES:
+    expr_methods = [
+        i
+        for i in getattr(nw.Expr(lambda: 0), namespace).__dir__()
+        if not i[0].isupper() and i[0] != "_"
+    ]
+    with open(f"docs/api-reference/expr_{namespace}.md") as fd:
+        content = fd.read()
+    documented = [
+        remove_prefix(i, "        - ")
+        for i in content.splitlines()
+        if i.startswith("        - ")
+    ]
+    if missing := set(expr_methods).difference(documented):
+        print(f"Expr.{namespace}: not documented")  # noqa: T201
+        print(missing)  # noqa: T201
+        ret = 1
+    if extra := set(documented).difference(expr_methods):
+        print(f"Expr.{namespace}: outdated")  # noqa: T201
+        print(extra)  # noqa: T201
+        ret = 1
+
 # DTypes
 dtypes = [
     i for i in nw.dtypes.__dir__() if i[0].isupper() and not i.isupper() and i[0] != "_"
@@ -156,32 +204,8 @@ if missing := set(dtypes).difference(documented).difference(BASE_DTYPES):
     print("Dtype: not documented")  # noqa: T201
     print(missing)  # noqa: T201
     ret = 1
-
-# dt
-
-# Series.str methods
-series_str_methods = [
-    i
-    for i in nw.from_native(pl.Series(), series_only=True).str.__dir__()
-    if not i[0].isupper() and i[0] != "_"
-]
-
-with open("docs/api-reference/series_str.md") as fd:
-    content = fd.read()
-
-documented = [
-    remove_prefix(i, "        - ")
-    for i in content.splitlines()
-    if i.startswith("        - ") and not i.startswith("        - _")
-]
-
-if missing := set(series_str_methods).difference(documented):
-    print("Series.str: not documented")  # noqa: T201
-    print(missing)  # noqa: T201
-    ret = 1
-
-if extra := set(documented).difference(series_str_methods):
-    print("Series.str: outdated")  # noqa: T201
+if extra := set(documented).difference(dtypes):
+    print("Dtype: outdated")  # noqa: T201
     print(extra)  # noqa: T201
     ret = 1
 


Test patch:
diff --git a/tests/expr_and_series/str/to_datetime_test.py b/tests/expr_and_series/str/to_datetime_test.py
index a64a3c58..8474357e 100644
--- a/tests/expr_and_series/str/to_datetime_test.py
+++ b/tests/expr_and_series/str/to_datetime_test.py
@@ -1,5 +1,7 @@
 from typing import Any
 
+import pytest
+
 import narwhals.stable.v1 as nw
 from tests.utils import Constructor
 
@@ -34,3 +36,41 @@ def test_to_datetime_series(constructor_eager: Any) -> None:
         )
     ).item(0)
     assert str(result) == expected
+
+
+def test_to_datetime_infer_fmt(
+    request: pytest.FixtureRequest, constructor: Constructor
+) -> None:
+    if "pyarrow_table" in str(constructor):
+        request.applymarker(pytest.mark.xfail)
+
+    if "cudf" in str(constructor):  # pragma: no cover
+        expected = "2020-01-01T12:34:56.000000000"
+    else:
+        expected = "2020-01-01 12:34:56"
+
+    result = (
+        nw.from_native(constructor(data))
+        .lazy()
+        .select(b=nw.col("a").str.to_datetime())
+        .collect()
+        .item(row=0, column="b")
+    )
+    assert str(result) == expected
+
+
+def test_to_datetime_series_infer_fmt(
+    request: pytest.FixtureRequest, constructor_eager: Any
+) -> None:
+    if "pyarrow_table" in str(constructor_eager):
+        request.applymarker(pytest.mark.xfail)
+
+    if "cudf" in str(constructor_eager):  # pragma: no cover
+        expected = "2020-01-01T12:34:56.000000000"
+    else:
+        expected = "2020-01-01 12:34:56"
+
+    result = (
+        nw.from_native(constructor_eager(data), eager_only=True)["a"].str.to_datetime()
+    ).item(0)
+    assert str(result) == expected