Repository: dask/dask
File: dask/dataframe/dask_expr/_util.py (Python)

Code before:
PANDAS_GE_300 = PANDAS_VERSION.major >= 3


def _calc_maybe_new_divisions(df, periods, freq):
    """Maybe calculate new divisions by periods of size freq

    Used to shift the divisions for the `shift` method. If freq isn't a fixed
    size (not anchored or relative), then the divisions are shifted
    appropriately.

    Returning None, indicates divisions ought to be cleared.

    Parameters
    ----------
    df : dd.DataFrame, dd.Series, or dd.Index
    periods : int
        The number of periods to shift.
    freq : DateOffset, timedelta, or time rule string
        The frequency to shift by.
    """
    if isinstance(freq, str):
        freq = pd.tseries.frequencies.to_offset(freq)

    is_offset = isinstance(freq, pd.DateOffset)
    if is_offset:
        if not isinstance(freq, pd.offsets.Tick):
            # Can't infer divisions on relative or anchored offsets, as
            # divisions may now split identical index value.
            # (e.g. index_partitions = [[1, 2, 3], [3, 4, 5]])
            return None  # Would need to clear divisions
    if df.known_divisions:
        divs = pd.Series(range(len(df.divisions)), index=df.divisions)
        divisions = divs.shift(periods, freq=freq).index
        return tuple(divisions)
    return df.divisions


def _validate_axis(axis=0, none_is_zero: bool = True) -> None | Literal[0, 1]:
    if axis not in (0, 1, "index", "columns", None):
        raise ValueError(f"No axis named {axis}")
    # convert to numeric axis
    numeric_axis: dict[str | None, Literal[0, 1]] = {"index": 0, "columns": 1}
    if none_is_zero:
        numeric_axis[None] = 0

    return numeric_axis.get(axis, axis)


def _convert_to_list(column) -> list | None:
    if column is None or isinstance(column, list):
        pass

Review comment (suggestion):
https://github.com/pandas-dev/pandas/pull/61985 fixed a handful of issues with the behavior of `offsets.Day`. We previously relied on `Day` being a subclass of `Tick` (which IIUC means a *fixed* 24H period), but as described in https://github.com/pandas-dev/pandas/issues/41943, Daylight Savings transitions means that's not always true (`Day` can mean 23, 24, or 25 H).

What does that mean for us in this function, `_calc_maybe_new_divisions`? The docstring says:

> If freq isn't a fixed size (not anchored or relative), then the divisions are shifted appropriately.

IIUC, the thing we're trying to prevent with this specific check is something like a shift by `pd.offsets.MonthEnd()`, which shifts all values to the end of the month that they're end, which might collapse many input values to the same output value:

```
In [12]: idx = pd.date_range("2025-03-07", "2025-03-11", freq="D")

In [13]: idx.shift(freq=pd.offsets.MonthEnd())
Out[13]:
DatetimeIndex(['2025-03-31', '2025-03-31', '2025-03-31', '2025-03-31',
               '2025-03-31'],
              dtype='datetime64[ns]', freq=None)
```

which would ruin our divisions. So is it possible to take a previously unique set of values and shift them in a way that collapses multiple inputs into the same value? I don't think so. This shift from before to into a DST transition errors:

```python
In [40]: before = pd.date_range("2025-03-08", "2025-03-09", freq="h", tz="US/Central")

In [41]: before.shift(freq='1D')
...

ValueError: 2025-03-09 02:00:00 is a nonexistent time due to daylight savings time. Try using the 'nonexistent' argument.
```

And shifting an index that already spans a DST transition and shifting doesn't cause any issues. So I think we're OK.

Code after:
PANDAS_GE_300 = PANDAS_VERSION.major >= 3


def _calc_maybe_new_divisions(df, periods, freq):
    """Maybe calculate new divisions by periods of size freq

    Used to shift the divisions for the `shift` method. If freq isn't a fixed
    size (not anchored or relative), then the divisions are shifted
    appropriately.

    Returning None, indicates divisions ought to be cleared.

    Parameters
    ----------
    df : dd.DataFrame, dd.Series, or dd.Index
    periods : int
        The number of periods to shift.
    freq : DateOffset, timedelta, or time rule string
        The frequency to shift by.
    """
    if isinstance(freq, str):
        freq = pd.tseries.frequencies.to_offset(freq)

    is_offset = isinstance(freq, pd.DateOffset)
    if is_offset:
        if not isinstance(freq, (pd.offsets.Tick, pd.offsets.Day)):
            # Can't infer divisions on relative or anchored offsets, as
            # divisions may now split identical index value.
            # (e.g. index_partitions = [[1, 2, 3], [3, 4, 5]])
            # As of https://github.com/pandas-dev/pandas/pull/61985
            # Day doesn't subclass Tick.
            return None  # Would need to clear divisions
    if df.known_divisions:
        divs = pd.Series(range(len(df.divisions)), index=df.divisions)
        divisions = divs.shift(periods, freq=freq).index
        return tuple(divisions)
    return df.divisions


def _validate_axis(axis=0, none_is_zero: bool = True) -> None | Literal[0, 1]:
    if axis not in (0, 1, "index", "columns", None):
        raise ValueError(f"No axis named {axis}")
    # convert to numeric axis
    numeric_axis: dict[str | None, Literal[0, 1]] = {"index": 0, "columns": 1}
    if none_is_zero:
        numeric_axis[None] = 0

    return numeric_axis.get(axis, axis)


def _convert_to_list(column) -> list | None: