Repository: pandas-dev/pandas
Pull request #58384 (2024-04-23 16:05:54)

DOC: Enforce Numpy Docstring Validation for pandas.HDFStore.put
- [ ] xref #58067 



Patch:
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
index 801fe7eccd..5993fabfc9 100755
--- a/ci/code_checks.sh
+++ b/ci/code_checks.sh
@@ -99,7 +99,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.DataFrame.std PR01,RT03,SA01" \
         -i "pandas.DataFrame.sum RT03" \
         -i "pandas.DataFrame.swaplevel SA01" \
-        -i "pandas.DataFrame.to_feather SA01" \
         -i "pandas.DataFrame.to_markdown SA01" \
         -i "pandas.DataFrame.to_parquet RT03" \
         -i "pandas.DataFrame.var PR01,RT03,SA01" \
@@ -119,7 +118,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.DatetimeTZDtype.tz SA01" \
         -i "pandas.DatetimeTZDtype.unit SA01" \
         -i "pandas.Grouper PR02" \
-        -i "pandas.HDFStore.put PR01,SA01" \
         -i "pandas.HDFStore.walk SA01" \
         -i "pandas.Index PR07" \
         -i "pandas.Index.T SA01" \
diff --git a/doc/source/development/community.rst b/doc/source/development/community.rst
index ccf7be8e47..ab8294b8f1 100644
--- a/doc/source/development/community.rst
+++ b/doc/source/development/community.rst
@@ -100,6 +100,8 @@ The pandas mailing list `pandas-dev@python.org <mailto://pandas-dev@python
 conversations and to engage people in the wider community who might not
 be active on the issue tracker but we would like to include in discussions.
 
+Join the mailing list and view the archives `here <https://mail.python.org/mailman/listinfo/pandas-dev>`_.
+
 .. _community.slack:
 
 Community slack
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
index 7823f74b7a..4213cc8e6c 100644
--- a/doc/source/whatsnew/v3.0.0.rst
+++ b/doc/source/whatsnew/v3.0.0.rst
@@ -360,6 +360,7 @@ Datetimelike
 - Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`)
 - Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56382`)
 - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`)
+- Bug in setting scalar values with mismatched resolution into arrays with non-nanosecond ``datetime64``, ``timedelta64`` or :class:`DatetimeTZDtype` incorrectly truncating those scalars (:issue:`56410`)
 
 Timedelta
 ^^^^^^^^^
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 203308b4f0..be087e19ce 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -539,7 +539,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):  # type: ignore[misc]
         if value is NaT:
             return np.datetime64(value._value, self.unit)
         else:
-            return value.as_unit(self.unit).asm8
+            return value.as_unit(self.unit, round_ok=False).asm8
 
     def _scalar_from_string(self, value) -> Timestamp | NaTType:
         return Timestamp(value, tz=self.tz)
diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py
index 6eb4d234b3..ff43f97161 100644
--- a/pandas/core/arrays/timedeltas.py
+++ b/pandas/core/arrays/timedeltas.py
@@ -322,7 +322,7 @@ class TimedeltaArray(dtl.TimelikeOps):
         if value is NaT:
             return np.timedelta64(value._value, self.unit)
         else:
-            return value.as_unit(self.unit).asm8
+            return value.as_unit(self.unit, round_ok=False).asm8
 
     def _scalar_from_string(self, value) -> Timedelta | NaTType:
         return Timedelta(value)
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index 4d89272013..e8a0e37b70 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -2685,6 +2685,16 @@ class DataFrame(NDFrame, OpsMixin):
             This includes the `compression`, `compression_level`, `chunksize`
             and `version` keywords.
 
+        See Also
+        --------
+        DataFrame.to_parquet : Write a DataFrame to the binary parquet format.
+        DataFrame.to_excel : Write object to an Excel sheet.
+        DataFrame.to_sql : Write to a sql table.
+        DataFrame.to_csv : Write a csv file.
+        DataFrame.to_json : Convert the object to a JSON string.
+        DataFrame.to_html : Render a DataFrame as an HTML table.
+        DataFrame.to_string : Convert DataFrame to a string.
+
         Notes
         -----
         This function writes the dataframe as a `feather file
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 7122de745e..6d5f32774f 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -515,6 +515,8 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
         freq = OFFSET_TO_PERIOD_FREQSTR.get(reso.attr_abbrev, reso.attr_abbrev)
         per = Period(parsed, freq=freq)
         start, end = per.start_time, per.end_time
+        start = start.as_unit(self.unit)
+        end = end.as_unit(self.unit)
 
         # GH 24076
         # If an incoming date string contained a UTC offset, need to localize
diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py
index 1b72c164f7..28d3292a1c 100644
--- a/pandas/core/internals/blocks.py
+++ b/pandas/core/internals/blocks.py
@@ -38,7 +38,10 @@ from pandas._typing import (
     Shape,
     npt,
 )
-from pandas.errors import AbstractMethodError
+from pandas.errors import (
+    AbstractMethodError,
+    OutOfBoundsDatetime,
+)
 from pandas.util._decorators import cache_readonly
 from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_bool_kwarg
@@ -478,7 +481,17 @@ class Block(PandasObject, libinternals.Block):
                 f"{self.values.dtype}. Please report a bug at "
                 "https://github.com/pandas-dev/pandas/issues."
             )
-        return self.astype(new_dtype)
+        try:
+            return self.astype(new_dtype)
+        except OutOfBoundsDatetime as err:
+            # e.g. GH#56419 if self.dtype is a low-resolution dt64 and we try to
+            #  upcast to a higher-resolution dt64, we may have entries that are
+            #  out of bounds for the higher resolution.
+            #  Re-raise with a more informative message.
+            raise OutOfBoundsDatetime(
+                f"Incompatible (high-resolution) value for dtype='{self.dtype}'. "
+                "Explicitly cast before operating."
+            ) from err
 
     @final
     def convert(self) -> list[Block]:
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
index 0af5c75397..75e9b779e5 100644
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -1144,12 +1144,27 @@ class HDFStore:
             Write DataFrame index as a column.
         append : bool, default False
             This will force Table format, append the input data to the existing.
+        complib : default None
+            This parameter is currently not accepted.
+        complevel : int, 0-9, default None
+            Specifies a compression level for data.
+            A value of 0 or None disables compression.
+        min_itemsize : int, dict, or None
+            Dict of columns that specify minimum str sizes.
+        nan_rep : str
+            Str to use as str nan representation.
         data_columns : list of columns or True, default None
             List of columns to create as data columns, or True to use all columns.
             See `here
             <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
         encoding : str, default None
             Provide an encoding for strings.
+        errors : str, default 'strict'
+            The error handling scheme to use for encoding errors.
+            The default is 'strict' meaning that encoding errors raise a
+            UnicodeEncodeError.  Other possible values are 'ignore', 'replace' and
+            'xmlcharrefreplace' as well as any other name registered with
+            codecs.register_error that can handle UnicodeEncodeErrors.
         track_times : bool, default True
             Parameter is propagated to 'create_table' method of 'PyTables'.
             If set to False it enables to have the same h5 files (same hashes)
@@ -1157,6 +1172,11 @@ class HDFStore:
         dropna : bool, default False, optional
             Remove missing values.
 
+        See Also
+        --------
+        HDFStore.info : Prints detailed information on the store.
+        HDFStore.get_storer : Returns the storer object for a key.
+
         Examples
         --------
         >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])


Test patch:
diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py
index 99535f2730..7a2a4892f6 100644
--- a/pandas/tests/series/indexing/test_setitem.py
+++ b/pandas/tests/series/indexing/test_setitem.py
@@ -1467,6 +1467,39 @@ class TestCoercionFloat32(CoercionTest):
             raise AssertionError("xfail not relevant for this test.")
 
 
+@pytest.mark.parametrize(
+    "exp_dtype",
+    [
+        "M8[ms]",
+        "M8[ms, UTC]",
+        "m8[ms]",
+    ],
+)
+class TestCoercionDatetime64HigherReso(CoercionTest):
+    @pytest.fixture
+    def obj(self, exp_dtype):
+        idx = date_range("2011-01-01", freq="D", periods=4, unit="s")
+        if exp_dtype == "m8[ms]":
+            idx = idx - Timestamp("1970-01-01")
+            assert idx.dtype == "m8[s]"
+        elif exp_dtype == "M8[ms, UTC]":
+            idx = idx.tz_localize("UTC")
+        return Series(idx)
+
+    @pytest.fixture
+    def val(self, exp_dtype):
+        ts = Timestamp("2011-01-02 03:04:05.678").as_unit("ms")
+        if exp_dtype == "m8[ms]":
+            return ts - Timestamp("1970-01-01")
+        elif exp_dtype == "M8[ms, UTC]":
+            return ts.tz_localize("UTC")
+        return ts
+
+    @pytest.fixture
+    def warn(self):
+        return FutureWarning
+
+
 @pytest.mark.parametrize(
     "val,exp_dtype,warn",
     [
diff --git a/pandas/tests/series/methods/test_clip.py b/pandas/tests/series/methods/test_clip.py
index 75b4050c18..8ed422fc11 100644
--- a/pandas/tests/series/methods/test_clip.py
+++ b/pandas/tests/series/methods/test_clip.py
@@ -3,6 +3,8 @@ from datetime import datetime
 import numpy as np
 import pytest
 
+from pandas.errors import OutOfBoundsDatetime
+
 import pandas as pd
 from pandas import (
     Series,
@@ -131,12 +133,30 @@ class TestSeriesClip:
         )
         tm.assert_series_equal(result, expected)
 
-    @pytest.mark.parametrize("dtype", [object, "M8[us]"])
-    def test_clip_with_timestamps_and_oob_datetimes(self, dtype):
+    def test_clip_with_timestamps_and_oob_datetimes_object(self):
         # GH-42794
-        ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=dtype)
+        ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=object)
 
         result = ser.clip(lower=Timestamp.min, upper=Timestamp.max)
-        expected = Series([Timestamp.min, Timestamp.max], dtype=dtype)
+        expected = Series([Timestamp.min, Timestamp.max], dtype=object)
+
+        tm.assert_series_equal(result, expected)
+
+    def test_clip_with_timestamps_and_oob_datetimes_non_nano(self):
+        # GH#56410
+        dtype = "M8[us]"
+        ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=dtype)
+
+        msg = (
+            r"Incompatible \(high-resolution\) value for dtype='datetime64\[us\]'. "
+            "Explicitly cast before operating"
+        )
+        with pytest.raises(OutOfBoundsDatetime, match=msg):
+            ser.clip(lower=Timestamp.min, upper=Timestamp.max)
+
+        lower = Timestamp.min.as_unit("us")
+        upper = Timestamp.max.as_unit("us")
+        result = ser.clip(lower=lower, upper=upper)
+        expected = Series([lower, upper], dtype=dtype)
 
         tm.assert_series_equal(result, expected)
diff --git a/pandas/tests/series/methods/test_fillna.py b/pandas/tests/series/methods/test_fillna.py
index 0965d36e48..592dba2535 100644
--- a/pandas/tests/series/methods/test_fillna.py
+++ b/pandas/tests/series/methods/test_fillna.py
@@ -308,12 +308,7 @@ class TestSeriesFillNA:
         "scalar",
         [
             False,
-            pytest.param(
-                True,
-                marks=pytest.mark.xfail(
-                    reason="GH#56410 scalar case not yet addressed"
-                ),
-            ),
+            True,
         ],
     )
     @pytest.mark.parametrize("tz", [None, "UTC"])
@@ -342,12 +337,7 @@ class TestSeriesFillNA:
         "scalar",
         [
             False,
-            pytest.param(
-                True,
-                marks=pytest.mark.xfail(
-                    reason="GH#56410 scalar case not yet addressed"
-                ),
-            ),
+            True,
         ],
     )
     def test_timedelta64_fillna_mismatched_reso_no_rounding(self, scalar):