Repository: pandas-dev/pandas
Pull request #55617 (2023-10-21 15:54:11)

REF: implement ParseState
In the near future, the state being tracked in array_to_datetime and array_strptime will include a resolution (xref #55564).  This implements a ParseState object to de-duplicate tracking, validation, and updating.


Patch:
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index 9eb5bbc8f0..fc199d3fd0 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -365,8 +365,10 @@ I/O
 - Bug in :func:`read_csv` with ``engine="pyarrow"`` where ``usecols`` wasn't working with a csv with no headers (:issue:`54459`)
 - Bug in :func:`read_excel`, with ``engine="xlrd"`` (``xls`` files) erroring when file contains NaNs/Infs (:issue:`54564`)
 - Bug in :func:`to_excel`, with ``OdsWriter`` (``ods`` files) writing boolean/string value (:issue:`54994`)
+- Bug in :meth:`DataFrame.to_hdf` and :func:`read_hdf` with ``datetime64`` dtypes with non-nanosecond resolution failing to round-trip correctly (:issue:`55622`)
 - Bug in :meth:`pandas.read_excel` with an ODS file without cached formatted cell for float values (:issue:`55219`)
 - Bug where :meth:`DataFrame.to_json` would raise an ``OverflowError`` instead of a ``TypeError`` with unsupported NumPy types (:issue:`55403`)
+-
 
 Period
 ^^^^^^
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
index 4989feaf84..8fdbd9affa 100644
--- a/pandas/_libs/tslib.pyx
+++ b/pandas/_libs/tslib.pyx
@@ -46,7 +46,10 @@ from pandas._libs.tslibs.np_datetime cimport (
 import_pandas_datetime()
 
 
-from pandas._libs.tslibs.strptime cimport parse_today_now
+from pandas._libs.tslibs.strptime cimport (
+    DatetimeParseState,
+    parse_today_now,
+)
 from pandas._libs.util cimport (
     is_float_object,
     is_integer_object,
@@ -58,7 +61,6 @@ from pandas._libs.tslibs.conversion cimport (
     _TSObject,
     cast_from_unit,
     convert_str_to_tsobject,
-    convert_timezone,
     get_datetime64_nanos,
     parse_pydatetime,
 )
@@ -454,9 +456,9 @@ cpdef array_to_datetime(
         float tz_offset
         set out_tzoffset_vals = set()
         tzinfo tz_out = None
-        bint found_tz = False, found_naive = False
         cnp.flatiter it = cnp.PyArray_IterNew(values)
         NPY_DATETIMEUNIT creso = NPY_FR_ns
+        DatetimeParseState state = DatetimeParseState()
 
     # specify error conditions
     assert is_raise or is_ignore or is_coerce
@@ -474,17 +476,7 @@ cpdef array_to_datetime(
                 iresult[i] = NPY_NAT
 
             elif PyDateTime_Check(val):
-                if val.tzinfo is not None:
-                    found_tz = True
-                else:
-                    found_naive = True
-                tz_out = convert_timezone(
-                    val.tzinfo,
-                    tz_out,
-                    found_naive,
-                    found_tz,
-                    utc_convert,
-                )
+                tz_out = state.process_datetime(val, tz_out, utc_convert)
                 iresult[i] = parse_pydatetime(val, &dts, utc_convert, creso=creso)
 
             elif PyDate_Check(val):
diff --git a/pandas/_libs/tslibs/conversion.pxd b/pandas/_libs/tslibs/conversion.pxd
index 3e5a79e833..36c57a8b9c 100644
--- a/pandas/_libs/tslibs/conversion.pxd
+++ b/pandas/_libs/tslibs/conversion.pxd
@@ -47,13 +47,6 @@ cpdef (int64_t, int) precision_from_unit(str unit, NPY_DATETIMEUNIT out_reso=*)
 
 cdef maybe_localize_tso(_TSObject obj, tzinfo tz, NPY_DATETIMEUNIT reso)
 
-cdef tzinfo convert_timezone(
-    tzinfo tz_in,
-    tzinfo tz_out,
-    bint found_naive,
-    bint found_tz,
-    bint utc_convert,
-)
 
 cdef int64_t parse_pydatetime(
     datetime val,
diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx
index a1c59489b3..0568576c83 100644
--- a/pandas/_libs/tslibs/conversion.pyx
+++ b/pandas/_libs/tslibs/conversion.pyx
@@ -673,59 +673,6 @@ cpdef inline datetime localize_pydatetime(datetime dt, tzinfo tz):
     return _localize_pydatetime(dt, tz)
 
 
-cdef tzinfo convert_timezone(
-    tzinfo tz_in,
-    tzinfo tz_out,
-    bint found_naive,
-    bint found_tz,
-    bint utc_convert,
-):
-    """
-    Validate that ``tz_in`` can be converted/localized to ``tz_out``.
-
-    Parameters
-    ----------
-    tz_in : tzinfo or None
-        Timezone info of element being processed.
-    tz_out : tzinfo or None
-        Timezone info of output.
-    found_naive : bool
-        Whether a timezone-naive element has been found so far.
-    found_tz : bool
-        Whether a timezone-aware element has been found so far.
-    utc_convert : bool
-        Whether to convert/localize to UTC.
-
-    Returns
-    -------
-    tz_info
-        Timezone info of output.
-
-    Raises
-    ------
-    ValueError
-        If ``tz_in`` can't be converted/localized to ``tz_out``.
-    """
-    if tz_in is not None:
-        if utc_convert:
-            pass
-        elif found_naive:
-            raise ValueError("Tz-aware datetime.datetime "
-                             "cannot be converted to "
-                             "datetime64 unless utc=True")
-        elif tz_out is not None and not tz_compare(tz_out, tz_in):
-            raise ValueError("Tz-aware datetime.datetime "
-                             "cannot be converted to "
-                             "datetime64 unless utc=True")
-        else:
-            tz_out = tz_in
-    else:
-        if found_tz and not utc_convert:
-            raise ValueError("Cannot mix tz-aware with "
-                             "tz-naive values")
-    return tz_out
-
-
 cdef int64_t parse_pydatetime(
     datetime val,
     npy_datetimestruct *dts,
diff --git a/pandas/_libs/tslibs/strptime.pxd b/pandas/_libs/tslibs/strptime.pxd
index 175195d436..d09612f4fb 100644
--- a/pandas/_libs/tslibs/strptime.pxd
+++ b/pandas/_libs/tslibs/strptime.pxd
@@ -1,4 +1,16 @@
+from cpython.datetime cimport (
+    datetime,
+    tzinfo,
+)
 from numpy cimport int64_t
 
 
 cdef bint parse_today_now(str val, int64_t* iresult, bint utc)
+
+
+cdef class DatetimeParseState:
+    cdef:
+        bint found_tz
+        bint found_naive
+
+    cdef tzinfo process_datetime(self, datetime dt, tzinfo tz, bint utc_convert)
diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx
index 717ec41a91..d1c6bab180 100644
--- a/pandas/_libs/tslibs/strptime.pyx
+++ b/pandas/_libs/tslibs/strptime.pyx
@@ -48,10 +48,7 @@ from numpy cimport (
 )
 
 from pandas._libs.missing cimport checknull_with_nat_and_na
-from pandas._libs.tslibs.conversion cimport (
-    convert_timezone,
-    get_datetime64_nanos,
-)
+from pandas._libs.tslibs.conversion cimport get_datetime64_nanos
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_nat_strings as nat_strings,
@@ -73,6 +70,7 @@ import_pandas_datetime()
 from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
 
 from pandas._libs.tslibs.timestamps cimport _Timestamp
+from pandas._libs.tslibs.timezones cimport tz_compare
 from pandas._libs.util cimport (
     is_float_object,
     is_integer_object,
@@ -156,6 +154,37 @@ cdef dict _parse_code_table = {"y": 0,
                                "u": 22}
 
 
+cdef class DatetimeParseState:
+    def __cinit__(self):
+        self.found_tz = False
+        self.found_naive = False
+
+    cdef tzinfo process_datetime(self, datetime dt, tzinfo tz, bint utc_convert):
+        if dt.tzinfo is not None:
+            self.found_tz = True
+        else:
+            self.found_naive = True
+
+        if dt.tzinfo is not None:
+            if utc_convert:
+                pass
+            elif self.found_naive:
+                raise ValueError("Tz-aware datetime.datetime "
+                                 "cannot be converted to "
+                                 "datetime64 unless utc=True")
+            elif tz is not None and not tz_compare(tz, dt.tzinfo):
+                raise ValueError("Tz-aware datetime.datetime "
+                                 "cannot be converted to "
+                                 "datetime64 unless utc=True")
+            else:
+                tz = dt.tzinfo
+        else:
+            if self.found_tz and not utc_convert:
+                raise ValueError("Cannot mix tz-aware with "
+                                 "tz-naive values")
+        return tz
+
+
 def array_strptime(
     ndarray[object] values,
     str fmt,
@@ -183,13 +212,12 @@ def array_strptime(
         bint is_raise = errors=="raise"
         bint is_ignore = errors=="ignore"
         bint is_coerce = errors=="coerce"
-        bint found_naive = False
-        bint found_tz = False
         tzinfo tz_out = None
         bint iso_format = format_is_iso(fmt)
         NPY_DATETIMEUNIT out_bestunit
         int out_local = 0, out_tzoffset = 0
         bint string_to_dts_succeeded = 0
+        DatetimeParseState state = DatetimeParseState()
 
     assert is_raise or is_ignore or is_coerce
 
@@ -276,17 +304,7 @@ def array_strptime(
                 iresult[i] = NPY_NAT
                 continue
             elif PyDateTime_Check(val):
-                if val.tzinfo is not None:
-                    found_tz = True
-                else:
-                    found_naive = True
-                tz_out = convert_timezone(
-                    val.tzinfo,
-                    tz_out,
-                    found_naive,
-                    found_tz,
-                    utc,
-                )
+                tz_out = state.process_datetime(val, tz_out, utc)
                 if isinstance(val, _Timestamp):
                     iresult[i] = val.tz_localize(None).as_unit("ns")._value
                 else:
diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py
index 138a3ee42f..04a8ad7ef0 100644
--- a/pandas/core/computation/pytables.py
+++ b/pandas/core/computation/pytables.py
@@ -217,7 +217,7 @@ class BinOp(ops.BinOp):
 
         kind = ensure_decoded(self.kind)
         meta = ensure_decoded(self.meta)
-        if kind in ("datetime64", "datetime"):
+        if kind == "datetime" or (kind and kind.startswith("datetime64")):
             if isinstance(v, (int, float)):
                 v = stringify(v)
             v = ensure_decoded(v)
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
index c0a27ecfec..91f7b2afec 100644
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -2152,7 +2152,6 @@ class IndexCol:
 
         val_kind = _ensure_decoded(self.kind)
         values = _maybe_convert(values, val_kind, encoding, errors)
-
         kwargs = {}
         kwargs["name"] = _ensure_decoded(self.index_name)
 
@@ -2577,7 +2576,7 @@ class DataCol(IndexCol):
         dtype = _ensure_decoded(dtype_name)
 
         # reverse converts
-        if dtype == "datetime64":
+        if dtype.startswith("datetime64"):
             # recreate with tz if indicated
             converted = _set_tz(converted, tz, coerce=True)
 
@@ -2870,7 +2869,9 @@ class GenericFixed(Fixed):
 
             def f(values, freq=None, tz=None):
                 # data are already in UTC, localize and convert if tz present
-                dta = DatetimeArray._simple_new(values.values, freq=freq)
+                dta = DatetimeArray._simple_new(
+                    values.values, dtype=values.dtype, freq=freq
+                )
                 result = DatetimeIndex._simple_new(dta, name=None)
                 if tz is not None:
                     result = result.tz_localize("UTC").tz_convert(tz)
@@ -2961,7 +2962,7 @@ class GenericFixed(Fixed):
             else:
                 ret = node[start:stop]
 
-            if dtype == "datetime64":
+            if dtype and dtype.startswith("datetime64"):
                 # reconstruct a timezone if indicated
                 tz = getattr(attrs, "tz", None)
                 ret = _set_tz(ret, tz, coerce=True)
@@ -3170,7 +3171,7 @@ class GenericFixed(Fixed):
 
         elif lib.is_np_dtype(value.dtype, "M"):
             self._handle.create_array(self.group, key, value.view("i8"))
-            getattr(self.group, key)._v_attrs.value_type = "datetime64"
+            getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
         elif isinstance(value.dtype, DatetimeTZDtype):
             # store as UTC
             # with a zone
@@ -3185,7 +3186,7 @@ class GenericFixed(Fixed):
             # error: Item "ExtensionArray" of "Union[Any, ExtensionArray]" has no
             # attribute "tz"
             node._v_attrs.tz = _get_tz(value.tz)  # type: ignore[union-attr]
-            node._v_attrs.value_type = "datetime64"
+            node._v_attrs.value_type = f"datetime64[{value.dtype.unit}]"
         elif lib.is_np_dtype(value.dtype, "m"):
             self._handle.create_array(self.group, key, value.view("i8"))
             getattr(self.group, key)._v_attrs.value_type = "timedelta64"
@@ -4689,7 +4690,6 @@ class AppendableFrameTable(AppendableTable):
         selection = Selection(self, where=where, start=start, stop=stop)
         # apply the selection filters & axis orderings
         df = self.process_axes(df, selection=selection, columns=columns)
-
         return df
 
 
@@ -4932,11 +4932,12 @@ def _set_tz(
         #  call below (which returns an ndarray).  So we are only non-lossy
         #  if `tz` matches `values.tz`.
         assert values.tz is None or values.tz == tz
+        if values.tz is not None:
+            return values
 
     if tz is not None:
         if isinstance(values, DatetimeIndex):
             name = values.name
-            values = values.asi8
         else:
             name = None
             values = values.ravel()
@@ -5019,8 +5020,12 @@ def _convert_index(name: str, index: Index, encoding: str, errors: str) -> Index
 def _unconvert_index(data, kind: str, encoding: str, errors: str) -> np.ndarray | Index:
     index: Index | np.ndarray
 
-    if kind == "datetime64":
-        index = DatetimeIndex(data)
+    if kind.startswith("datetime64"):
+        if kind == "datetime64":
+            # created before we stored resolution information
+            index = DatetimeIndex(data)
+        else:
+            index = DatetimeIndex(data.view(kind))
     elif kind == "timedelta64":
         index = TimedeltaIndex(data)
     elif kind == "date":
@@ -5194,6 +5199,8 @@ def _maybe_convert(values: np.ndarray, val_kind: str, encoding: str, errors: str
 def _get_converter(kind: str, encoding: str, errors: str):
     if kind == "datetime64":
         return lambda x: np.asarray(x, dtype="M8[ns]")
+    elif "datetime64" in kind:
+        return lambda x: np.asarray(x, dtype=kind)
     elif kind == "string":
         return lambda x: _unconvert_string_array(
             x, nan_rep=None, encoding=encoding, errors=errors
@@ -5203,7 +5210,7 @@ def _get_converter(kind: str, encoding: str, errors: str):
 
 
 def _need_convert(kind: str) -> bool:
-    if kind in ("datetime64", "string"):
+    if kind in ("datetime64", "string") or "datetime64" in kind:
         return True
     return False
 
@@ -5248,7 +5255,7 @@ def _dtype_to_kind(dtype_str: str) -> str:
     elif dtype_str.startswith(("int", "uint")):
         kind = "integer"
     elif dtype_str.startswith("datetime64"):
-        kind = "datetime64"
+        kind = dtype_str
     elif dtype_str.startswith("timedelta"):
         kind = "timedelta64"
     elif dtype_str.startswith("bool"):
@@ -5273,8 +5280,11 @@ def _get_data_and_dtype_name(data: ArrayLike):
     if isinstance(data, Categorical):
         data = data.codes
 
-    # For datetime64tz we need to drop the TZ in tests TODO: why?
-    dtype_name = data.dtype.name.split("[")[0]
+    if isinstance(data.dtype, DatetimeTZDtype):
+        # For datetime64tz we need to drop the TZ in tests TODO: why?
+        dtype_name = f"datetime64[{data.dtype.unit}]"
+    else:
+        dtype_name = data.dtype.name
 
     if data.dtype.kind in "mM":
         data = np.asarray(data.view("i8"))


Test patch:
diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py
index 3ad9abc138..dace843559 100644
--- a/pandas/tests/io/pytables/test_append.py
+++ b/pandas/tests/io/pytables/test_append.py
@@ -772,7 +772,7 @@ because its data contents are not [string] but [mixed] object dtype"""
             "dtype->bytes24,kind->string,shape->(1, 30)] "
             "vs current table "
             "[name->values_block_1,cname->values_block_1,"
-            "dtype->datetime64,kind->datetime64,shape->None]"
+            "dtype->datetime64[s],kind->datetime64[s],shape->None]"
         )
         with pytest.raises(ValueError, match=msg):
             store.append("df", df)
diff --git a/pandas/tests/io/pytables/test_errors.py b/pandas/tests/io/pytables/test_errors.py
index 4c1f766787..adf42cc7e8 100644
--- a/pandas/tests/io/pytables/test_errors.py
+++ b/pandas/tests/io/pytables/test_errors.py
@@ -49,7 +49,7 @@ def test_table_index_incompatible_dtypes(setup_path):
 
     with ensure_clean_store(setup_path) as store:
         store.put("frame", df1, format="table")
-        msg = re.escape("incompatible kind in col [integer - datetime64]")
+        msg = re.escape("incompatible kind in col [integer - datetime64[ns]]")
         with pytest.raises(TypeError, match=msg):
             store.put("frame", df2, format="table", append=True)
 
diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py
index f031ac46c6..b61b6f0772 100644
--- a/pandas/tests/io/pytables/test_store.py
+++ b/pandas/tests/io/pytables/test_store.py
@@ -541,16 +541,22 @@ def test_store_index_name(setup_path):
         tm.assert_frame_equal(recons, df)
 
 
+@pytest.mark.parametrize("tz", [None, "US/Pacific"])
+@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
 @pytest.mark.parametrize("table_format", ["table", "fixed"])
-def test_store_index_name_numpy_str(tmp_path, table_format, setup_path):
+def test_store_index_name_numpy_str(tmp_path, table_format, setup_path, unit, tz):
     # GH #13492
     idx = Index(
         pd.to_datetime([dt.date(2000, 1, 1), dt.date(2000, 1, 2)]),
         name="cols\u05d2",
-    )
-    idx1 = Index(
-        pd.to_datetime([dt.date(2010, 1, 1), dt.date(2010, 1, 2)]),
-        name="rows\u05d0",
+    ).tz_localize(tz)
+    idx1 = (
+        Index(
+            pd.to_datetime([dt.date(2010, 1, 1), dt.date(2010, 1, 2)]),
+            name="rows\u05d0",
+        )
+        .as_unit(unit)
+        .tz_localize(tz)
     )
     df = DataFrame(np.arange(4).reshape(2, 2), columns=idx, index=idx1)