Repository: pandas-dev/pandas
Pull request #55813 (2023-11-03 20:26:11)

BUG: DatetimeIndex with dayfirst/yearfirst and tz
- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [x] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [x] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

surfaced while implementing #55564


Patch:
diff --git a/doc/source/user_guide/basics.rst b/doc/source/user_guide/basics.rst
index 33e9f9cabb..17ff5d821e 100644
--- a/doc/source/user_guide/basics.rst
+++ b/doc/source/user_guide/basics.rst
@@ -2007,7 +2007,7 @@ documentation sections for more on each type.
 |                                                 |                           |                    |                               | ``'Int64'``, ``'UInt8'``, ``'UInt16'``,|
 |                                                 |                           |                    |                               | ``'UInt32'``, ``'UInt64'``             |
 +-------------------------------------------------+---------------------------+--------------------+-------------------------------+----------------------------------------+
-| ``nullable float``                              | :class:`Float64Dtype`, ...| (none)             | :class:`arrays.FloatingArray` | ``'Float32'``, ``'Float64'``           |
+| :ref:`nullable float <api.arrays.float_na>`     | :class:`Float64Dtype`, ...| (none)             | :class:`arrays.FloatingArray` | ``'Float32'``, ``'Float64'``           |
 +-------------------------------------------------+---------------------------+--------------------+-------------------------------+----------------------------------------+
 | :ref:`Strings <text>`                           | :class:`StringDtype`      | :class:`str`       | :class:`arrays.StringArray`   | ``'string'``                           |
 +-------------------------------------------------+---------------------------+--------------------+-------------------------------+----------------------------------------+
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index 65532c4cbd..0b1eb1760b 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -343,6 +343,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
+- Bug in :class:`DatetimeIndex` construction when passing both a ``tz`` and either ``dayfirst`` or ``yearfirst`` ignoring dayfirst/yearfirst (:issue:`55813`)
 - Bug in :class:`DatetimeIndex` when passing an object-dtype ndarray of float objects and a ``tz`` incorrectly localizing the result (:issue:`55780`)
 - Bug in :func:`concat` raising ``AttributeError`` when concatenating all-NA DataFrame with :class:`DatetimeTZDtype` dtype DataFrame. (:issue:`52093`)
 - Bug in :func:`to_datetime` and :class:`DatetimeIndex` when passing a list of mixed-string-and-numeric types incorrectly raising (:issue:`55780`)
diff --git a/pandas/_libs/tslib.pyi b/pandas/_libs/tslib.pyi
index a803ea0692..5a340c1d88 100644
--- a/pandas/_libs/tslib.pyi
+++ b/pandas/_libs/tslib.pyi
@@ -29,5 +29,9 @@ def array_to_datetime(
 # returned ndarray may be object dtype or datetime64[ns]
 
 def array_to_datetime_with_tz(
-    values: npt.NDArray[np.object_], tz: tzinfo, creso: int
+    values: npt.NDArray[np.object_],
+    tz: tzinfo,
+    dayfirst: bool,
+    yearfirst: bool,
+    creso: int,
 ) -> npt.NDArray[np.int64]: ...
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
index 3c694ab26d..da9d65bcfc 100644
--- a/pandas/_libs/tslib.pyx
+++ b/pandas/_libs/tslib.pyx
@@ -61,6 +61,7 @@ from pandas._libs.tslibs.conversion cimport (
     _TSObject,
     cast_from_unit,
     convert_str_to_tsobject,
+    convert_to_tsobject,
     get_datetime64_nanos,
     parse_pydatetime,
 )
@@ -673,7 +674,9 @@ cdef _array_to_datetime_object(
     return oresult_nd, None
 
 
-def array_to_datetime_with_tz(ndarray values, tzinfo tz, NPY_DATETIMEUNIT creso):
+def array_to_datetime_with_tz(
+    ndarray values, tzinfo tz, bint dayfirst, bint yearfirst, NPY_DATETIMEUNIT creso
+):
     """
     Vectorized analogue to pd.Timestamp(value, tz=tz)
 
@@ -689,7 +692,7 @@ def array_to_datetime_with_tz(ndarray values, tzinfo tz, NPY_DATETIMEUNIT creso)
         Py_ssize_t i, n = values.size
         object item
         int64_t ival
-        datetime ts
+        _TSObject tsobj
 
     for i in range(n):
         # Analogous to `item = values[i]`
@@ -700,17 +703,12 @@ def array_to_datetime_with_tz(ndarray values, tzinfo tz, NPY_DATETIMEUNIT creso)
             ival = NPY_NAT
 
         else:
-            if PyDateTime_Check(item) and item.tzinfo is not None:
-                # We can't call Timestamp constructor with a tz arg, have to
-                #  do 2-step
-                ts = Timestamp(item).tz_convert(tz)
-            else:
-                ts = Timestamp(item, tz=tz)
-            if ts is NaT:
-                ival = NPY_NAT
-            else:
-                ts = (<_Timestamp>ts)._as_creso(creso)
-                ival = ts._value
+            tsobj = convert_to_tsobject(
+                item, tz=tz, unit="ns", dayfirst=dayfirst, yearfirst=yearfirst, nanos=0
+            )
+            if tsobj.value != NPY_NAT:
+                tsobj.ensure_reso(creso, item, round_ok=True)
+            ival = tsobj.value
 
         # Analogous to: result[i] = ival
         (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
diff --git a/pandas/_libs/tslibs/conversion.pxd b/pandas/_libs/tslibs/conversion.pxd
index ec743dbf98..e97f4900d2 100644
--- a/pandas/_libs/tslibs/conversion.pxd
+++ b/pandas/_libs/tslibs/conversion.pxd
@@ -24,7 +24,9 @@ cdef class _TSObject:
         bint fold
         NPY_DATETIMEUNIT creso
 
-    cdef int64_t ensure_reso(self, NPY_DATETIMEUNIT creso, str val=*) except? -1
+    cdef int64_t ensure_reso(
+        self, NPY_DATETIMEUNIT creso, val=*, bint round_ok=*
+    ) except? -1
 
 
 cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx
index 68aaaafd20..a2bc6d4d52 100644
--- a/pandas/_libs/tslibs/conversion.pyx
+++ b/pandas/_libs/tslibs/conversion.pyx
@@ -235,10 +235,14 @@ cdef class _TSObject:
         self.fold = 0
         self.creso = NPY_FR_ns  # default value
 
-    cdef int64_t ensure_reso(self, NPY_DATETIMEUNIT creso, str val=None) except? -1:
+    cdef int64_t ensure_reso(
+        self, NPY_DATETIMEUNIT creso, val=None, bint round_ok=False
+    ) except? -1:
         if self.creso != creso:
             try:
-                self.value = convert_reso(self.value, self.creso, creso, False)
+                self.value = convert_reso(
+                    self.value, self.creso, creso, round_ok=round_ok
+                )
             except OverflowError as err:
                 if val is not None:
                     raise OutOfBoundsDatetime(
@@ -283,6 +287,11 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
         obj.value = get_datetime64_nanos(ts, reso)
         if obj.value != NPY_NAT:
             pandas_datetime_to_datetimestruct(obj.value, reso, &obj.dts)
+            if tz is not None:
+                # GH#24559, GH#42288 We treat np.datetime64 objects as *wall* times
+                obj.value = tz_localize_to_utc_single(
+                    obj.value, tz, ambiguous="raise", nonexistent=None, creso=reso
+                )
     elif is_integer_object(ts):
         try:
             ts = <int64_t>ts
diff --git a/pandas/_libs/tslibs/strptime.pxd b/pandas/_libs/tslibs/strptime.pxd
index 64db2b59df..fd8cafeaef 100644
--- a/pandas/_libs/tslibs/strptime.pxd
+++ b/pandas/_libs/tslibs/strptime.pxd
@@ -7,7 +7,9 @@ from numpy cimport int64_t
 from pandas._libs.tslibs.np_datetime cimport NPY_DATETIMEUNIT
 
 
-cdef bint parse_today_now(str val, int64_t* iresult, bint utc, NPY_DATETIMEUNIT creso)
+cdef bint parse_today_now(
+    str val, int64_t* iresult, bint utc, NPY_DATETIMEUNIT creso, bint infer_reso=*
+)
 
 
 cdef class DatetimeParseState:
diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx
index 6b744ce4c8..fc44d3aad4 100644
--- a/pandas/_libs/tslibs/strptime.pyx
+++ b/pandas/_libs/tslibs/strptime.pyx
@@ -117,7 +117,7 @@ def _test_format_is_iso(f: str) -> bool:
 
 
 cdef bint parse_today_now(
-    str val, int64_t* iresult, bint utc, NPY_DATETIMEUNIT creso
+    str val, int64_t* iresult, bint utc, NPY_DATETIMEUNIT creso, bint infer_reso = False
 ):
     # We delay this check for as long as possible
     # because it catches relatively rare cases
@@ -125,6 +125,8 @@ cdef bint parse_today_now(
         _Timestamp ts
 
     if val == "now":
+        if infer_reso:
+            creso = NPY_DATETIMEUNIT.NPY_FR_us
         if utc:
             ts = <_Timestamp>Timestamp.utcnow()
             iresult[0] = ts._as_creso(creso)._value
@@ -135,6 +137,8 @@ cdef bint parse_today_now(
             iresult[0] = ts._as_creso(creso)._value
         return True
     elif val == "today":
+        if infer_reso:
+            creso = NPY_DATETIMEUNIT.NPY_FR_us
         ts = <_Timestamp>Timestamp.today()
         iresult[0] = ts._as_creso(creso)._value
         return True
@@ -348,27 +352,33 @@ def array_strptime(
                 else:
                     item_reso = NPY_DATETIMEUNIT.NPY_FR_us
                 state.update_creso(item_reso)
+                if infer_reso:
+                    creso = state.creso
                 tz_out = state.process_datetime(val, tz_out, utc)
                 if isinstance(val, _Timestamp):
-                    val = (<_Timestamp>val)._as_creso(state.creso)
+                    val = (<_Timestamp>val)._as_creso(creso)
                     iresult[i] = val.tz_localize(None)._value
                 else:
                     iresult[i] = pydatetime_to_dt64(
-                        val.replace(tzinfo=None), &dts, reso=state.creso
+                        val.replace(tzinfo=None), &dts, reso=creso
                     )
-                    check_dts_bounds(&dts, state.creso)
+                    check_dts_bounds(&dts, creso)
                 result_timezone[i] = val.tzinfo
                 continue
             elif PyDate_Check(val):
                 item_reso = NPY_DATETIMEUNIT.NPY_FR_s
                 state.update_creso(item_reso)
-                iresult[i] = pydate_to_dt64(val, &dts, reso=state.creso)
-                check_dts_bounds(&dts, state.creso)
+                if infer_reso:
+                    creso = state.creso
+                iresult[i] = pydate_to_dt64(val, &dts, reso=creso)
+                check_dts_bounds(&dts, creso)
                 continue
             elif is_datetime64_object(val):
                 item_reso = get_supported_reso(get_datetime64_unit(val))
                 state.update_creso(item_reso)
-                iresult[i] = get_datetime64_nanos(val, state.creso)
+                if infer_reso:
+                    creso = state.creso
+                iresult[i] = get_datetime64_nanos(val, creso)
                 continue
             elif (
                     (is_integer_object(val) or is_float_object(val))
@@ -394,7 +404,9 @@ def array_strptime(
                 # where we left off
                 item_reso = get_supported_reso(out_bestunit)
                 state.update_creso(item_reso)
-                value = npy_datetimestruct_to_datetime(state.creso, &dts)
+                if infer_reso:
+                    creso = state.creso
+                value = npy_datetimestruct_to_datetime(creso, &dts)
                 if out_local == 1:
                     # Store the out_tzoffset in seconds
                     # since we store the total_seconds of
@@ -404,12 +416,14 @@ def array_strptime(
                     out_local = 0
                     out_tzoffset = 0
                 iresult[i] = value
-                check_dts_bounds(&dts)
+                check_dts_bounds(&dts, creso)
                 continue
 
-            if parse_today_now(val, &iresult[i], utc, state.creso):
+            if parse_today_now(val, &iresult[i], utc, creso, infer_reso=infer_reso):
                 item_reso = NPY_DATETIMEUNIT.NPY_FR_us
                 state.update_creso(item_reso)
+                if infer_reso:
+                    creso = state.creso
                 continue
 
             # Some ISO formats can't be parsed by string_to_dts
@@ -424,8 +438,10 @@ def array_strptime(
                 val, fmt, exact, format_regex, locale_time, &dts, &item_reso
             )
             state.update_creso(item_reso)
-            iresult[i] = npy_datetimestruct_to_datetime(state.creso, &dts)
-            check_dts_bounds(&dts)
+            if infer_reso:
+                creso = state.creso
+            iresult[i] = npy_datetimestruct_to_datetime(creso, &dts)
+            check_dts_bounds(&dts, creso)
             result_timezone[i] = tz
 
         except (ValueError, OutOfBoundsDatetime) as ex:
diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx
index 01aea60268..d0c3c5c23b 100644
--- a/pandas/_libs/tslibs/timestamps.pyx
+++ b/pandas/_libs/tslibs/timestamps.pyx
@@ -1873,10 +1873,6 @@ class Timestamp(_Timestamp):
                              "the tz parameter. Use tz_convert instead.")
 
         tzobj = maybe_get_tz(tz)
-        if tzobj is not None and is_datetime64_object(ts_input):
-            # GH#24559, GH#42288 As of 2.0 we treat datetime64 as
-            #  wall-time (consistent with DatetimeIndex)
-            return cls(ts_input).tz_localize(tzobj)
 
         if nanosecond is None:
             nanosecond = 0
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 968b64e2c3..86402fe05e 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -2240,10 +2240,13 @@ def _sequence_to_dt64(
         if lib.infer_dtype(data, skipna=False) == "integer":
             data = data.astype(np.int64)
         elif tz is not None and ambiguous == "raise":
-            # TODO: yearfirst/dayfirst/etc?
             obj_data = np.asarray(data, dtype=object)
             i8data = tslib.array_to_datetime_with_tz(
-                obj_data, tz, abbrev_to_npy_unit(out_unit)
+                obj_data,
+                tz=tz,
+                dayfirst=dayfirst,
+                yearfirst=yearfirst,
+                creso=abbrev_to_npy_unit(out_unit),
             )
             return i8data.view(out_dtype), tz, None
         else:


Test patch:
diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py
index ab43287885..f717165d11 100644
--- a/pandas/tests/indexes/datetimes/test_constructors.py
+++ b/pandas/tests/indexes/datetimes/test_constructors.py
@@ -1247,6 +1247,21 @@ class TestTimeSeries:
         assert len(idx1) == len(idx2)
         assert idx1.freq == idx2.freq
 
+    def test_dti_constructor_object_dtype_dayfirst_yearfirst_with_tz(self):
+        # GH#55813
+        val = "5/10/16"
+
+        dfirst = Timestamp(2016, 10, 5, tz="US/Pacific")
+        yfirst = Timestamp(2005, 10, 16, tz="US/Pacific")
+
+        result1 = DatetimeIndex([val], tz="US/Pacific", dayfirst=True)
+        expected1 = DatetimeIndex([dfirst])
+        tm.assert_index_equal(result1, expected1)
+
+        result2 = DatetimeIndex([val], tz="US/Pacific", yearfirst=True)
+        expected2 = DatetimeIndex([yfirst])
+        tm.assert_index_equal(result2, expected2)
+
     def test_pass_datetimeindex_to_index(self):
         # Bugs in #1396
         rng = date_range("1/1/2000", "3/1/2000")
diff --git a/pandas/tests/tslibs/test_strptime.py b/pandas/tests/tslibs/test_strptime.py
index 0992eecf0e..ce45bdd10b 100644
--- a/pandas/tests/tslibs/test_strptime.py
+++ b/pandas/tests/tslibs/test_strptime.py
@@ -59,3 +59,39 @@ class TestArrayStrptimeResolutionInference:
         fmt = "ISO8601"
         res, _ = array_strptime(arr, fmt=fmt, utc=False, creso=creso_infer)
         tm.assert_numpy_array_equal(res, expected)
+
+    def test_array_strptime_resolution_todaynow(self):
+        # specifically case where today/now is the *first* item
+        vals = np.array(["today", np.datetime64("2017-01-01", "us")], dtype=object)
+
+        now = Timestamp("now").asm8
+        res, _ = array_strptime(vals, fmt="%Y-%m-%d", utc=False, creso=creso_infer)
+        res2, _ = array_strptime(
+            vals[::-1], fmt="%Y-%m-%d", utc=False, creso=creso_infer
+        )
+
+        # 1s is an arbitrary cutoff for call overhead; in local testing the
+        #  actual difference is about 250us
+        tolerance = np.timedelta64(1, "s")
+
+        assert res.dtype == "M8[us]"
+        assert abs(res[0] - now) < tolerance
+        assert res[1] == vals[1]
+
+        assert res2.dtype == "M8[us]"
+        assert abs(res2[1] - now) < tolerance * 2
+        assert res2[0] == vals[1]
+
+    def test_array_strptime_str_outside_nano_range(self):
+        vals = np.array(["2401-09-15"], dtype=object)
+        expected = np.array(["2401-09-15"], dtype="M8[s]")
+        fmt = "ISO8601"
+        res, _ = array_strptime(vals, fmt=fmt, creso=creso_infer)
+        tm.assert_numpy_array_equal(res, expected)
+
+        # non-iso -> different path
+        vals2 = np.array(["Sep 15, 2401"], dtype=object)
+        expected2 = np.array(["2401-09-15"], dtype="M8[s]")
+        fmt2 = "%b %d, %Y"
+        res2, _ = array_strptime(vals2, fmt=fmt2, creso=creso_infer)
+        tm.assert_numpy_array_equal(res2, expected2)