Repository: pandas-dev/pandas
Pull request #55768 (2023-10-30 21:04:23)

BUG: OutOfBoundsDatetime with non-nano dt64tz dtype
- [x] closes #54620 (Replace xxxx with the GitHub issue number)
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [x] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

Updated version of #54620.

Progress towards #55564


Patch:
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index d9909b0dbf..99b6310a80 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -330,6 +330,7 @@ Datetimelike
 - Bug in addition or subtraction of :class:`BusinessDay` offset with ``offset`` attribute to non-nanosecond :class:`Index`, :class:`Series`, or :class:`DataFrame` column giving incorrect results (:issue:`55608`)
 - Bug in addition or subtraction of :class:`DateOffset` objects with microsecond components to ``datetime64`` :class:`Index`, :class:`Series`, or :class:`DataFrame` columns with non-nanosecond resolution (:issue:`55595`)
 - Bug in addition or subtraction of very large :class:`Tick` objects with :class:`Timestamp` or :class:`Timedelta` objects raising ``OverflowError`` instead of ``OutOfBoundsTimedelta`` (:issue:`55503`)
+- Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond :class:`DatetimeTZDtype` and inputs that would be out of bounds with nanosecond resolution incorrectly raising ``OutOfBoundsDatetime`` (:issue:`54620`)
 - Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond ``datetime64`` dtype and inputs that would be out of bounds for a ``datetime64[ns]`` incorrectly raising ``OutOfBoundsDatetime`` (:issue:`55756`)
 -
 
diff --git a/pandas/_libs/tslib.pyi b/pandas/_libs/tslib.pyi
index 7f95bfc717..a803ea0692 100644
--- a/pandas/_libs/tslib.pyi
+++ b/pandas/_libs/tslib.pyi
@@ -29,5 +29,5 @@ def array_to_datetime(
 # returned ndarray may be object dtype or datetime64[ns]
 
 def array_to_datetime_with_tz(
-    values: npt.NDArray[np.object_], tz: tzinfo
+    values: npt.NDArray[np.object_], tz: tzinfo, creso: int
 ) -> npt.NDArray[np.int64]: ...
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
index d2eeea78ee..bb96a89f7d 100644
--- a/pandas/_libs/tslib.pyx
+++ b/pandas/_libs/tslib.pyx
@@ -671,7 +671,7 @@ cdef _array_to_datetime_object(
     return oresult_nd, None
 
 
-def array_to_datetime_with_tz(ndarray values, tzinfo tz):
+def array_to_datetime_with_tz(ndarray values, tzinfo tz, NPY_DATETIMEUNIT creso):
     """
     Vectorized analogue to pd.Timestamp(value, tz=tz)
 
@@ -707,7 +707,7 @@ def array_to_datetime_with_tz(ndarray values, tzinfo tz):
                 else:
                     # datetime64, tznaive pydatetime, int, float
                     ts = ts.tz_localize(tz)
-                ts = ts.as_unit("ns")
+                ts = (<_Timestamp>ts)._as_creso(creso)
                 ival = ts._value
 
         # Analogous to: result[i] = ival
diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx
index c23bccdea3..df3a2e3ecd 100644
--- a/pandas/_libs/tslibs/offsets.pyx
+++ b/pandas/_libs/tslibs/offsets.pyx
@@ -4616,7 +4616,28 @@ _lite_rule_alias = {
     "ns": "ns",
 }
 
-_dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s", "me", "qe"}
+_dont_uppercase = {
+    "h",
+    "bh",
+    "cbh",
+    "MS",
+    "ms",
+    "s",
+    "me",
+    "qe",
+    "qe-dec",
+    "qe-jan",
+    "qe-feb",
+    "qe-mar",
+    "qe-apr",
+    "qe-may",
+    "qe-jun",
+    "qe-jul",
+    "qe-aug",
+    "qe-sep",
+    "qe-oct",
+    "qe-nov",
+}
 
 
 INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"
@@ -4635,7 +4656,7 @@ def _get_offset(name: str) -> BaseOffset:
     --------
     _get_offset('EOM') --> BMonthEnd(1)
     """
-    if name not in _dont_uppercase:
+    if name.lower() not in _dont_uppercase:
         name = name.upper()
         name = _lite_rule_alias.get(name, name)
         name = _lite_rule_alias.get(name.lower(), name)
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 3fff5cb2aa..968b64e2c3 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -355,7 +355,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):  # type: ignore[misc]
                 # DatetimeTZDtype
                 unit = dtype.unit
 
-        subarr, tz, inferred_freq = _sequence_to_dt64ns(
+        subarr, tz, inferred_freq = _sequence_to_dt64(
             data,
             copy=copy,
             tz=tz,
@@ -2179,7 +2179,7 @@ default 'raise'
 # Constructor Helpers
 
 
-def _sequence_to_dt64ns(
+def _sequence_to_dt64(
     data,
     *,
     copy: bool = False,
@@ -2205,7 +2205,8 @@ def _sequence_to_dt64ns(
     Returns
     -------
     result : numpy.ndarray
-        The sequence converted to a numpy array with dtype ``datetime64[ns]``.
+        The sequence converted to a numpy array with dtype ``datetime64[unit]``.
+        Where `unit` is "ns" unless specified otherwise by `out_unit`.
     tz : tzinfo or None
         Either the user-provided tzinfo or one inferred from the data.
     inferred_freq : Tick or None
@@ -2228,9 +2229,9 @@ def _sequence_to_dt64ns(
     data, copy = maybe_convert_dtype(data, copy, tz=tz)
     data_dtype = getattr(data, "dtype", None)
 
-    out_dtype = DT64NS_DTYPE
-    if out_unit is not None:
-        out_dtype = np.dtype(f"M8[{out_unit}]")
+    if out_unit is None:
+        out_unit = "ns"
+    out_dtype = np.dtype(f"M8[{out_unit}]")
 
     if data_dtype == object or is_string_dtype(data_dtype):
         # TODO: We do not have tests specific to string-dtypes,
@@ -2241,8 +2242,10 @@ def _sequence_to_dt64ns(
         elif tz is not None and ambiguous == "raise":
             # TODO: yearfirst/dayfirst/etc?
             obj_data = np.asarray(data, dtype=object)
-            i8data = tslib.array_to_datetime_with_tz(obj_data, tz)
-            return i8data.view(DT64NS_DTYPE), tz, None
+            i8data = tslib.array_to_datetime_with_tz(
+                obj_data, tz, abbrev_to_npy_unit(out_unit)
+            )
+            return i8data.view(out_dtype), tz, None
         else:
             # data comes back here as either i8 to denote UTC timestamps
             #  or M8[ns] to denote wall times
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
index 7c2247101e..45139d8461 100644
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -9309,8 +9309,6 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
         bucket ``2000-01-01 00:03:00`` contains the value 3, but the summed
         value in the resampled bucket with the label ``2000-01-01 00:03:00``
         does not include 3 (if it did, the summed value would be 6, not 3).
-        To include this value close the right side of the bin interval as
-        illustrated in the example below this one.
 
         >>> series.resample('3min', label='right').sum()
         2000-01-01 00:03:00     3
@@ -9318,8 +9316,8 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
         2000-01-01 00:09:00    21
         Freq: 3min, dtype: int64
 
-        Downsample the series into 3 minute bins as above, but close the right
-        side of the bin interval.
+        To include this value close the right side of the bin interval,
+        as shown below.
 
         >>> series.resample('3min', label='right', closed='right').sum()
         2000-01-01 00:00:00     0


Test patch:
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index d00dfc87a0..0b6843c5a7 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -69,7 +69,7 @@ jobs:
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
-          - name: "Copy-on-Write (warnings)"
+          - name: "Copy-on-Write 3.11 (warnings)"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "warn"
@@ -98,7 +98,7 @@ jobs:
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
       cancel-in-progress: true
 
     services:
diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py
index ef86e7800d..90ddc9b5f6 100644
--- a/pandas/tests/indexes/datetimes/test_constructors.py
+++ b/pandas/tests/indexes/datetimes/test_constructors.py
@@ -1013,16 +1013,19 @@ class TestDatetimeIndex:
         dr2 = DatetimeIndex(list(dr), name="foo", freq="D")
         tm.assert_index_equal(dr, dr2)
 
-    def test_dti_constructor_with_non_nano_dtype(self):
-        # GH#55756
+    @pytest.mark.parametrize("tz", [None, "UTC", "US/Pacific"])
+    def test_dti_constructor_with_non_nano_dtype(self, tz):
+        # GH#55756, GH#54620
         ts = Timestamp("2999-01-01")
         dtype = "M8[us]"
+        if tz is not None:
+            dtype = f"M8[us, {tz}]"
         # NB: the 2500 is interpreted as nanoseconds and rounded *down*
         # to 2 microseconds
         vals = [ts, "2999-01-02 03:04:05.678910", 2500]
         result = DatetimeIndex(vals, dtype=dtype)
-        exp_arr = np.array([ts.asm8, vals[1], 2], dtype=dtype)
-        expected = DatetimeIndex(exp_arr, dtype=dtype)
+        exp_arr = np.array([ts.asm8, vals[1], 2], dtype="M8[us]")
+        expected = DatetimeIndex(exp_arr, dtype="M8[us]").tz_localize(tz)
         tm.assert_index_equal(result, expected)
 
         result2 = DatetimeIndex(np.array(vals, dtype=object), dtype=dtype)
diff --git a/pandas/tests/resample/test_period_index.py b/pandas/tests/resample/test_period_index.py
index e768d9266a..f4032c1257 100644
--- a/pandas/tests/resample/test_period_index.py
+++ b/pandas/tests/resample/test_period_index.py
@@ -134,7 +134,7 @@ class TestPeriodIndex:
         "rule,expected_error_msg",
         [
             ("y-dec", "<YearEnd: month=12>"),
-            ("qe-mar", "<QuarterEnd: startingMonth=3>"),
+            ("Q-MAR", "<QuarterEnd: startingMonth=3>"),
             ("M", "<MonthEnd>"),
             ("w-thu", "<Week: weekday=3>"),
         ],
diff --git a/pandas/tests/series/methods/test_astype.py b/pandas/tests/series/methods/test_astype.py
index edd3062f7d..2434290616 100644
--- a/pandas/tests/series/methods/test_astype.py
+++ b/pandas/tests/series/methods/test_astype.py
@@ -107,18 +107,21 @@ class TestAstypeAPI:
 
 
 class TestAstype:
-    def test_astype_object_to_dt64_non_nano(self):
-        # GH#55756
+    @pytest.mark.parametrize("tz", [None, "UTC", "US/Pacific"])
+    def test_astype_object_to_dt64_non_nano(self, tz):
+        # GH#55756, GH#54620
         ts = Timestamp("2999-01-01")
         dtype = "M8[us]"
+        if tz is not None:
+            dtype = f"M8[us, {tz}]"
         # NB: the 2500 is interpreted as nanoseconds and rounded *down*
         # to 2 microseconds
         vals = [ts, "2999-01-02 03:04:05.678910", 2500]
         ser = Series(vals, dtype=object)
         result = ser.astype(dtype)
 
-        exp_arr = np.array([ts.asm8, vals[1], 2], dtype=dtype)
-        expected = Series(exp_arr, dtype=dtype)
+        exp_arr = np.array([ts.asm8, vals[1], 2], dtype="M8[us]")
+        expected = Series(exp_arr, dtype="M8[us]").dt.tz_localize(tz)
         tm.assert_series_equal(result, expected)
 
     def test_astype_mixed_object_to_dt64tz(self):