Repository: pandas-dev/pandas
Pull request #57326 (2024-02-09 22:07:08)

Backport PR #57250 on branch 2.2.x (REGR/DOC: pd.concat should special case DatetimeIndex to sort even when sort=False)
Backport PR #57250: REGR/DOC: pd.concat should special case DatetimeIndex to sort even when sort=False


Patch:
diff --git a/doc/source/whatsnew/v2.2.1.rst b/doc/source/whatsnew/v2.2.1.rst
index 883627bd4b..69e14a9028 100644
--- a/doc/source/whatsnew/v2.2.1.rst
+++ b/doc/source/whatsnew/v2.2.1.rst
@@ -15,14 +15,18 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed memory leak in :func:`read_csv` (:issue:`57039`)
 - Fixed performance regression in :meth:`Series.combine_first` (:issue:`55845`)
+- Fixed regression in :func:`concat` changing long-standing behavior that always sorted the non-concatenation axis when the axis was a :class:`DatetimeIndex` (:issue:`57006`)
 - Fixed regression in :func:`merge_ordered` raising ``TypeError`` for ``fill_method="ffill"`` and ``how="left"`` (:issue:`57010`)
 - Fixed regression in :func:`wide_to_long` raising an ``AttributeError`` for string columns (:issue:`57066`)
+- Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` ignoring the ``skipna`` argument (:issue:`57040`)
+- Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` where values containing the minimum or maximum value for the dtype could produce incorrect results (:issue:`57040`)
 - Fixed regression in :meth:`DataFrame.loc` raising ``IndexError`` for non-unique, masked dtype indexes where result has more than 10,000 rows (:issue:`57027`)
 - Fixed regression in :meth:`DataFrame.sort_index` not producing a stable sort for a index with duplicates (:issue:`57151`)
 - Fixed regression in :meth:`DataFrame.to_dict` with ``orient='list'`` and datetime or timedelta types returning integers (:issue:`54824`)
 - Fixed regression in :meth:`DataFrame.to_json` converting nullable integers to floats (:issue:`57224`)
 - Fixed regression in :meth:`DataFrameGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmax` ignoring the ``skipna`` argument (:issue:`57040`)
 - Fixed regression in :meth:`DataFrameGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmax` where values containing the minimum or maximum value for the dtype could produce incorrect results (:issue:`57040`)
+- Fixed regression in :meth:`ExtensionArray.to_numpy` raising for non-numeric masked dtypes (:issue:`56991`)
 - Fixed regression in :meth:`Index.join` raising ``TypeError`` when joining an empty index to a non-empty index containing mixed dtype values (:issue:`57048`)
 - Fixed regression in :meth:`Series.pct_change` raising a ``ValueError`` for an empty :class:`Series` (:issue:`57056`)
 - Fixed regression in :meth:`Series.to_numpy` when dtype is given as float and the data contains NaNs (:issue:`57121`)
diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py
index 8c41de9c9f..7c49ce5343 100644
--- a/pandas/core/arrays/masked.py
+++ b/pandas/core/arrays/masked.py
@@ -501,6 +501,8 @@ class BaseMaskedArray(OpsMixin, ExtensionArray):
         """
         hasna = self._hasna
         dtype, na_value = to_numpy_dtype_inference(self, dtype, na_value, hasna)
+        if dtype is None:
+            dtype = object
 
         if hasna:
             if (
diff --git a/pandas/core/indexes/api.py b/pandas/core/indexes/api.py
index 560285bd57..15292953e7 100644
--- a/pandas/core/indexes/api.py
+++ b/pandas/core/indexes/api.py
@@ -295,6 +295,7 @@ def union_indexes(indexes, sort: bool | None = True) -> Index:
             raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex")
 
         if len(dtis) == len(indexes):
+            sort = True
             result = indexes[0]
 
         elif len(dtis) > 1:
diff --git a/pandas/core/reshape/concat.py b/pandas/core/reshape/concat.py
index aacea92611..dc18bb65b3 100644
--- a/pandas/core/reshape/concat.py
+++ b/pandas/core/reshape/concat.py
@@ -205,8 +205,10 @@ def concat(
         Check whether the new concatenated axis contains duplicates. This can
         be very expensive relative to the actual data concatenation.
     sort : bool, default False
-        Sort non-concatenation axis if it is not already aligned.
-
+        Sort non-concatenation axis if it is not already aligned. One exception to
+        this is when the non-concatentation axis is a DatetimeIndex and join='outer'
+        and the axis is not already aligned. In that case, the non-concatenation
+        axis is always sorted lexicographically.
     copy : bool, default True
         If False, do not copy data unnecessarily.
 


Test patch:
diff --git a/pandas/tests/arrays/masked/test_function.py b/pandas/tests/arrays/masked/test_function.py
index 4c7bd6e293..b259018cd6 100644
--- a/pandas/tests/arrays/masked/test_function.py
+++ b/pandas/tests/arrays/masked/test_function.py
@@ -5,6 +5,7 @@ from pandas.core.dtypes.common import is_integer_dtype
 
 import pandas as pd
 import pandas._testing as tm
+from pandas.core.arrays import BaseMaskedArray
 
 arrays = [pd.array([1, 2, 3, None], dtype=dtype) for dtype in tm.ALL_INT_EA_DTYPES]
 arrays += [
@@ -55,3 +56,19 @@ def test_tolist(data):
     result = data.tolist()
     expected = list(data)
     tm.assert_equal(result, expected)
+
+
+def test_to_numpy():
+    # GH#56991
+
+    class MyStringArray(BaseMaskedArray):
+        dtype = pd.StringDtype()
+        _dtype_cls = pd.StringDtype
+        _internal_fill_value = pd.NA
+
+    arr = MyStringArray(
+        values=np.array(["a", "b", "c"]), mask=np.array([False, True, False])
+    )
+    result = arr.to_numpy()
+    expected = np.array(["a", pd.NA, "c"])
+    tm.assert_numpy_array_equal(result, expected)
diff --git a/pandas/tests/reshape/concat/test_datetimes.py b/pandas/tests/reshape/concat/test_datetimes.py
index 71ddff7438..4c94dc0d51 100644
--- a/pandas/tests/reshape/concat/test_datetimes.py
+++ b/pandas/tests/reshape/concat/test_datetimes.py
@@ -73,23 +73,23 @@ class TestDatetimeConcat:
 
         exp_idx = DatetimeIndex(
             [
-                "2010-12-31 23:00:00+00:00",
-                "2011-01-01 00:00:00+00:00",
-                "2011-01-01 01:00:00+00:00",
                 "2010-12-31 15:00:00+00:00",
                 "2010-12-31 16:00:00+00:00",
                 "2010-12-31 17:00:00+00:00",
+                "2010-12-31 23:00:00+00:00",
+                "2011-01-01 00:00:00+00:00",
+                "2011-01-01 01:00:00+00:00",
             ]
         ).as_unit("ns")
 
         expected = DataFrame(
             [
-                [1, np.nan],
-                [2, np.nan],
-                [3, np.nan],
                 [np.nan, 1],
                 [np.nan, 2],
                 [np.nan, 3],
+                [1, np.nan],
+                [2, np.nan],
+                [3, np.nan],
             ],
             index=exp_idx,
             columns=["a", "b"],