Repository: pandas-dev/pandas
Pull request #55393 (2023-10-04 03:34:53)

TYP: format.formats
- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.



Patch:
diff --git a/doc/source/whatsnew/v2.1.2.rst b/doc/source/whatsnew/v2.1.2.rst
index df19b01610..87b71eac76 100644
--- a/doc/source/whatsnew/v2.1.2.rst
+++ b/doc/source/whatsnew/v2.1.2.rst
@@ -29,6 +29,7 @@ Bug fixes
 - Fixed bug in :meth:`DataFrame.resample` not respecting ``closed`` and ``label`` arguments for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55282`)
 - Fixed bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55281`)
 - Fixed bug in :meth:`Index.insert` raising when inserting ``None`` into :class:`Index` with ``dtype="string[pyarrow_numpy]"`` (:issue:`55365`)
+- Fixed bug in :meth:`Series.all`  and :meth:`Series.any` not treating missing values correctly for ``dtype="string[pyarrow_numpy]"`` (:issue:`55367`)
 - Fixed bug in :meth:`Series.rank` for ``string[pyarrow_numpy]`` dtype (:issue:`55362`)
 - Silence ``Period[B]`` warnings introduced by :issue:`53446` during normal plotting activity (:issue:`55138`)
 -
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index 7a177344a4..4749ceec4a 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -74,12 +74,12 @@ enhancement2
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
+- :attr:`Series.attrs` / :attr:`DataFrame.attrs` now uses a deepcopy for propagating ``attrs`` (:issue:`54134`).
 - :func:`read_csv` now supports ``on_bad_lines`` parameter with ``engine="pyarrow"``. (:issue:`54480`)
 - :meth:`ExtensionArray._explode` interface method added to allow extension type implementations of the ``explode`` method (:issue:`54833`)
 - :meth:`ExtensionArray.duplicated` added to allow extension type implementations of the ``duplicated`` method (:issue:`55255`)
 - DataFrame.apply now allows the usage of numba (via ``engine="numba"``) to JIT compile the passed function, allowing for potential speedups (:issue:`54666`)
 - Implement masked algorithms for :meth:`Series.value_counts` (:issue:`54984`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_220.notable_bug_fixes:
@@ -279,6 +279,7 @@ Bug fixes
 ~~~~~~~~~
 - Bug in :class:`AbstractHolidayCalendar` where timezone data was not propagated when computing holiday observances (:issue:`54580`)
 - Bug in :class:`pandas.core.window.Rolling` where duplicate datetimelike indexes are treated as consecutive rather than equal with ``closed='left'`` and ``closed='neither'`` (:issue:`20712`)
+- Bug in :func:`pandas.api.types.is_string_dtype` while checking object array with no elements is of the string dtype (:issue:`54661`)
 - Bug in :meth:`DataFrame.apply` where passing ``raw=True`` ignored ``args`` passed to the applied function (:issue:`55009`)
 - Bug in :meth:`pandas.DataFrame.melt` where it would not preserve the datetime (:issue:`55254`)
 - Bug in :meth:`pandas.read_excel` with a ODS file without cached formatted cell for float values (:issue:`55219`)
diff --git a/pandas/compat/numpy/__init__.py b/pandas/compat/numpy/__init__.py
index 1b974a92f8..51c9892b64 100644
--- a/pandas/compat/numpy/__init__.py
+++ b/pandas/compat/numpy/__init__.py
@@ -1,4 +1,6 @@
 """ support numpy compatibility across versions """
+import warnings
+
 import numpy as np
 
 from pandas.util.version import Version
@@ -26,8 +28,14 @@ np_ulong: type
 
 if _nlv >= Version("2.0.0.dev0"):
     try:
-        np_long = np.long  # type: ignore[attr-defined]
-        np_ulong = np.ulong  # type: ignore[attr-defined]
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                r".*In the future `np\.long` will be defined as.*",
+                FutureWarning,
+            )
+            np_long = np.long  # type: ignore[attr-defined]
+            np_ulong = np.ulong  # type: ignore[attr-defined]
     except AttributeError:
         np_long = np.int_
         np_ulong = np.uint
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
index 8d2633c10b..4152d2a50e 100644
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -1819,7 +1819,7 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
 
         return arr._from_backing_data(backing)
 
-    def _internal_get_values(self):
+    def _internal_get_values(self) -> ArrayLike:
         """
         Return the values.
 
@@ -1827,15 +1827,19 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
 
         Returns
         -------
-        np.ndarray or Index
-            A numpy array of the same dtype as categorical.categories.dtype or
-            Index if datetime / periods.
+        np.ndarray or ExtensionArray
+            A numpy array or ExtensionArray of the same dtype as
+            categorical.categories.dtype.
         """
         # if we are a datetime and period index, return Index to keep metadata
         if needs_i8_conversion(self.categories.dtype):
-            return self.categories.take(self._codes, fill_value=NaT)
+            return self.categories.take(self._codes, fill_value=NaT)._values
         elif is_integer_dtype(self.categories.dtype) and -1 in self._codes:
-            return self.categories.astype("object").take(self._codes, fill_value=np.nan)
+            return (
+                self.categories.astype("object")
+                .take(self._codes, fill_value=np.nan)
+                ._values
+            )
         return np.array(self)
 
     def check_for_ordered(self, op) -> None:
diff --git a/pandas/core/arrays/string_arrow.py b/pandas/core/arrays/string_arrow.py
index db11e74951..24b99b5d48 100644
--- a/pandas/core/arrays/string_arrow.py
+++ b/pandas/core/arrays/string_arrow.py
@@ -631,9 +631,11 @@ class ArrowStringArrayNumpySemantics(ArrowStringArray):
         self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
     ):
         if name in ["any", "all"]:
-            arr = pc.and_kleene(
-                pc.invert(pc.is_null(self._pa_array)), pc.not_equal(self._pa_array, "")
-            )
+            if not skipna and name == "all":
+                nas = pc.invert(pc.is_null(self._pa_array))
+                arr = pc.and_kleene(nas, pc.not_equal(self._pa_array, ""))
+            else:
+                arr = pc.not_equal(self._pa_array, "")
             return ArrowExtensionArray(arr)._reduce(
                 name, skipna=skipna, keepdims=keepdims, **kwargs
             )
diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py
index b7b81b8271..931a220d7a 100644
--- a/pandas/core/arrays/timedeltas.py
+++ b/pandas/core/arrays/timedeltas.py
@@ -471,7 +471,7 @@ class TimedeltaArray(dtl.TimelikeOps):
         from pandas.io.formats.format import get_format_timedelta64
 
         # Relies on TimeDelta._repr_base
-        formatter = get_format_timedelta64(self._ndarray, na_rep)
+        formatter = get_format_timedelta64(self, na_rep)
         # equiv: np.array([formatter(x) for x in self._ndarray])
         #  but independent of dimension
         return np.frompyfunc(formatter, 1, 1)(self._ndarray)
diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py
index 9da4eac6a4..42e909a6b9 100644
--- a/pandas/core/dtypes/common.py
+++ b/pandas/core/dtypes/common.py
@@ -1664,9 +1664,12 @@ def is_all_strings(value: ArrayLike) -> bool:
     dtype = value.dtype
 
     if isinstance(dtype, np.dtype):
-        return dtype == np.dtype("object") and lib.is_string_array(
-            np.asarray(value), skipna=False
-        )
+        if len(value) == 0:
+            return dtype == np.dtype("object")
+        else:
+            return dtype == np.dtype("object") and lib.is_string_array(
+                np.asarray(value), skipna=False
+            )
     elif isinstance(dtype, CategoricalDtype):
         return dtype.categories.inferred_type == "string"
     return dtype == "string"
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
index 738f4cbe6b..2e6c8919ef 100644
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import collections
+from copy import deepcopy
 import datetime as dt
 from functools import partial
 import gc
@@ -368,6 +369,13 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
         --------
         DataFrame.flags : Global flags applying to this object.
 
+        Notes
+        -----
+        Many operations that create new datasets will copy ``attrs``. Copies
+        are always deep so that changing ``attrs`` will only affect the
+        present dataset. ``pandas.concat`` copies ``attrs`` only if all input
+        datasets have the same ``attrs``.
+
         Examples
         --------
         For Series:
@@ -6191,8 +6199,12 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
                stable across pandas releases.
         """
         if isinstance(other, NDFrame):
-            for name in other.attrs:
-                self.attrs[name] = other.attrs[name]
+            if other.attrs:
+                # We want attrs propagation to have minimal performance
+                # impact if attrs are not used; i.e. attrs is an empty dict.
+                # One could make the deepcopy unconditionally, but a deepcopy
+                # of an empty dict is 50x more expensive than the empty check.
+                self.attrs = deepcopy(other.attrs)
 
             self.flags.allows_duplicate_labels = other.flags.allows_duplicate_labels
             # For subclasses using _metadata.
@@ -6201,11 +6213,13 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
                 object.__setattr__(self, name, getattr(other, name, None))
 
         if method == "concat":
-            attrs = other.objs[0].attrs
-            check_attrs = all(objs.attrs == attrs for objs in other.objs[1:])
-            if check_attrs:
-                for name in attrs:
-                    self.attrs[name] = attrs[name]
+            # propagate attrs only if all concat arguments have the same attrs
+            if all(bool(obj.attrs) for obj in other.objs):
+                # all concatenate arguments have non-empty attrs
+                attrs = other.objs[0].attrs
+                have_same_attrs = all(obj.attrs == attrs for obj in other.objs[1:])
+                if have_same_attrs:
+                    self.attrs = deepcopy(attrs)
 
             allows_duplicate_labels = all(
                 x.flags.allows_duplicate_labels for x in other.objs
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
index 144045d40a..8955329a7a 100644
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -73,9 +73,7 @@ from pandas.core.dtypes.dtypes import (
 )
 from pandas.core.dtypes.generic import (
     ABCDataFrame,
-    ABCDatetimeIndex,
     ABCSeries,
-    ABCTimedeltaIndex,
 )
 from pandas.core.dtypes.inference import is_array_like
 from pandas.core.dtypes.missing import (
@@ -768,8 +766,8 @@ class MultiIndex(Index):
                 vals = cast("CategoricalIndex", vals)
                 vals = vals._data._internal_get_values()
 
-            if isinstance(vals.dtype, ExtensionDtype) or isinstance(
-                vals, (ABCDatetimeIndex, ABCTimedeltaIndex)
+            if isinstance(vals.dtype, ExtensionDtype) or lib.is_np_dtype(
+                vals.dtype, "mM"
             ):
                 vals = vals.astype(object)
 
@@ -1948,7 +1946,7 @@ class MultiIndex(Index):
                     # indexer to reorder the level codes
                     indexer = ensure_platform_int(indexer)
                     ri = lib.get_reverse_indexer(indexer, len(indexer))
-                    level_codes = algos.take_nd(ri, level_codes)
+                    level_codes = algos.take_nd(ri, level_codes, fill_value=-1)
 
             new_levels.append(lev)
             new_codes.append(level_codes)
diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py
index 922d0f37be..8b65a09ee5 100644
--- a/pandas/io/formats/format.py
+++ b/pandas/io/formats/format.py
@@ -72,6 +72,7 @@ from pandas.core.dtypes.missing import (
 from pandas.core.arrays import (
     Categorical,
     DatetimeArray,
+    ExtensionArray,
     TimedeltaArray,
 )
 from pandas.core.arrays.string_ import StringDtype
@@ -108,6 +109,7 @@ if TYPE_CHECKING:
         SequenceNotStr,
         StorageOptions,
         WriteBuffer,
+        npt,
     )
 
     from pandas import (
@@ -1216,7 +1218,7 @@ def get_buffer(
 
 
 def format_array(
-    values: Any,
+    values: ArrayLike,
     formatter: Callable | None,
     float_format: FloatFormatType | None = None,
     na_rep: str = "NaN",
@@ -1233,7 +1235,7 @@ def format_array(
 
     Parameters
     ----------
-    values
+    values : np.ndarray or ExtensionArray
     formatter
     float_format
     na_rep
@@ -1258,10 +1260,13 @@ def format_array(
     fmt_klass: type[GenericArrayFormatter]
     if lib.is_np_dtype(values.dtype, "M"):
         fmt_klass = Datetime64Formatter
+        values = cast(DatetimeArray, values)
     elif isinstance(values.dtype, DatetimeTZDtype):
         fmt_klass = Datetime64TZFormatter
+        values = cast(DatetimeArray, values)
     elif lib.is_np_dtype(values.dtype, "m"):
         fmt_klass = Timedelta64Formatter
+        values = cast(TimedeltaArray, values)
     elif isinstance(values.dtype, ExtensionDtype):
         fmt_klass = ExtensionArrayFormatter
     elif lib.is_np_dtype(values.dtype, "fc"):
@@ -1300,7 +1305,7 @@ def format_array(
 class GenericArrayFormatter:
     def __init__(
         self,
-        values: Any,
+        values: ArrayLike,
         digits: int = 7,
         formatter: Callable | None = None,
         na_rep: str = "NaN",
@@ -1622,9 +1627,11 @@ class IntArrayFormatter(GenericArrayFormatter):
 
 
 class Datetime64Formatter(GenericArrayFormatter):
+    values: DatetimeArray
+
     def __init__(
         self,
-        values: np.ndarray | Series | DatetimeIndex | DatetimeArray,
+        values: DatetimeArray,
         nat_rep: str = "NaT",
         date_format: None = None,
         **kwargs,
@@ -1637,21 +1644,23 @@ class Datetime64Formatter(GenericArrayFormatter):
         """we by definition have DO NOT have a TZ"""
         values = self.values
 
-        if not isinstance(values, DatetimeIndex):
-            values = DatetimeIndex(values)
+        dti = DatetimeIndex(values)
 
         if self.formatter is not None and callable(self.formatter):
-            return [self.formatter(x) for x in values]
+            return [self.formatter(x) for x in dti]
 
-        fmt_values = values._data._format_native_types(
+        fmt_values = dti._data._format_native_types(
             na_rep=self.nat_rep, date_format=self.date_format
         )
         return fmt_values.tolist()
 
 
 class ExtensionArrayFormatter(GenericArrayFormatter):
+    values: ExtensionArray
+
     def _format_strings(self) -> list[str]:
         values = extract_array(self.values, extract_numpy=True)
+        values = cast(ExtensionArray, values)
 
         formatter = self.formatter
         fallback_formatter = None
@@ -1813,13 +1822,10 @@ def get_format_datetime64(
 
 
 def get_format_datetime64_from_values(
-    values: np.ndarray | DatetimeArray | DatetimeIndex, date_format: str | None
+    values: DatetimeArray, date_format: str | None
 ) -> str | None:
     """given values and a date_format, return a string format"""
-    if isinstance(values, np.ndarray) and values.ndim > 1:
-        # We don't actually care about the order of values, and DatetimeIndex
-        #  only accepts 1D values
-        values = values.ravel()
+    assert isinstance(values, DatetimeArray)
 
     ido = is_dates_only(values)
     if ido:
@@ -1829,6 +1835,8 @@ def get_format_datetime64_from_values(
 
 
 class Datetime64TZFormatter(Datetime64Formatter):
+    values: DatetimeArray
+
     def _format_strings(self) -> list[str]:
         """we by definition have a TZ"""
         ido = is_dates_only(self.values)
@@ -1842,9 +1850,11 @@ class Datetime64TZFormatter(Datetime64Formatter):
 
 
 class Timedelta64Formatter(GenericArrayFormatter):
+    values: TimedeltaArray
+
     def __init__(
         self,
-        values: np.ndarray | TimedeltaIndex,
+        values: TimedeltaArray,
         nat_rep: str = "NaT",
         box: bool = False,
         **kwargs,
@@ -1861,7 +1871,7 @@ class Timedelta64Formatter(GenericArrayFormatter):
 
 
 def get_format_timedelta64(
-    values: np.ndarray | TimedeltaIndex | TimedeltaArray,
+    values: TimedeltaArray,
     nat_rep: str | float = "NaT",
     box: bool = False,
 ) -> Callable:
@@ -1872,18 +1882,13 @@ def get_format_timedelta64(
     If box, then show the return in quotes
     """
     values_int = values.view(np.int64)
+    values_int = cast("npt.NDArray[np.int64]", values_int)
 
     consider_values = values_int != iNaT
 
     one_day_nanos = 86400 * 10**9
-    # error: Unsupported operand types for % ("ExtensionArray" and "int")
-    not_midnight = values_int % one_day_nanos != 0  # type: ignore[operator]
-    # error: Argument 1 to "__call__" of "ufunc" has incompatible type
-    # "Union[Any, ExtensionArray, ndarray]"; expected
-    # "Union[Union[int, float, complex, str, bytes, generic],
-    # Sequence[Union[int, float, complex, str, bytes, generic]],
-    # Sequence[Sequence[Any]], _SupportsArray]"
-    both = np.logical_and(consider_values, not_midnight)  # type: ignore[arg-type]
+    not_midnight = values_int % one_day_nanos != 0
+    both = np.logical_and(consider_values, not_midnight)
     even_days = both.sum() == 0
 
     if even_days:
@@ -1941,7 +1946,7 @@ def _make_fixed_width(
     return result
 
 
-def _trim_zeros_complex(str_complexes: np.ndarray, decimal: str = ".") -> list[str]:
+def _trim_zeros_complex(str_complexes: ArrayLike, decimal: str = ".") -> list[str]:
     """
     Separates the real and imaginary parts from the complex number, and
     executes the _trim_zeros_float method on each of those.
@@ -1987,7 +1992,7 @@ def _trim_zeros_single_float(str_float: str) -> str:
 
 
 def _trim_zeros_float(
-    str_floats: np.ndarray | list[str], decimal: str = "."
+    str_floats: ArrayLike | list[str], decimal: str = "."
 ) -> list[str]:
     """
     Trims the maximum number of trailing zeros equally from
@@ -2000,7 +2005,7 @@ def _trim_zeros_float(
     def is_number_with_decimal(x) -> bool:
         return re.match(number_regex, x) is not None
 
-    def should_trim(values: np.ndarray | list[str]) -> bool:
+    def should_trim(values: ArrayLike | list[str]) -> bool:
         """
         Determine if an array of strings should be trimmed.
 


Test patch:
diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py
index 4507857418..6f6cc5a5ad 100644
--- a/pandas/tests/dtypes/test_common.py
+++ b/pandas/tests/dtypes/test_common.py
@@ -301,14 +301,23 @@ def test_is_categorical_dtype():
         assert com.is_categorical_dtype(pd.CategoricalIndex([1, 2, 3]))
 
 
-def test_is_string_dtype():
-    assert not com.is_string_dtype(int)
-    assert not com.is_string_dtype(pd.Series([1, 2]))
-
-    assert com.is_string_dtype(str)
-    assert com.is_string_dtype(object)
-    assert com.is_string_dtype(np.array(["a", "b"]))
-    assert com.is_string_dtype(pd.StringDtype())
+@pytest.mark.parametrize(
+    "dtype, expected",
+    [
+        (int, False),
+        (pd.Series([1, 2]), False),
+        (str, True),
+        (object, True),
+        (np.array(["a", "b"]), True),
+        (pd.StringDtype(), True),
+        (pd.Index([], dtype="O"), True),
+    ],
+)
+def test_is_string_dtype(dtype, expected):
+    # GH#54661
+
+    result = com.is_string_dtype(dtype)
+    assert result is expected
 
 
 @pytest.mark.parametrize(
diff --git a/pandas/tests/frame/test_api.py b/pandas/tests/frame/test_api.py
index 8fc78629be..06bf169bf4 100644
--- a/pandas/tests/frame/test_api.py
+++ b/pandas/tests/frame/test_api.py
@@ -315,6 +315,15 @@ class TestDataFrameMisc:
         result = df.rename(columns=str)
         assert result.attrs == {"version": 1}
 
+    def test_attrs_deepcopy(self):
+        df = DataFrame({"A": [2, 3]})
+        assert df.attrs == {}
+        df.attrs["tags"] = {"spam", "ham"}
+
+        result = df.rename(columns=str)
+        assert result.attrs == df.attrs
+        assert result.attrs["tags"] is not df.attrs["tags"]
+
     @pytest.mark.parametrize("allows_duplicate_labels", [True, False, None])
     def test_set_flags(
         self, allows_duplicate_labels, frame_or_series, using_copy_on_write
diff --git a/pandas/tests/indexes/multi/test_sorting.py b/pandas/tests/indexes/multi/test_sorting.py
index 08c1a40929..b4dcef71dc 100644
--- a/pandas/tests/indexes/multi/test_sorting.py
+++ b/pandas/tests/indexes/multi/test_sorting.py
@@ -338,3 +338,12 @@ def test_sort_values_with_na_na_position(dtype, na_position):
         ]
     expected = MultiIndex.from_arrays(arrays)
     tm.assert_index_equal(result, expected)
+
+
+def test_sort_unnecessary_warning():
+    # GH#55386
+    midx = MultiIndex.from_tuples([(1.5, 2), (3.5, 3), (0, 1)])
+    midx = midx.set_levels([2.5, np.nan, 1], level=0)
+    result = midx.sort_values()
+    expected = MultiIndex.from_tuples([(1, 3), (2.5, 1), (np.nan, 2)])
+    tm.assert_index_equal(result, expected)
diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py
index 021252500e..560b2377ad 100644
--- a/pandas/tests/reductions/test_reductions.py
+++ b/pandas/tests/reductions/test_reductions.py
@@ -1087,7 +1087,8 @@ class TestSeriesReductions:
 
         ser = Series([None, "a"], dtype="string[pyarrow_numpy]")
         assert ser.any()
-        assert not ser.all()
+        assert ser.all()
+        assert not ser.all(skipna=False)
 
         ser = Series([None, ""], dtype="string[pyarrow_numpy]")
         assert not ser.any()