Repository: pandas-dev/pandas
Pull request #58058 (2024-03-29 01:48:32)

DEPR: allowing non-standard types in unique, factorize, isin
- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.



Patch:
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
index 2ebd6fb62b..0160687b1b 100644
--- a/doc/source/whatsnew/v3.0.0.rst
+++ b/doc/source/whatsnew/v3.0.0.rst
@@ -206,10 +206,13 @@ Removal of prior version deprecations/changes
 - :meth:`SeriesGroupBy.agg` no longer pins the name of the group to the input passed to the provided ``func`` (:issue:`51703`)
 - All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`)
 - All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`)
+- Disallow non-standard (``np.ndarray``, :class:`Index`, :class:`ExtensionArray`, or :class:`Series`) to :func:`isin`, :func:`unique`, :func:`factorize` (:issue:`52986`)
 - Disallow passing a pandas type to :meth:`Index.view` (:issue:`55709`)
+- Disallow units other than "s", "ms", "us", "ns" for datetime64 and timedelta64 dtypes in :func:`array` (:issue:`53817`)
 - Removed "freq" keyword from :class:`PeriodArray` constructor, use "dtype" instead (:issue:`52462`)
 - Removed deprecated "method" and "limit" keywords from :meth:`Series.replace` and :meth:`DataFrame.replace` (:issue:`53492`)
 - Removed the "closed" and "normalize" keywords in :meth:`DatetimeIndex.__new__` (:issue:`52628`)
+- Stopped performing dtype inference with in :meth:`Index.insert` with object-dtype index; this often affects the index/columns that result when setting new entries into an empty :class:`Series` or :class:`DataFrame` (:issue:`51363`)
 - Removed the "closed" and "unit" keywords in :meth:`TimedeltaIndex.__new__` (:issue:`52628`, :issue:`55499`)
 - All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`)
 - All arguments in :meth:`Series.to_dict` are now keyword only (:issue:`56493`)
diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py
index 6a6096567c..33beef2319 100644
--- a/pandas/core/algorithms.py
+++ b/pandas/core/algorithms.py
@@ -226,12 +226,9 @@ def _ensure_arraylike(values, func_name: str) -> ArrayLike:
         # GH#52986
         if func_name != "isin-targets":
             # Make an exception for the comps argument in isin.
-            warnings.warn(
-                f"{func_name} with argument that is not not a Series, Index, "
-                "ExtensionArray, or np.ndarray is deprecated and will raise in a "
-                "future version.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
+            raise TypeError(
+                f"{func_name} requires a Series, Index, "
+                f"ExtensionArray, or np.ndarray, got {type(values).__name__}."
             )
 
         inferred = lib.infer_dtype(values, skipna=False)
diff --git a/pandas/core/construction.py b/pandas/core/construction.py
index e6d99ab773..ec49340e9a 100644
--- a/pandas/core/construction.py
+++ b/pandas/core/construction.py
@@ -15,7 +15,6 @@ from typing import (
     cast,
     overload,
 )
-import warnings
 
 import numpy as np
 from numpy import ma
@@ -35,7 +34,6 @@ from pandas._typing import (
     DtypeObj,
     T,
 )
-from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.base import ExtensionDtype
 from pandas.core.dtypes.cast import (
@@ -373,13 +371,10 @@ def array(
         return TimedeltaArray._from_sequence(data, dtype=dtype, copy=copy)
 
     elif lib.is_np_dtype(dtype, "mM"):
-        warnings.warn(
+        raise ValueError(
+            # GH#53817
             r"datetime64 and timedelta64 dtype resolutions other than "
-            r"'s', 'ms', 'us', and 'ns' are deprecated. "
-            r"In future releases passing unsupported resolutions will "
-            r"raise an exception.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
+            r"'s', 'ms', 'us', and 'ns' are no longer supported."
         )
 
     return NumpyExtensionArray._from_sequence(data, dtype=dtype, copy=copy)
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index ae9b0eb4e4..30cf6f0b86 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -20,10 +20,7 @@ import warnings
 
 import numpy as np
 
-from pandas._config import (
-    get_option,
-    using_pyarrow_string_dtype,
-)
+from pandas._config import get_option
 
 from pandas._libs import (
     NaT,
@@ -6610,23 +6607,8 @@ class Index(IndexOpsMixin, PandasObject):
             loc = loc if loc >= 0 else loc - 1
             new_values[loc] = item
 
-        out = Index._with_infer(new_values, name=self.name)
-        if (
-            using_pyarrow_string_dtype()
-            and is_string_dtype(out.dtype)
-            and new_values.dtype == object
-        ):
-            out = out.astype(new_values.dtype)
-        if self.dtype == object and out.dtype != object:
-            # GH#51363
-            warnings.warn(
-                "The behavior of Index.insert with object-dtype is deprecated, "
-                "in a future version this will return an object-dtype Index "
-                "instead of inferring a non-object dtype. To retain the old "
-                "behavior, do `idx.insert(loc, item).infer_objects(copy=False)`",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
+        # GH#51363 stopped doing dtype inference here
+        out = Index(new_values, dtype=new_values.dtype, name=self.name)
         return out
 
     def drop(
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index 6b4070ed63..982e305b7e 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -1896,15 +1896,7 @@ class _iLocIndexer(_LocationIndexer):
                     # just replacing the block manager here
                     # so the object is the same
                     index = self.obj._get_axis(i)
-                    with warnings.catch_warnings():
-                        # TODO: re-issue this with setitem-specific message?
-                        warnings.filterwarnings(
-                            "ignore",
-                            "The behavior of Index.insert with object-dtype "
-                            "is deprecated",
-                            category=FutureWarning,
-                        )
-                        labels = index.insert(len(index), key)
+                    labels = index.insert(len(index), key)
 
                     # We are expanding the Series/DataFrame values to match
                     #  the length of thenew index `labels`.  GH#40096 ensure
@@ -2222,14 +2214,7 @@ class _iLocIndexer(_LocationIndexer):
         # and set inplace
         if self.ndim == 1:
             index = self.obj.index
-            with warnings.catch_warnings():
-                # TODO: re-issue this with setitem-specific message?
-                warnings.filterwarnings(
-                    "ignore",
-                    "The behavior of Index.insert with object-dtype is deprecated",
-                    category=FutureWarning,
-                )
-                new_index = index.insert(len(index), indexer)
+            new_index = index.insert(len(index), indexer)
 
             # we have a coerced indexer, e.g. a float
             # that matches in an int64 Index, so
diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py
index af851e1fc8..8fda9cd23b 100644
--- a/pandas/core/internals/managers.py
+++ b/pandas/core/internals/managers.py
@@ -1480,14 +1480,7 @@ class BlockManager(libinternals.BlockManager, BaseBlockManager):
         value : np.ndarray or ExtensionArray
         refs : The reference tracking object of the value to set.
         """
-        with warnings.catch_warnings():
-            # TODO: re-issue this with setitem-specific message?
-            warnings.filterwarnings(
-                "ignore",
-                "The behavior of Index.insert with object-dtype is deprecated",
-                category=FutureWarning,
-            )
-            new_axis = self.items.insert(loc, item)
+        new_axis = self.items.insert(loc, item)
 
         if value.ndim == 2:
             value = value.T


Test patch:
diff --git a/pandas/tests/arrays/test_array.py b/pandas/tests/arrays/test_array.py
index a84fefebf0..50dafb5dbb 100644
--- a/pandas/tests/arrays/test_array.py
+++ b/pandas/tests/arrays/test_array.py
@@ -1,6 +1,5 @@
 import datetime
 import decimal
-import re
 
 import numpy as np
 import pytest
@@ -31,15 +30,13 @@ from pandas.tests.extension.decimal import (
 
 @pytest.mark.parametrize("dtype_unit", ["M8[h]", "M8[m]", "m8[h]"])
 def test_dt64_array(dtype_unit):
-    # PR 53817
+    # GH#53817
     dtype_var = np.dtype(dtype_unit)
     msg = (
         r"datetime64 and timedelta64 dtype resolutions other than "
-        r"'s', 'ms', 'us', and 'ns' are deprecated. "
-        r"In future releases passing unsupported resolutions will "
-        r"raise an exception."
+        r"'s', 'ms', 'us', and 'ns' are no longer supported."
     )
-    with tm.assert_produces_warning(FutureWarning, match=re.escape(msg)):
+    with pytest.raises(ValueError, match=msg):
         pd.array([], dtype=dtype_var)
 
 
diff --git a/pandas/tests/indexes/test_old_base.py b/pandas/tests/indexes/test_old_base.py
index d4dc248b39..f41c6870cd 100644
--- a/pandas/tests/indexes/test_old_base.py
+++ b/pandas/tests/indexes/test_old_base.py
@@ -409,19 +409,13 @@ class TestBase:
         tm.assert_index_equal(result, expected)
 
     def test_insert_base(self, index):
+        # GH#51363
         trimmed = index[1:4]
 
         if not len(index):
             pytest.skip("Not applicable for empty index")
 
-        # test 0th element
-        warn = None
-        if index.dtype == object and index.inferred_type == "boolean":
-            # GH#51363
-            warn = FutureWarning
-        msg = "The behavior of Index.insert with object-dtype is deprecated"
-        with tm.assert_produces_warning(warn, match=msg):
-            result = trimmed.insert(0, index[0])
+        result = trimmed.insert(0, index[0])
         assert index[0:4].equals(result)
 
     @pytest.mark.skipif(
diff --git a/pandas/tests/indexing/test_loc.py b/pandas/tests/indexing/test_loc.py
index c01a8647dd..01dab14c7e 100644
--- a/pandas/tests/indexing/test_loc.py
+++ b/pandas/tests/indexing/test_loc.py
@@ -2025,7 +2025,7 @@ class TestLocSetitemWithExpansion:
         ids=["self", "to_datetime64", "to_pydatetime", "np.datetime64"],
     )
     def test_loc_setitem_datetime_keys_cast(self, conv):
-        # GH#9516
+        # GH#9516, GH#51363 changed in 3.0 to not cast on Index.insert
         dt1 = Timestamp("20130101 09:00:00")
         dt2 = Timestamp("20130101 10:00:00")
         df = DataFrame()
@@ -2034,7 +2034,7 @@ class TestLocSetitemWithExpansion:
 
         expected = DataFrame(
             {"one": [100.0, 200.0]},
-            index=[dt1, dt2],
+            index=Index([conv(dt1), conv(dt2)], dtype=object),
             columns=Index(["one"], dtype=object),
         )
         tm.assert_frame_equal(df, expected)
diff --git a/pandas/tests/libs/test_hashtable.py b/pandas/tests/libs/test_hashtable.py
index e54764f9ac..b70386191d 100644
--- a/pandas/tests/libs/test_hashtable.py
+++ b/pandas/tests/libs/test_hashtable.py
@@ -730,12 +730,11 @@ class TestHelpFunctionsWithNans:
 
 def test_ismember_tuple_with_nans():
     # GH-41836
-    values = [("a", float("nan")), ("b", 1)]
+    values = np.empty(2, dtype=object)
+    values[:] = [("a", float("nan")), ("b", 1)]
     comps = [("a", float("nan"))]
 
-    msg = "isin with argument that is not not a Series"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        result = isin(values, comps)
+    result = isin(values, comps)
     expected = np.array([True, False], dtype=np.bool_)
     tm.assert_numpy_array_equal(result, expected)
 
diff --git a/pandas/tests/series/indexing/test_set_value.py b/pandas/tests/series/indexing/test_set_value.py
index cbe1a8bf29..99e71fa4b8 100644
--- a/pandas/tests/series/indexing/test_set_value.py
+++ b/pandas/tests/series/indexing/test_set_value.py
@@ -3,17 +3,17 @@ from datetime import datetime
 import numpy as np
 
 from pandas import (
-    DatetimeIndex,
+    Index,
     Series,
 )
 import pandas._testing as tm
 
 
 def test_series_set_value():
-    # GH#1561
+    # GH#1561, GH#51363 as of 3.0 we do not do inference in Index.insert
 
     dates = [datetime(2001, 1, 1), datetime(2001, 1, 2)]
-    index = DatetimeIndex(dates)
+    index = Index(dates, dtype=object)
 
     s = Series(dtype=object)
     s._set_value(dates[0], 1.0)
diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py
index 6be325073b..99535f2730 100644
--- a/pandas/tests/series/indexing/test_setitem.py
+++ b/pandas/tests/series/indexing/test_setitem.py
@@ -495,11 +495,11 @@ class TestSetitemCallable:
 
 class TestSetitemWithExpansion:
     def test_setitem_empty_series(self):
-        # GH#10193
+        # GH#10193, GH#51363 changed in 3.0 to not do inference in Index.insert
         key = Timestamp("2012-01-01")
         series = Series(dtype=object)
         series[key] = 47
-        expected = Series(47, [key])
+        expected = Series(47, Index([key], dtype=object))
         tm.assert_series_equal(series, expected)
 
     def test_setitem_empty_series_datetimeindex_preserves_freq(self):
diff --git a/pandas/tests/test_algos.py b/pandas/tests/test_algos.py
index 365ec452a7..1b5d33fc10 100644
--- a/pandas/tests/test_algos.py
+++ b/pandas/tests/test_algos.py
@@ -54,16 +54,13 @@ import pandas.core.common as com
 class TestFactorize:
     def test_factorize_complex(self):
         # GH#17927
-        array = [1, 2, 2 + 1j]
-        msg = "factorize with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            labels, uniques = algos.factorize(array)
+        array = np.array([1, 2, 2 + 1j], dtype=complex)
+        labels, uniques = algos.factorize(array)
 
         expected_labels = np.array([0, 1, 2], dtype=np.intp)
         tm.assert_numpy_array_equal(labels, expected_labels)
 
-        # Should return a complex dtype in the future
-        expected_uniques = np.array([(1 + 0j), (2 + 0j), (2 + 1j)], dtype=object)
+        expected_uniques = np.array([(1 + 0j), (2 + 0j), (2 + 1j)], dtype=complex)
         tm.assert_numpy_array_equal(uniques, expected_uniques)
 
     def test_factorize(self, index_or_series_obj, sort):
@@ -265,9 +262,8 @@ class TestFactorize:
     )
     def test_factorize_tuple_list(self, data, expected_codes, expected_uniques):
         # GH9454
-        msg = "factorize with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            codes, uniques = pd.factorize(data)
+        data = com.asarray_tuplesafe(data, dtype=object)
+        codes, uniques = pd.factorize(data)
 
         tm.assert_numpy_array_equal(codes, np.array(expected_codes, dtype=np.intp))
 
@@ -488,12 +484,12 @@ class TestFactorize:
         "data, expected_codes, expected_uniques",
         [
             (
-                [1, None, 1, 2],
+                np.array([1, None, 1, 2], dtype=object),
                 np.array([0, 1, 0, 2], dtype=np.dtype("intp")),
                 np.array([1, np.nan, 2], dtype="O"),
             ),
             (
-                [1, np.nan, 1, 2],
+                np.array([1, np.nan, 1, 2], dtype=np.float64),
                 np.array([0, 1, 0, 2], dtype=np.dtype("intp")),
                 np.array([1, np.nan, 2], dtype=np.float64),
             ),
@@ -502,9 +498,7 @@ class TestFactorize:
     def test_int_factorize_use_na_sentinel_false(
         self, data, expected_codes, expected_uniques
     ):
-        msg = "factorize with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            codes, uniques = algos.factorize(data, use_na_sentinel=False)
+        codes, uniques = algos.factorize(data, use_na_sentinel=False)
 
         tm.assert_numpy_array_equal(uniques, expected_uniques, strict_nan=True)
         tm.assert_numpy_array_equal(codes, expected_codes, strict_nan=True)
@@ -777,9 +771,8 @@ class TestUnique:
         result = pd.unique(Series([2] + [1] * 5))
         tm.assert_numpy_array_equal(result, np.array([2, 1], dtype="int64"))
 
-        msg = "unique with argument that is not not a Series, Index,"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = pd.unique(list("aabc"))
+        data = np.array(["a", "a", "b", "c"], dtype=object)
+        result = pd.unique(data)
         expected = np.array(["a", "b", "c"], dtype=object)
         tm.assert_numpy_array_equal(result, expected)
 
@@ -815,9 +808,8 @@ class TestUnique:
     )
     def test_tuple_with_strings(self, arg, expected):
         # see GH 17108
-        msg = "unique with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = pd.unique(arg)
+        arg = com.asarray_tuplesafe(arg, dtype=object)
+        result = pd.unique(arg)
         tm.assert_numpy_array_equal(result, expected)
 
     def test_obj_none_preservation(self):
@@ -904,12 +896,6 @@ class TestIsin:
             algos.isin([1], 1)
 
     def test_basic(self):
-        msg = "isin with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = algos.isin([1, 2], [1])
-        expected = np.array([True, False])
-        tm.assert_numpy_array_equal(result, expected)
-
         result = algos.isin(np.array([1, 2]), [1])
         expected = np.array([True, False])
         tm.assert_numpy_array_equal(result, expected)
@@ -926,21 +912,20 @@ class TestIsin:
         expected = np.array([True, False])
         tm.assert_numpy_array_equal(result, expected)
 
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = algos.isin(["a", "b"], ["a"])
+        arg = np.array(["a", "b"], dtype=object)
+        result = algos.isin(arg, ["a"])
         expected = np.array([True, False])
         tm.assert_numpy_array_equal(result, expected)
 
-        result = algos.isin(Series(["a", "b"]), Series(["a"]))
+        result = algos.isin(Series(arg), Series(["a"]))
         expected = np.array([True, False])
         tm.assert_numpy_array_equal(result, expected)
 
-        result = algos.isin(Series(["a", "b"]), {"a"})
+        result = algos.isin(Series(arg), {"a"})
         expected = np.array([True, False])
         tm.assert_numpy_array_equal(result, expected)
 
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = algos.isin(["a", "b"], [1])
+        result = algos.isin(arg, [1])
         expected = np.array([False, False])
         tm.assert_numpy_array_equal(result, expected)
 
@@ -1058,12 +1043,10 @@ class TestIsin:
         # at least, isin() should follow python's "np.nan in [nan] == True"
         # casting to -> np.float64 -> another float-object somewhere on
         # the way could lead jeopardize this behavior
-        comps = [np.nan]  # could be casted to float64
+        comps = np.array([np.nan], dtype=object)  # could be casted to float64
         values = [np.nan]
         expected = np.array([True])
-        msg = "isin with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = algos.isin(comps, values)
+        result = algos.isin(comps, values)
         tm.assert_numpy_array_equal(expected, result)
 
     def test_same_nan_is_in_large(self):
@@ -1098,12 +1081,12 @@ class TestIsin:
 
         a, b = LikeNan(), LikeNan()
 
-        msg = "isin with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            # same object -> True
-            tm.assert_numpy_array_equal(algos.isin([a], [a]), np.array([True]))
-            # different objects -> False
-            tm.assert_numpy_array_equal(algos.isin([a], [b]), np.array([False]))
+        arg = np.array([a], dtype=object)
+
+        # same object -> True
+        tm.assert_numpy_array_equal(algos.isin(arg, [a]), np.array([True]))
+        # different objects -> False
+        tm.assert_numpy_array_equal(algos.isin(arg, [b]), np.array([False]))
 
     def test_different_nans(self):
         # GH 22160
@@ -1132,12 +1115,11 @@ class TestIsin:
     def test_no_cast(self):
         # GH 22160
         # ensure 42 is not casted to a string
-        comps = ["ss", 42]
+        comps = np.array(["ss", 42], dtype=object)
         values = ["42"]
         expected = np.array([False, False])
-        msg = "isin with argument that is not not a Series, Index"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = algos.isin(comps, values)
+
+        result = algos.isin(comps, values)
         tm.assert_numpy_array_equal(expected, result)
 
     @pytest.mark.parametrize("empty", [[], Series(dtype=object), np.array([])])
@@ -1658,27 +1640,32 @@ class TestDuplicated:
         expected = np.empty(len(uniques), dtype=object)
         expected[:] = uniques
 
-        msg = "unique with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = pd.unique(arr)
-        tm.assert_numpy_array_equal(result, expected)
+        msg = "unique requires a Series, Index, ExtensionArray, or np.ndarray, got list"
+        with pytest.raises(TypeError, match=msg):
+            # GH#52986
+            pd.unique(arr)
+
+        res = pd.unique(com.asarray_tuplesafe(arr, dtype=object))
+        tm.assert_numpy_array_equal(res, expected)
 
     @pytest.mark.parametrize(
         "array,expected",
         [
             (
                 [1 + 1j, 0, 1, 1j, 1 + 2j, 1 + 2j],
-                # Should return a complex dtype in the future
-                np.array([(1 + 1j), 0j, (1 + 0j), 1j, (1 + 2j)], dtype=object),
+                np.array([(1 + 1j), 0j, (1 + 0j), 1j, (1 + 2j)], dtype=complex),
             )
         ],
     )
     def test_unique_complex_numbers(self, array, expected):
         # GH 17927
-        msg = "unique with argument that is not not a Series"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = pd.unique(array)
-        tm.assert_numpy_array_equal(result, expected)
+        msg = "unique requires a Series, Index, ExtensionArray, or np.ndarray, got list"
+        with pytest.raises(TypeError, match=msg):
+            # GH#52986
+            pd.unique(array)
+
+        res = pd.unique(np.array(array))
+        tm.assert_numpy_array_equal(res, expected)
 
 
 class TestHashTable: