Repository: pandas-dev/pandas
Pull request: DEPR/REGR: Fix pandas.util.testing deprecation
Closes https://github.com/pandas-dev/pandas/issues/30735

This avoids using _DeprecatedModule, which doesn't work for
direct imports from a module. Sorry for the importlib magic, but
I think this is the correct way to do things.

cc @jorisvandenbossche.

Problem statement:
BUG/DEPR: the deprecation of util.testing fails for direct imports
```
In [1]: from pandas.util.testing import assert_frame_equal  
---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-1-79d99d902fdd> in <module>
----> 1 from pandas.util.testing import assert_frame_equal

ImportError: cannot import name 'assert_frame_equal' from 'pandas.util.testing' (/home/joris/scipy/pandas/pandas/util/testing/__init__.py)
```

It works when accessing from top-level import:

```
In [3]: pd.util.testing.assert_frame_equal
/home/joris/miniconda3/envs/dev/bin/ipython:1: FutureWarning: pandas._testing.assert_frame_equal is deprecated. Please use pandas.testing.assert_frame_equal instead.
  #!/home/joris/miniconda3/envs/dev/bin/python
Out[3]: <function pandas._testing.assert_frame_equal(left, right, check_dtype=True, check_index_type='equiv', check_column_type='equiv', check_frame_type=True, check_less_precise=False, check_names=True, by_blocks=False, check_exact=False, check_datetimelike_compat=False, check_categorical=True, check_like=False, obj='DataFrame')>
```

Hints from issue discussion:
cc @TomAugspurger 
Thanks, looking into it.
I think the easiest way to restore these is as references to the _DeprecatedModule items. Something like

```python
# pandas/util/testing/__init__.py
testing = _DeprecatedModule("pandas._testing", "pandas.testing", _removals)

with warnings.catch_warnings():
    assert_frame_equal = testing.assert_frame_equal
    ...
```
Actually that doesn't work since the user doesn't see the warning...

Commit message: DEPR/REGR: Fix pandas.util.testing deprecation

Closes https://github.com/pandas-dev/pandas/issues/30735

This avoids using _DeprecatedModule, which doesn't work for
direct imports from a module.

Patch to review:
diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py
index bdb4e813023b6..72e2413ce87d9 100644
--- a/pandas/tests/api/test_api.py
+++ b/pandas/tests/api/test_api.py
@@ -1,3 +1,4 @@
+import sys
 from typing import List
 
 import pandas as pd
@@ -288,14 +289,20 @@ def test_testing(self):
         self.check(testing, self.funcs)
 
     def test_util_testing_deprecated(self):
-        s = pd.Series([], dtype="object")
-        with tm.assert_produces_warning(FutureWarning) as m:
-            import pandas.util.testing as tm2
+        # avoid cache state affecting the test
+        sys.modules.pop("pandas.util.testing", None)
 
-            tm2.assert_series_equal(s, s)
+        with tm.assert_produces_warning(FutureWarning) as m:
+            import pandas.util.testing  # noqa: F401
 
-        assert "pandas.testing.assert_series_equal" in str(m[0].message)
+        assert "pandas.util.testing is deprecated" in str(m[0].message)
+        assert "pandas.testing instead" in str(m[0].message)
 
+    def test_util_testing_deprecated_direct(self):
+        # avoid cache state affecting the test
+        sys.modules.pop("pandas.util.testing", None)
         with tm.assert_produces_warning(FutureWarning) as m:
-            tm2.DataFrame
-        assert "removed" in str(m[0].message)
+            from pandas.util.testing import assert_series_equal  # noqa: F401
+
+        assert "pandas.util.testing is deprecated" in str(m[0].message)
+        assert "pandas.testing instead" in str(m[0].message)
diff --git a/pandas/util/__init__.py b/pandas/util/__init__.py
index 231a23e247650..7a88f5c92bafa 100644
--- a/pandas/util/__init__.py
+++ b/pandas/util/__init__.py
@@ -1,4 +1,60 @@
-from pandas.util._decorators import Appender, Substitution, cache_readonly  # noqa
+import importlib as _importlib
+from importlib.abc import Loader as _Loader, MetaPathFinder
+from importlib.machinery import ModuleSpec as _ModuleSpec
+import sys as _sys
+import warnings as _warnings
 
-from pandas.core.util.hashing import hash_array, hash_pandas_object  # noqa
-from pandas.util.testing import testing  # noqa: F401
+from pandas.util._decorators import Appender, Substitution, cache_readonly
+
+from pandas.core.util.hashing import hash_array, hash_pandas_object
+
+# Custom import hook for the deprecated pandas.util.testing module.
+# The custom Finder only runs when python's standard import fail,
+# since we're adding to the end of sys.meta_path. In TestingLoader.create_module
+# we simply warn and then return the real testing module, pandas._testing.
+# This differs from _DeprecatedModule by using the import system, which
+# allows it to work with both `import foo; foo.bar` and `from foo import bar`.
+# But because Python caches imports, the warning appears only once.
+
+
+class _TestingFinder(MetaPathFinder):
+    @classmethod
+    def find_spec(cls, fullname, path=None, target=None):
+
+        name_parts = fullname.split(".")
+        if name_parts[:3] != ["pandas", "util", "testing"] or len(name_parts) > 3:
+            return None
+        else:
+            # return ModuleSpec(fullname, DataPackageImporter())
+            return _ModuleSpec(fullname, _TestingLoader())
+
+
+class _TestingLoader(_Loader):
+    @classmethod
+    def create_module(cls, spec):
+        module = _importlib.import_module("pandas._testing")
+        _warnings.warn(
+            (
+                "pandas.util.testing is deprecated. Use the "
+                "public methods from pandas.testing instead."
+            ),
+            FutureWarning,
+            stacklevel=2,
+        )
+        return module
+
+    @classmethod
+    def exec_module(cls, module):
+        pass
+
+
+_sys.meta_path.append(_TestingFinder())
+
+
+__all__ = [
+    "Appender",
+    "Substitution",
+    "cache_readonly",
+    "hash_array",
+    "hash_pandas_object",
+]
diff --git a/pandas/util/testing/__init__.py b/pandas/util/testing/__init__.py
deleted file mode 100644
index 02cbd19a9a888..0000000000000
--- a/pandas/util/testing/__init__.py
+++ /dev/null
@@ -1,144 +0,0 @@
-from pandas.util._depr_module import _DeprecatedModule
-
-_removals = [
-    "Categorical",
-    "CategoricalIndex",
-    "Counter",
-    "DataFrame",
-    "DatetimeArray",
-    "DatetimeIndex",
-    "ExtensionArray",
-    "FrameOrSeries",
-    "Index",
-    "IntervalArray",
-    "IntervalIndex",
-    "K",
-    "List",
-    "MultiIndex",
-    "N",
-    "Optional",
-    "PeriodArray",
-    "RANDS_CHARS",
-    "RANDU_CHARS",
-    "RNGContext",
-    "RangeIndex",
-    "Series",
-    "SubclassedCategorical",
-    "SubclassedDataFrame",
-    "SubclassedSeries",
-    "TimedeltaArray",
-    "Union",
-    "all_index_generator",
-    "all_timeseries_index_generator",
-    "array_equivalent",
-    "assert_attr_equal",
-    "assert_class_equal",
-    "assert_contains_all",
-    "assert_copy",
-    "assert_dict_equal",
-    "assert_is_sorted",
-    "assert_is_valid_plot_return_object",
-    "assert_produces_warning",
-    "bdate_range",
-    "box_expected",
-    "bz2",
-    "can_connect",
-    "can_set_locale",
-    "cast",
-    "close",
-    "contextmanager",
-    "convert_rows_list_to_csv_str",
-    "datetime",
-    "decompress_file",
-    "ensure_clean",
-    "ensure_clean_dir",
-    "ensure_safe_environment_variables",
-    "equalContents",
-    "getCols",
-    "getMixedTypeDict",
-    "getPeriodData",
-    "getSeriesData",
-    "getTimeSeriesData",
-    "get_locales",
-    "gzip",
-    "index_subclass_makers_generator",
-    "is_bool",
-    "is_categorical_dtype",
-    "is_datetime64_dtype",
-    "is_datetime64tz_dtype",
-    "is_extension_array_dtype",
-    "is_interval_dtype",
-    "is_list_like",
-    "is_number",
-    "is_period_dtype",
-    "is_sequence",
-    "is_timedelta64_dtype",
-    "isiterable",
-    "lzma",
-    "makeBoolIndex",
-    "makeCategoricalIndex",
-    "makeCustomDataframe",
-    "makeCustomIndex",
-    "makeDataFrame",
-    "makeDateIndex",
-    "makeFloatIndex",
-    "makeFloatSeries",
-    "makeIntIndex",
-    "makeIntervalIndex",
-    "makeMissingCustomDataframe",
-    "makeMissingDataframe",
-    "makeMixedDataFrame",
-    "makeMultiIndex",
-    "makeObjectSeries",
-    "makePeriodFrame",
-    "makePeriodIndex",
-    "makePeriodSeries",
-    "makeRangeIndex",
-    "makeStringIndex",
-    "makeStringSeries",
-    "makeTimeDataFrame",
-    "makeTimeSeries",
-    "makeTimedeltaIndex",
-    "makeUIntIndex",
-    "makeUnicodeIndex",
-    "needs_i8_conversion",
-    "network",
-    "np",
-    "optional_args",
-    "os",
-    "pd",
-    "period_array",
-    "pprint_thing",
-    "raise_assert_detail",
-    "rand",
-    "randbool",
-    "randn",
-    "rands",
-    "rands_array",
-    "randu",
-    "randu_array",
-    "reset_display_options",
-    "reset_testing_mode",
-    "rmtree",
-    "round_trip_localpath",
-    "round_trip_pathlib",
-    "round_trip_pickle",
-    "set_locale",
-    "set_testing_mode",
-    "set_timezone",
-    "string",
-    "take_1d",
-    "tempfile",
-    "test_parallel",
-    "to_array",
-    "urlopen",
-    "use_numexpr",
-    "warnings",
-    "with_connectivity_check",
-    "with_csv_dialect",
-    "wraps",
-    "write_to_compressed",
-    "zipfile",
-]
-
-testing = _DeprecatedModule("pandas._testing", "pandas.testing", _removals)


Code review comments:
diff_hunk: @@ -1,4 +1,60 @@
-from pandas.util._decorators import Appender, Substitution, cache_readonly  # noqa
+import importlib as _importlib
path: pandas/util/__init__.py
text: @user1:
umm why don't you actually just leave pandas.util.testing as a module itself? (and just re-import what you need). This is how we did all of the original deprecations.