Repository: pylint-dev/astroid

Issue:
Replace modutils.is_standard_module() logic with sys.stdlib_module_names

This extends from the conversation in https://github.com/PyCQA/pylint/pull/8190.

The logic in `modutils.is_standard_module()` should largely be able to be replaced with [sys.stdlib_module_names](https://docs.python.org/3/library/sys.html#sys.stdlib_module_names), which was introduced in 3.10. The advantages are it will be faster (no imports, no filesystem traversal), it's not dependent on the local environment,  and it's maintained upstream, generated from source. For the referenced PR, I backported the generating code in CPython to generate sets for a shim to support 3.7 - 3.9.

I started working on a PR for Astroid, but it seems `modutils.is_standard_module()` actually does two different things depending on how it's called.
1. If no path is specified, it tries to determine if a module is part of the standard library (or a builtin, or compiled in) by inspecting the path of module after importing it.
2. If a path is specified, it does the same logic, but ultimately is determining if the module is in the path specified.

For the second case, I could only find one case in the wild, in pyreverse.

https://github.com/PyCQA/pylint/blob/5bc4cd9a4b4c240227a41786823a6f226864dc4b/pylint/pyreverse/inspector.py#L308

These seem like different behaviors to me. I'm unsure how to proceed with PR. Here are some options I've considered.

- Option 1:
  - Introduce a new function, basically a wrapper for sys.stdlib_module_names and the shim
  - Old behavior is preserved
  - Advantage of a function, even though it's very simple, is it provides a space to add overriding logic if needed down the road
   
- Option 2:
   - Only introduce the shim, so the code is in a common place for Astroid and Pylint
   - Can be dropped with 3.9
   - Old behavior is preserved

- Option 3:
  - Fall back to old behavior if a path is given, check sys.stdlib_module_names otherwise

- Option 4:
  - Deprecate `is_standard_module()`
  - Introduce new functions more specific to how they are used

- Option 5:
  - Do Nothing

I'm sure there are more options, but this is what comes to mind now. Would appreciate your thoughts and ideas.



Discussion:
Option 4 seems best to me. I would even be okay with completely removing the weird behaviour described in point 2 and adding a new function that `pyreverse` can use. Imo, if we have a clear changelog entry for this it shouldn't be considered a breaking change, it is very weird that that behaviour exists in the first place.

Please go ahead with the PR! Any work on `astroid` is always highly appreciated!

Patch:
diff --git a/astroid/_backport_stdlib_names.py b/astroid/_backport_stdlib_names.py
new file mode 100644
--- /dev/null
+++ b/astroid/_backport_stdlib_names.py
@@ -0,0 +1,356 @@
+# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
+# For details: https://github.com/PyCQA/astroid/blob/main/LICENSE
+# Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt
+
+"""
+Shim to support Python versions < 3.10 that don't have sys.stdlib_module_names
+
+These values were created by cherry-picking the commits from
+https://bugs.python.org/issue42955 into each version, but may be updated
+manually if changes are needed.
+"""
+
+import sys
+
+# TODO: Remove this file when Python 3.9 is no longer supported
+
+PY_3_7 = frozenset(
+    {
+        "__future__",
+        "_abc",
+        "_ast",
+        "_asyncio",
+        "_bisect",
+        "_blake2",
+        "_bootlocale",
+        "_bz2",
+        "_codecs",
+        "_codecs_cn",
+        "_codecs_hk",
+        "_codecs_iso2022",
+        "_codecs_jp",
+        "_codecs_kr",
+        "_codecs_tw",
+        "_collections",
+        "_collections_abc",
+        "_compat_pickle",
+        "_compression",
+        "_contextvars",
+        "_crypt",
+        "_csv",
+        "_ctypes",
+        "_curses",
+        "_curses_panel",
+        "_datetime",
+        "_dbm",
+        "_decimal",
+        "_dummy_thread",
+        "_elementtree",
+        "_functools",
+        "_gdbm",
+        "_hashlib",
+        "_heapq",
+        "_imp",
+        "_io",
+        "_json",
+        "_locale",
+        "_lsprof",
+        "_lzma",
+        "_markupbase",
+        "_md5",
+        "_msi",
+        "_multibytecodec",
+        "_multiprocessing",
+        "_opcode",
+        "_operator",
+        "_osx_support",
+        "_pickle",
+        "_posixsubprocess",
+        "_py_abc",
+        "_pydecimal",
+        "_pyio",
+        "_queue",
+        "_random",
+        "_sha1",
+        "_sha256",
+        "_sha3",
+        "_sha512",
+        "_signal",
+        "_sitebuiltins",
+        "_socket",
+        "_sqlite3",
+        "_sre",
+        "_ssl",
+        "_stat",
+        "_string",
+        "_strptime",
+        "_struct",
+        "_symtable",
+        "_thread",
+        "_threading_local",
+        "_tkinter",
+        "_tracemalloc",
+        "_uuid",
+        "_warnings",
+        "_weakref",
+        "_weakrefset",
+        "_winapi",
+        "abc",
+        "aifc",
+        "antigravity",
+        "argparse",
+        "array",
+        "ast",
+        "asynchat",
+        "asyncio",
+        "asyncore",
+        "atexit",
+        "audioop",
+        "base64",
+        "bdb",
+        "binascii",
+        "binhex",
+        "bisect",
+        "builtins",
+        "bz2",
+        "cProfile",
+        "calendar",
+        "cgi",
+        "cgitb",
+        "chunk",
+        "cmath",
+        "cmd",
+        "code",
+        "codecs",
+        "codeop",
+        "collections",
+        "colorsys",
+        "compileall",
+        "concurrent",
+        "configparser",
+        "contextlib",
+        "contextvars",
+        "copy",
+        "copyreg",
+        "crypt",
+        "csv",
+        "ctypes",
+        "curses",
+        "dataclasses",
+        "datetime",
+        "dbm",
+        "decimal",
+        "difflib",
+        "dis",
+        "distutils",
+        "doctest",
+        "dummy_threading",
+        "email",
+        "encodings",
+        "ensurepip",
+        "enum",
+        "errno",
+        "faulthandler",
+        "fcntl",
+        "filecmp",
+        "fileinput",
+        "fnmatch",
+        "formatter",
+        "fractions",
+        "ftplib",
+        "functools",
+        "gc",
+        "genericpath",
+        "getopt",
+        "getpass",
+        "gettext",
+        "glob",
+        "grp",
+        "gzip",
+        "hashlib",
+        "heapq",
+        "hmac",
+        "html",
+        "http",
+        "idlelib",
+        "imaplib",
+        "imghdr",
+        "imp",
+        "importlib",
+        "inspect",
+        "io",
+        "ipaddress",
+        "itertools",
+        "json",
+        "keyword",
+        "lib2to3",
+        "linecache",
+        "locale",
+        "logging",
+        "lzma",
+        "macpath",
+        "mailbox",
+        "mailcap",
+        "marshal",
+        "math",
+        "mimetypes",
+        "mmap",
+        "modulefinder",
+        "msilib",
+        "msvcrt",
+        "multiprocessing",
+        "netrc",
+        "nis",
+        "nntplib",
+        "nt",
+        "ntpath",
+        "nturl2path",
+        "numbers",
+        "opcode",
+        "operator",
+        "optparse",
+        "os",
+        "ossaudiodev",
+        "parser",
+        "pathlib",
+        "pdb",
+        "pickle",
+        "pickletools",
+        "pipes",
+        "pkgutil",
+        "platform",
+        "plistlib",
+        "poplib",
+        "posix",
+        "posixpath",
+        "pprint",
+        "profile",
+        "pstats",
+        "pty",
+        "pwd",
+        "py_compile",
+        "pyclbr",
+        "pydoc",
+        "pydoc_data",
+        "pyexpat",
+        "queue",
+        "quopri",
+        "random",
+        "re",
+        "readline",
+        "reprlib",
+        "resource",
+        "rlcompleter",
+        "runpy",
+        "sched",
+        "secrets",
+        "select",
+        "selectors",
+        "shelve",
+        "shlex",
+        "shutil",
+        "signal",
+        "site",
+        "smtpd",
+        "smtplib",
+        "sndhdr",
+        "socket",
+        "socketserver",
+        "spwd",
+        "sqlite3",
+        "sre_compile",
+        "sre_constants",
+        "sre_parse",
+        "ssl",
+        "stat",
+        "statistics",
+        "string",
+        "stringprep",
+        "struct",
+        "subprocess",
+        "sunau",
+        "symbol",
+        "symtable",
+        "sys",
+        "sysconfig",
+        "syslog",
+        "tabnanny",
+        "tarfile",
+        "telnetlib",
+        "tempfile",
+        "termios",
+        "textwrap",
+        "this",
+        "threading",
+        "time",
+        "timeit",
+        "tkinter",
+        "token",
+        "tokenize",
+        "trace",
+        "traceback",
+        "tracemalloc",
+        "tty",
+        "turtle",
+        "turtledemo",
+        "types",
+        "typing",
+        "unicodedata",
+        "unittest",
+        "urllib",
+        "uu",
+        "uuid",
+        "venv",
+        "warnings",
+        "wave",
+        "weakref",
+        "webbrowser",
+        "winreg",
+        "winsound",
+        "wsgiref",
+        "xdrlib",
+        "xml",
+        "xmlrpc",
+        "zipapp",
+        "zipfile",
+        "zipimport",
+        "zlib",
+    }
+)
+
+PY_3_8 = frozenset(
+    PY_3_7
+    - {
+        "macpath",
+    }
+    | {
+        "_posixshmem",
+        "_statistics",
+        "_xxsubinterpreters",
+    }
+)
+
+PY_3_9 = frozenset(
+    PY_3_8
+    - {
+        "_dummy_thread",
+        "dummy_threading",
+    }
+    | {
+        "_aix_support",
+        "_bootsubprocess",
+        "_peg_parser",
+        "_zoneinfo",
+        "graphlib",
+        "zoneinfo",
+    }
+)
+
+if sys.version_info[:2] == (3, 7):
+    stdlib_module_names = PY_3_7
+elif sys.version_info[:2] == (3, 8):
+    stdlib_module_names = PY_3_8
+elif sys.version_info[:2] == (3, 9):
+    stdlib_module_names = PY_3_9
+else:
+    raise AssertionError("This module is only intended as a backport for Python <= 3.9")
diff --git a/astroid/manager.py b/astroid/manager.py
--- a/astroid/manager.py
+++ b/astroid/manager.py
@@ -30,7 +30,7 @@
     get_source_file,
     is_module_name_part_of_extension_package_whitelist,
     is_python_source,
-    is_standard_module,
+    is_stdlib_module,
     load_module_from_name,
     modpath_from_file,
 )
@@ -154,7 +154,7 @@ def _build_namespace_module(
     def _can_load_extension(self, modname: str) -> bool:
         if self.always_load_extensions:
             return True
-        if is_standard_module(modname):
+        if is_stdlib_module(modname):
             return True
         return is_module_name_part_of_extension_package_whitelist(
             modname, self.extension_package_whitelist
diff --git a/astroid/modutils.py b/astroid/modutils.py
--- a/astroid/modutils.py
+++ b/astroid/modutils.py
@@ -26,14 +26,20 @@
 import sys
 import sysconfig
 import types
+import warnings
 from collections.abc import Callable, Iterable, Sequence
 from contextlib import redirect_stderr, redirect_stdout
 from functools import lru_cache
 from pathlib import Path
 
-from astroid.const import IS_JYTHON, IS_PYPY
+from astroid.const import IS_JYTHON, IS_PYPY, PY310_PLUS
 from astroid.interpreter._import import spec, util
 
+if PY310_PLUS:
+    from sys import stdlib_module_names
+else:
+    from astroid._backport_stdlib_names import stdlib_module_names
+
 logger = logging.getLogger(__name__)
 
 
@@ -510,6 +516,41 @@ def is_python_source(filename: str | None) -> bool:
     return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
 
 
+def is_stdlib_module(modname: str) -> bool:
+    """Return: True if the modname is in the standard library"""
+    return modname.split(".")[0] in stdlib_module_names
+
+
+def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
+    """Try to determine if a module is imported from one of the specified paths
+
+    :param modname: name of the module
+
+    :param path: paths to consider
+
+    :return:
+      true if the module:
+      - is located on the path listed in one of the directory in `paths`
+    """
+
+    modname = modname.split(".")[0]
+    try:
+        filename = file_from_modpath([modname])
+    except ImportError:
+        # Import failed, we can't check path if we don't know it
+        return False
+
+    if filename is None:
+        # No filename likely means it's compiled in, or potentially a namespace
+        return False
+    filename = _normalize_path(filename)
+
+    if isinstance(path, str):
+        return filename.startswith(_cache_normalize_path(path))
+
+    return any(filename.startswith(_cache_normalize_path(entry)) for entry in path)
+
+
 def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> bool:
     """Try to guess if a module is a standard python module (by default,
     see `std_path` parameter's description).
@@ -523,6 +564,12 @@ def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> b
       - is located on the path listed in one of the directory in `std_path`
       - is a built-in module
     """
+    warnings.warn(
+        "is_standard_module() is deprecated. Use, is_stdlib_module() or module_in_path() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
     modname = modname.split(".")[0]
     try:
         filename = file_from_modpath([modname])


Test patch:
diff --git a/tests/test_manager.py b/tests/test_manager.py
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -22,7 +22,7 @@
     AttributeInferenceError,
 )
 from astroid.interpreter._import import util
-from astroid.modutils import EXT_LIB_DIRS, is_standard_module
+from astroid.modutils import EXT_LIB_DIRS, module_in_path
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef
 
@@ -411,7 +411,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
 
         # Generate some hits and misses
         ClassDef().lookup("garbage")
-        is_standard_module("unittest", std_path=["garbage_path"])
+        module_in_path("unittest", "garbage_path")
         util.is_namespace("unittest")
         astroid.interpreter.objectmodel.ObjectModel().attributes()
         with pytest.raises(AttributeInferenceError):
diff --git a/tests/test_modutils.py b/tests/test_modutils.py
--- a/tests/test_modutils.py
+++ b/tests/test_modutils.py
@@ -20,6 +20,7 @@
 
 import astroid
 from astroid import modutils
+from astroid.const import PY310_PLUS
 from astroid.interpreter._import import spec
 
 from . import resources
@@ -287,7 +288,7 @@ def test_raise(self) -> None:
         self.assertRaises(modutils.NoSourceFile, modutils.get_source_file, "whatever")
 
 
-class StandardLibModuleTest(resources.SysPathSetup, unittest.TestCase):
+class IsStandardModuleTest(resources.SysPathSetup, unittest.TestCase):
     """
     Return true if the module may be considered as a module from the standard
     library.
@@ -296,50 +297,153 @@ class StandardLibModuleTest(resources.SysPathSetup, unittest.TestCase):
     def test_datetime(self) -> None:
         # This is an interesting example, since datetime, on pypy,
         # is under lib_pypy, rather than the usual Lib directory.
-        self.assertTrue(modutils.is_standard_module("datetime"))
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("datetime")
 
     def test_builtins(self) -> None:
-        self.assertFalse(modutils.is_standard_module("__builtin__"))
-        self.assertTrue(modutils.is_standard_module("builtins"))
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("__builtin__")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("builtins")
 
     def test_builtin(self) -> None:
-        self.assertTrue(modutils.is_standard_module("sys"))
-        self.assertTrue(modutils.is_standard_module("marshal"))
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("sys")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("marshal")
 
     def test_nonstandard(self) -> None:
-        self.assertFalse(modutils.is_standard_module("astroid"))
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("astroid")
 
     def test_unknown(self) -> None:
-        self.assertFalse(modutils.is_standard_module("unknown"))
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("unknown")
 
     def test_4(self) -> None:
-        self.assertTrue(modutils.is_standard_module("hashlib"))
-        self.assertTrue(modutils.is_standard_module("pickle"))
-        self.assertTrue(modutils.is_standard_module("email"))
-        self.assertTrue(modutils.is_standard_module("io"))
-        self.assertFalse(modutils.is_standard_module("StringIO"))
-        self.assertTrue(modutils.is_standard_module("unicodedata"))
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("hashlib")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("pickle")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("email")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("io")
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("StringIO")
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("unicodedata")
 
     def test_custom_path(self) -> None:
         datadir = resources.find("")
         if any(datadir.startswith(p) for p in modutils.EXT_LIB_DIRS):
             self.skipTest("known breakage of is_standard_module on installed package")
 
-        self.assertTrue(modutils.is_standard_module("data.module", (datadir,)))
-        self.assertTrue(
-            modutils.is_standard_module("data.module", (os.path.abspath(datadir),))
-        )
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("data.module", (datadir,))
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module(
+                "data.module", (os.path.abspath(datadir),)
+            )
         # "" will evaluate to cwd
-        self.assertTrue(modutils.is_standard_module("data.module", ("",)))
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("data.module", ("",))
 
     def test_failing_edge_cases(self) -> None:
         # using a subpackage/submodule path as std_path argument
-        self.assertFalse(modutils.is_standard_module("xml.etree", etree.__path__))
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("xml.etree", etree.__path__)
+        # using a module + object name as modname argument
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("sys.path")
+        # this is because only the first package/module is considered
+        with pytest.warns(DeprecationWarning):
+            assert modutils.is_standard_module("sys.whatever")
+        with pytest.warns(DeprecationWarning):
+            assert not modutils.is_standard_module("xml.whatever", etree.__path__)
+
+
+class IsStdLibModuleTest(resources.SysPathSetup, unittest.TestCase):
+    """
+    Return true if the module is path of the standard library
+    """
+
+    def test_datetime(self) -> None:
+        # This is an interesting example, since datetime, on pypy,
+        # is under lib_pypy, rather than the usual Lib directory.
+        assert modutils.is_stdlib_module("datetime")
+
+    def test_builtins(self) -> None:
+        assert not modutils.is_stdlib_module("__builtin__")
+        assert modutils.is_stdlib_module("builtins")
+
+    def test_builtin(self) -> None:
+        assert modutils.is_stdlib_module("sys")
+        assert modutils.is_stdlib_module("marshal")
+
+    def test_nonstandard(self) -> None:
+        assert not modutils.is_stdlib_module("astroid")
+
+    def test_unknown(self) -> None:
+        assert not modutils.is_stdlib_module("unknown")
+
+    def test_4(self) -> None:
+        assert modutils.is_stdlib_module("hashlib")
+        assert modutils.is_stdlib_module("pickle")
+        assert modutils.is_stdlib_module("email")
+        assert modutils.is_stdlib_module("io")
+        assert not modutils.is_stdlib_module("StringIO")
+        assert modutils.is_stdlib_module("unicodedata")
+
+    def test_subpackages(self) -> None:
         # using a module + object name as modname argument
-        self.assertTrue(modutils.is_standard_module("sys.path"))
+        assert modutils.is_stdlib_module("sys.path")
         # this is because only the first package/module is considered
-        self.assertTrue(modutils.is_standard_module("sys.whatever"))
-        self.assertFalse(modutils.is_standard_module("xml.whatever", etree.__path__))
+        assert modutils.is_stdlib_module("sys.whatever")
+
+    def test_platform_specific(self) -> None:
+        assert modutils.is_stdlib_module("_curses")
+        assert modutils.is_stdlib_module("msvcrt")
+        assert modutils.is_stdlib_module("termios")
+
+
+class ModuleInPathTest(resources.SysPathSetup, unittest.TestCase):
+    """
+    Return true if the module is imported from the specified path
+    """
+
+    def test_success(self) -> None:
+        datadir = resources.find("")
+        assert modutils.module_in_path("data.module", datadir)
+        assert modutils.module_in_path("data.module", (datadir,))
+        assert modutils.module_in_path("data.module", os.path.abspath(datadir))
+        # "" will evaluate to cwd
+        assert modutils.module_in_path("data.module", "")
+
+    def test_bad_import(self) -> None:
+        datadir = resources.find("")
+        assert not modutils.module_in_path("this_module_is_no_more", datadir)
+
+    def test_no_filename(self) -> None:
+        datadir = resources.find("")
+        assert not modutils.module_in_path("sys", datadir)
+
+    def test_failure(self) -> None:
+        datadir = resources.find("")
+        assert not modutils.module_in_path("etree", datadir)
+        assert not modutils.module_in_path("astroid", datadir)
+
+
+class BackportStdlibNamesTest(resources.SysPathSetup, unittest.TestCase):
+    """
+    Verify backport raises exception on newer versions
+    """
+
+    @pytest.mark.skipif(not PY310_PLUS, reason="Backport valid on <=3.9")
+    def test_import_error(self) -> None:
+        with pytest.raises(AssertionError):
+            # pylint: disable-next=import-outside-toplevel, unused-import
+            from astroid import _backport_stdlib_names  # noqa
 
 
 class IsRelativeTest(unittest.TestCase):