Repository: pylint-dev/astroid

Issue:
Remove dependency to ``setuptools`` and ``distutil``
As see in #1030  and #1100 there is a hidden dependency to setuptools (and probably distutils too), we made it explicit in #1100 but the better thing to do would be to remove it entirely if possible.

Decrease the time required to import astroid
### Steps to reproduce

``python3 -X importtime -c 'import astroid'``

### Current behavior

It takes a half second to import astroid. This account for 92% of pylint's startup time in 2.13.0-dev0. See https://github.com/PyCQA/pylint/issues/4814 for details.

```
import time: self [us] | cumulative | imported package
import time:       182 |        182 |   _io
import time:        59 |         59 |   marshal
import time:       213 |        213 |   posix
import time:       463 |        916 | _frozen_importlib_external
import time:       143 |        143 |   time
import time:       210 |        353 | zipimport
import time:        68 |         68 |     _codecs
import time:      8157 |       8225 |   codecs
import time:      4532 |       4532 |   encodings.aliases
import time:      2092 |      14848 | encodings
import time:       526 |        526 | encodings.utf_8
import time:       183 |        183 | _signal
import time:       910 |        910 | encodings.latin_1
import time:        34 |         34 |     _abc
import time:       527 |        560 |   abc
import time:       539 |       1099 | io
import time:        43 |         43 |       _stat
import time:       522 |        564 |     stat
import time:      1066 |       1066 |     _collections_abc
import time:       459 |        459 |       genericpath
import time:       481 |        939 |     posixpath
import time:       781 |       3349 |   os
import time:       451 |        451 |   _sitebuiltins
import time:        57 |         57 |     _locale
import time:       411 |        467 |   _bootlocale
import time:      1220 |       1220 |     apport_python_hook
import time:       396 |       1615 |   sitecustomize
import time:      4927 |      10808 | site
import time:       989 |        989 |     types
import time:       833 |        833 |     warnings
import time:      1833 |       3654 |   importlib
import time:       954 |        954 |         enum
import time:        58 |         58 |           _sre
import time:       560 |        560 |             sre_constants
import time:      1033 |       1593 |           sre_parse
import time:       788 |       2437 |         sre_compile
import time:        58 |         58 |               _operator
import time:      1129 |       1187 |             operator
import time:       424 |        424 |             keyword
import time:        34 |         34 |               _heapq
import time:       499 |        533 |             heapq
import time:        85 |         85 |             itertools
import time:       496 |        496 |             reprlib
import time:        47 |         47 |             _collections
import time:      1360 |       4129 |           collections
import time:        37 |         37 |           _functools
import time:      3949 |       8114 |         functools
import time:       451 |        451 |         copyreg
import time:       751 |      12704 |       re
import time:       547 |      13251 |     fnmatch
import time:        70 |         70 |       nt
import time:        49 |         49 |       nt
import time:        46 |         46 |       nt
import time:        46 |         46 |       nt
import time:       608 |        817 |     ntpath
import time:        47 |         47 |     errno
import time:       734 |        734 |       urllib
import time:      1519 |       2253 |     urllib.parse
import time:      3700 |      20066 |   pathlib
import time:      1277 |       1277 |   astroid.__pkginfo__
import time:       423 |        423 |         collections.abc
import time:       704 |        704 |         contextlib
import time:      1683 |       2809 |       typing
import time:       127 |        127 |               _opcode
import time:       619 |        745 |             opcode
import time:       744 |       1488 |           dis
import time:       442 |        442 |           importlib.machinery
import time:       480 |        480 |               token
import time:      1048 |       1527 |             tokenize
import time:       389 |       1915 |           linecache
import time:      1941 |       5784 |         inspect
import time:       518 |        518 |               _weakrefset
import time:       873 |       1390 |             weakref
import time:      1384 |       1384 |             wrapt._wrappers
import time:      6921 |       9695 |           wrapt.wrappers
import time:      1350 |       1350 |             threading
import time:      1461 |       2811 |           wrapt.decorators
import time:      1089 |       1089 |           wrapt.importer
import time:      2134 |      15727 |         wrapt
import time:        77 |         77 |             copy_reg
import time:       449 |        449 |               lazy_object_proxy.utils_py3
import time:       550 |        999 |             lazy_object_proxy.utils
import time:      1466 |       1466 |             lazy_object_proxy.cext
import time:       572 |        572 |             lazy_object_proxy._version
import time:      4391 |       7503 |           lazy_object_proxy
import time:       687 |       8190 |         astroid.util
import time:       635 |        635 |           pprint
import time:      2008 |       2643 |         astroid.context
import time:      1714 |       1714 |         astroid.exceptions
import time:      7641 |       7641 |         typing_extensions
import time:       616 |      42312 |       astroid.decorators
import time:       594 |        594 |       astroid.mixins
import time:      1371 |       1371 |         astroid.const
import time:      1881 |       1881 |           astroid.interpreter._import
import time:      1275 |       1275 |             distutils
import time:       385 |        385 |                 __future__
import time:        68 |         68 |                   binascii
import time:       631 |        631 |                     importlib.abc
import time:       503 |       1133 |                   importlib.util
import time:        66 |         66 |                     zlib
import time:       367 |        367 |                       _compression
import time:       251 |        251 |                       _bz2
import time:       485 |       1103 |                     bz2
import time:       217 |        217 |                       _lzma
import time:       431 |        647 |                     lzma
import time:        36 |         36 |                     pwd
import time:        26 |         26 |                     grp
import time:      1033 |       2908 |                   shutil
import time:        66 |         66 |                     _struct
import time:       337 |        402 |                   struct
import time:      1019 |       5528 |                 zipfile
import time:       599 |        599 |                 pkgutil
import time:      1913 |       1913 |                 platform
import time:        53 |         53 |                     math
import time:       120 |        120 |                     _datetime
import time:      1114 |       1286 |                   datetime
import time:      1129 |       1129 |                       xml
import time:      1078 |       2206 |                     xml.parsers
import time:       107 |        107 |                     pyexpat
import time:       431 |       2744 |                   xml.parsers.expat
import time:      1234 |       5263 |                 plistlib
import time:       578 |        578 |                   email
import time:       674 |        674 |                     email.errors
import time:        36 |         36 |                             _string
import time:       843 |        878 |                           string
import time:       465 |       1343 |                         email.quoprimime
import time:       524 |        524 |                           base64
import time:       402 |        925 |                         email.base64mime
import time:       451 |        451 |                             quopri
import time:       418 |        869 |                           email.encoders
import time:       874 |       1743 |                         email.charset
import time:       950 |       4960 |                       email.header
import time:        34 |         34 |                             _bisect
import time:       424 |        457 |                           bisect
import time:        27 |         27 |                           _sha512
import time:        22 |         22 |                           _random
import time:       616 |       1122 |                         random
import time:       136 |        136 |                           _socket
import time:       617 |        617 |                             select
import time:       885 |       1501 |                           selectors
import time:      1593 |       3229 |                         socket
import time:      1011 |       1011 |                             locale
import time:       919 |       1929 |                           calendar
import time:       557 |       2485 |                         email._parseaddr
import time:       813 |       7647 |                       email.utils
import time:       561 |      13167 |                     email._policybase
import time:       945 |      14785 |                   email.feedparser
import time:      8210 |      23572 |                 email.parser
import time:       721 |        721 |                 tempfile
import time:      1194 |       1194 |                 textwrap
import time:      6297 |       6297 |                 pkg_resources.extern
import time:      1130 |       1130 |                     pkg_resources._vendor
import time:      1168 |       2297 |                   pkg_resources._vendor.six
import time:       149 |       2446 |                 pkg_resources.extern.six
import time:       560 |        560 |                     pkg_resources._vendor.six
import time:        70 |        629 |                   pkg_resources._vendor.six.moves
import time:        56 |        685 |                 pkg_resources.extern.six.moves
import time:        20 |         20 |                   pkg_resources._vendor.six.moves
import time:        50 |         69 |                 pkg_resources._vendor.six.moves.urllib
import time:       461 |        461 |                 pkg_resources.py31compat
import time:       910 |        910 |                   pkg_resources._vendor.appdirs
import time:        81 |        990 |                 pkg_resources.extern.appdirs
import time:       436 |        436 |                     pkg_resources._vendor.packaging.__about__
import time:      3589 |       4025 |                   pkg_resources._vendor.packaging
import time:        72 |       4097 |                 pkg_resources.extern.packaging
import time:       378 |        378 |                   pkg_resources.extern.packaging._structures
import time:      2111 |       2489 |                 pkg_resources.extern.packaging.version
import time:       379 |        379 |                   pkg_resources.extern.packaging._compat
import time:      6610 |       6989 |                 pkg_resources.extern.packaging.specifiers
import time:        74 |         74 |                             org
import time:        17 |         90 |                           org.python
import time:        13 |        103 |                         org.python.core
import time:       539 |        641 |                       copy
import time:      1431 |       1431 |                       traceback
import time:     27210 |      29281 |                     pkg_resources._vendor.pyparsing
import time:       107 |      29388 |                   pkg_resources.extern.pyparsing
import time:        42 |         42 |                   pkg_resources.extern.six.moves.urllib
import time:      1463 |       1463 |                   pkg_resources.extern.packaging.markers
import time:      8461 |      39353 |                 pkg_resources.extern.packaging.requirements
import time:      4606 |       4606 |                 sysconfig
import time:     52302 |     159948 |               pkg_resources
import time:       338 |     160286 |             astroid.interpreter._import.util
import time:      1009 |     162569 |           astroid.interpreter._import.spec
import time:       614 |        614 |             distutils.errors
import time:       426 |        426 |                 distutils.dep_util
import time:       355 |        355 |                   distutils.debug
import time:       504 |        504 |                   distutils.log
import time:       548 |       1406 |                 distutils.spawn
import time:       559 |       2391 |               distutils.util
import time:       889 |       3280 |             distutils.sysconfig
import time:       854 |       4746 |           astroid.modutils
import time:       463 |        463 |           astroid.transforms
import time:      3030 |     172688 |         astroid.manager
import time:      4830 |     178888 |       astroid.bases
import time:      8974 |       8974 |       astroid.nodes.const
import time:       737 |        737 |         astroid.nodes.as_string
import time:      1306 |       2042 |       astroid.nodes.node_ng
import time:      5136 |     240752 |     astroid.nodes.node_classes
import time:       467 |        467 |       astroid.interpreter.dunder_lookup
import time:      1416 |       1883 |     astroid.nodes.scoped_nodes
import time:      4791 |     247424 |   astroid.nodes
import time:        57 |         57 |       _ast
import time:       764 |        820 |     ast
import time:    177912 |     177912 |       astroid.raw_building
import time:       656 |     178567 |     astroid.helpers
import time:       569 |        569 |       astroid.arguments
import time:       972 |       1541 |     astroid.protocols
import time:      1415 |     182342 |   astroid.inference
import time:      1575 |       1575 |   astroid.astroid_manager
import time:      1293 |       1293 |     astroid.brain
import time:       712 |       2005 |   astroid.brain.helpers
import time:        76 |         76 |           typed_ast
import time:        53 |        128 |         typed_ast.ast3
import time:       655 |        783 |       astroid._ast
import time:      1616 |       2399 |     astroid.rebuilder
import time:       561 |       2959 |   astroid.builder
import time:       355 |        355 |   astroid.inference_tip
import time:       701 |        701 |   astroid.objects
import time:       305 |        305 |   astroid.brain.brain_numpy_utils
import time:     46840 |     509498 | astroid
```

### Expected behavior

Faster import, so pylint's can analyses empty file in less than half a second.



Discussion / hints:
How can I learn as begineer
Hi @Keuricloud8866, what do you want to learn ? In order to fix this ticket you should track the place where setuptools and distutils are used in pylint and find an alternative. For finding alternative we'll probably have to discuss how to do what we were doing before without setuptools.
I am new and begineer in software IT world i just jumped in and want to be in so 
You should probably check a tutorial about learning programming first. Another useful thing would be to know how to ask question:
- First read the error or documentation
- Second search Google or Stackoverflow
- Third ask for help in the right place

Did you search how to learn programming by yourself first and read a blog about it ? I think you jumped to three and this is not the right place to ask either. This is an issue about a particular issue in a particular software, you were very unlikely to get answers here. (in fact please do not ask other off-topic questions or I'll have to ban you).
Do we still need to dependency on `setuptools` now? It was added in https://github.com/PyCQA/astroid/commit/40629baba2de2c9eb5e11b65798b3fae79f7284a.
Now that we have removed `distutils` I don't think it is necessary, right?
Probably ? I'm trying to think of a way to test that. Maybe launching pylint test suite in a venv after we deinstalled ``setuptools`` ?

Doing #1103 would remove the import time from ``pkg_resources`` and ``distutils`` which are non negligible.
Another possible culprit is the use of `NamedTuple`, see https://lwn.net/Articles/730915/



Fix (patch):
diff --git a/ChangeLog b/ChangeLog
index bd6ed51cda..06a7163ff6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,10 @@ Release date: TBA
 
   Closes #1512
 
+* Remove dependency on ``pkg_resources`` from ``setuptools``.
+
+  Closes #1103
+
 * Allowed ``AstroidManager.clear_cache`` to reload necessary brain plugins.
 
 * Rename ``ModuleSpec`` -> ``module_type`` constructor parameter to match attribute
diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py
index c0601613d9..74a0e8081f 100644
--- a/astroid/interpreter/_import/spec.py
+++ b/astroid/interpreter/_import/spec.py
@@ -10,6 +10,7 @@
 import importlib.machinery
 import importlib.util
 import os
+import pathlib
 import sys
 import zipimport
 from collections.abc import Sequence
@@ -147,7 +148,7 @@ def contribute_to_path(self, spec, processed):
             # Builtin.
             return None
 
-        if _is_setuptools_namespace(spec.location):
+        if _is_setuptools_namespace(Path(spec.location)):
             # extend_path is called, search sys.path for module/packages
             # of this name see pkgutil.extend_path documentation
             path = [
@@ -179,7 +180,7 @@ def contribute_to_path(self, spec, processed):
 
 
 class ExplicitNamespacePackageFinder(ImportlibFinder):
-    """A finder for the explicit namespace packages, generated through pkg_resources."""
+    """A finder for the explicit namespace packages."""
 
     def find_module(self, modname, module_parts, processed, submodule_path):
         if processed:
@@ -256,12 +257,12 @@ def contribute_to_path(self, spec, processed):
 )
 
 
-def _is_setuptools_namespace(location):
+def _is_setuptools_namespace(location: pathlib.Path) -> bool:
     try:
-        with open(os.path.join(location, "__init__.py"), "rb") as stream:
+        with open(location / "__init__.py", "rb") as stream:
             data = stream.read(4096)
     except OSError:
-        return None
+        return False
     else:
         extend_path = b"pkgutil" in data and b"extend_path" in data
         declare_namespace = (
diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py
index ce3da7eac2..53c6922c33 100644
--- a/astroid/interpreter/_import/util.py
+++ b/astroid/interpreter/_import/util.py
@@ -2,15 +2,35 @@
 # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE
 # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt
 
-try:
-    import pkg_resources
-except ImportError:
-    pkg_resources = None  # type: ignore[assignment]
+import sys
+from functools import lru_cache
+from importlib.util import _find_spec_from_path
 
 
-def is_namespace(modname):
-    return (
-        pkg_resources is not None
-        and hasattr(pkg_resources, "_namespace_packages")
-        and modname in pkg_resources._namespace_packages
-    )
+@lru_cache(maxsize=4096)
+def is_namespace(modname: str) -> bool:
+    if modname in sys.builtin_module_names:
+        return False
+
+    found_spec = None
+
+    # find_spec() attempts to import parent packages when given dotted paths.
+    # That's unacceptable here, so we fallback to _find_spec_from_path(), which does
+    # not, but requires instead that each single parent ('astroid', 'nodes', etc.)
+    # be specced from left to right.
+    processed_components = []
+    last_parent = None
+    for component in modname.split("."):
+        processed_components.append(component)
+        working_modname = ".".join(processed_components)
+        try:
+            found_spec = _find_spec_from_path(working_modname, last_parent)
+        except ValueError:
+            # executed .pth files may not have __spec__
+            return True
+        last_parent = working_modname
+
+    if found_spec is None:
+        return False
+
+    return found_spec.origin is None
diff --git a/astroid/manager.py b/astroid/manager.py
index 23330d5b95..6e7dcf4cdf 100644
--- a/astroid/manager.py
+++ b/astroid/manager.py
@@ -17,7 +17,7 @@
 
 from astroid.const import BRAIN_MODULES_DIRECTORY
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
-from astroid.interpreter._import import spec
+from astroid.interpreter._import import spec, util
 from astroid.modutils import (
     NoSourceFile,
     _cache_normalize_path_,
@@ -382,6 +382,7 @@ def clear_cache(self) -> None:
         for lru_cache in (
             LookupMixIn.lookup,
             _cache_normalize_path_,
+            util.is_namespace,
             ObjectModel.attributes,
         ):
             lru_cache.cache_clear()
diff --git a/setup.cfg b/setup.cfg
index fa7c436e00..1e2341a882 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -39,7 +39,6 @@ packages = find:
 install_requires =
     lazy_object_proxy>=1.4.0
     wrapt>=1.11,<2
-    setuptools>=20.0
     typed-ast>=1.4.0,<2.0;implementation_name=="cpython" and python_version<"3.8"
     typing-extensions>=3.10;python_version<"3.10"
 python_requires = >=3.7.2


Test changes:
diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py
index cd63950ea2..6c105a12bb 100644
--- a/tests/unittest_manager.py
+++ b/tests/unittest_manager.py
@@ -10,12 +10,11 @@
 from collections.abc import Iterator
 from contextlib import contextmanager
 
-import pkg_resources
-
 import astroid
 from astroid import manager, test_utils
 from astroid.const import IS_JYTHON
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
+from astroid.interpreter._import import util
 from astroid.modutils import is_standard_module
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef
@@ -111,6 +110,16 @@ def test_ast_from_namespace_pkgutil(self) -> None:
     def test_ast_from_namespace_pkg_resources(self) -> None:
         self._test_ast_from_old_namespace_package_protocol("pkg_resources")
 
+    def test_identify_old_namespace_package_protocol(self) -> None:
+        # Like the above cases, this package follows the old namespace package protocol
+        # astroid currently assumes such packages are in sys.modules, so import it
+        # pylint: disable-next=import-outside-toplevel
+        import tests.testdata.python3.data.path_pkg_resources_1.package.foo as _  # noqa
+
+        self.assertTrue(
+            util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1")
+        )
+
     def test_implicit_namespace_package(self) -> None:
         data_dir = os.path.dirname(resources.find("data/namespace_pep_420"))
         contribute = os.path.join(data_dir, "contribute_to_namespace")
@@ -131,7 +140,6 @@ def test_implicit_namespace_package(self) -> None:
     def test_namespace_package_pth_support(self) -> None:
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = []
 
         try:
             module = self.manager.ast_from_module_name("foogle.fax")
@@ -141,18 +149,14 @@ def test_namespace_package_pth_support(self) -> None:
             with self.assertRaises(AstroidImportError):
                 self.manager.ast_from_module_name("foogle.moogle")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def test_nested_namespace_import(self) -> None:
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = ["foogle.crank"]
-        pkg_resources._namespace_packages["foogle.crank"] = []
         try:
             self.manager.ast_from_module_name("foogle.crank")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def test_namespace_and_file_mismatch(self) -> None:
@@ -161,12 +165,10 @@ def test_namespace_and_file_mismatch(self) -> None:
         self.assertEqual(ast.name, "unittest")
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = []
         try:
             with self.assertRaises(AstroidImportError):
                 self.manager.ast_from_module_name("unittest.foogle.fax")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def _test_ast_from_zip(self, archive: str) -> None:
@@ -323,6 +325,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
         lrus = (
             astroid.nodes.node_classes.LookupMixIn.lookup,
             astroid.modutils._cache_normalize_path_,
+            util.is_namespace,
             astroid.interpreter.objectmodel.ObjectModel.attributes,
         )
 
@@ -332,6 +335,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
         # Generate some hits and misses
         ClassDef().lookup("garbage")
         is_standard_module("unittest", std_path=["garbage_path"])
+        util.is_namespace("unittest")
         astroid.interpreter.objectmodel.ObjectModel().attributes()
 
         # Did the hits or misses actually happen?