Repository: pypa/pip
Pull request #10358 (2021-08-13 10:14:51)

Switch install scheme backend to sysconfig
Now we’ve weeded out (almost) all the known distutils-sysconfig inompatibilities, let’s make the switch? Installations running on Python 3.9 or lower are not affected. Only Python 3.10 and up will use sysconfig.

Right now I’m targeting 21.3 although this is a bit awkward since CPython 3.10 is scheduled to be released slightly before that. But that’s probably a good thing since we can at least be sure the pip bundled in 3.10.0 is “guaranteed” to work? We can always bundle this switch in the first 3.10.x patch release anyway.

uranusjr: Side note: CPython currently has an unresolved bug that installs user-site platform-dependent packages (`platlib`) into a wrong location on certain platforms e.g. Fedora 34 ([bpo-44860](https://bugs.python.org/issue44860)). I’m pretty sure this will be fixed before 3.10.0 proper is released, but if it’s not, that’s CPython’s problem.
pradyunsg: Do we need to wait for https://github.com/python/cpython/pull/27655 to be released then?

https://github.com/pypa/pip/runs/3321341848#step:5:2976 seems to be an esoteric error. :/

uranusjr: > [pypa/pip/runs/3321341848#step:5:2976](https://github.com/pypa/pip/runs/3321341848#step:5:2976) seems to be an esoteric error. :/

Huh, what the hell? This also seems to be a pretty environment-specific thing—the Windows tests are all passing, including the 3.10 ones.
pradyunsg: Gonna move the failure output to a comment, in case it's useful for future reference; and re-running the build.

```
_________________ TestInstallUnpackedWheel.test_install_prefix _________________
[gw0] linux -- Python 3.10.0 /home/runner/work/pip/pip/.tox/py/bin/python

self = <tests.unit.test_wheel.TestInstallUnpackedWheel object at 0x7f4125518df0>
data = <tests.lib.TestData object at 0x7f41253b71f0>
tmpdir = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0')

    def test_install_prefix(self, data, tmpdir):
        prefix = os.path.join(os.path.sep, 'some', 'path')
        self.prep(data, tmpdir)
        scheme = get_scheme(
            self.name,
            user=False,
            home=None,
            root=tmpdir,
            isolated=False,
            prefix=prefix,
        )
>       wheel.install_wheel(
            self.name,
            self.wheelpath,
            scheme=scheme,
            req_description=str(self.req),
        )

data       = <tests.lib.TestData object at 0x7f41253b71f0>
prefix     = '/some/path'
scheme     = <pip._internal.models.scheme.Scheme object at 0x7f41253d2d90>
self       = <tests.unit.test_wheel.TestInstallUnpackedWheel object at 0x7f4125518df0>
tmpdir     = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0')

tests/unit/test_wheel.py:411: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/py/lib/python3.10/site-packages/pip/_internal/operations/install/wheel.py:762: in install_wheel
    _install_wheel(
        direct_url = None
        name       = 'sample'
        pycompile  = True
        req_description = 'sample'
        requested  = False
        scheme     = <pip._internal.models.scheme.Scheme object at 0x7f41253d2d90>
        warn_script_location = True
        wheel_path = '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/sample-1.2.0-py2.py3-none-any.whl'
        z          = <zipfile.ZipFile [closed]>
.tox/py/lib/python3.10/site-packages/pip/_internal/operations/install/wheel.py:644: in _install_wheel
    success = compileall.compile_file(path, force=True, quiet=True)
        all_paths  = <function _install_wheel.<locals>.all_paths at 0x7f41253f4f70>
        assert_no_path_traversal = <function _install_wheel.<locals>.assert_no_path_traversal at 0x7f41253f4e50>
        changed    = set()
        console    = {'sample': 'sample:main'}
        data_scheme_file_maker = <function _install_wheel.<locals>.data_scheme_file_maker at 0x7f41253f4790>
        data_scheme_paths = <filter object at 0x7f41253b7c70>
        direct_url = None
        distribution = <pip._internal.metadata.pkg_resources.Distribution object at 0x7f41253b7430>
        file       = <pip._internal.operations.install.wheel.ZipBackedFile object at 0x7f41253b7160>
        file_paths = <itertools.filterfalse object at 0x7f41253b7ca0>
        files      = <itertools.chain object at 0x7f41253b7af0>
        generated  = []
        gui        = {'sample2': 'sample:main'}
        info_dir   = 'sample-1.2.0.dist-info'
        installed  = {'sample-1.2.0.data/data/my_data/data_file': '../../../my_data/data_file', 'sample-1.2.0.dist-info/DESCRIPTION.rst': '...fo/METADATA': 'sample-1.2.0.dist-info/METADATA', 'sample-1.2.0.dist-info/RECORD': 'sample-1.2.0.dist-info/RECORD', ...}
        is_data_scheme_path = <function _install_wheel.<locals>.is_data_scheme_path at 0x7f41253f48b0>
        is_dir_path = <function _install_wheel.<locals>.is_dir_path at 0x7f41253f4d30>
        is_entrypoint_wrapper = <function _install_wheel.<locals>.is_entrypoint_wrapper at 0x7f41253f4af0>
        is_script_scheme_path = <function _install_wheel.<locals>.is_script_scheme_path at 0x7f41253f4a60>
        lib_dir    = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages')
        make_data_scheme_file = <function _install_wheel.<locals>.data_scheme_file_maker.<locals>.make_data_scheme_file at 0x7f41253f4ca0>
        make_root_scheme_file = <function _install_wheel.<locals>.root_scheme_file_maker.<locals>.make_root_scheme_file at 0x7f41253f4160>
        metadata   = <email.message.Message object at 0x7f41253b6ec0>
        name       = 'sample'
        other_scheme_files = <map object at 0x7f41253b7e80>
        other_scheme_paths = <itertools.filterfalse object at 0x7f41253b7550>
        path       = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py')
        paths      = <generator object _install_wheel.<locals>.all_paths at 0x7f4125d21540>
        pyc_output_path = <function _install_wheel.<locals>.pyc_output_path at 0x7f41253f4040>
        pyc_source_file_paths = <function _install_wheel.<locals>.pyc_source_file_paths at 0x7f41253f41f0>
        pycompile  = True
        record_installed = <function _install_wheel.<locals>.record_installed at 0x7f41253f55a0>
        requested  = False
        root_scheme_file_maker = <function _install_wheel.<locals>.root_scheme_file_maker at 0x7f41253f4dc0>
        root_scheme_paths = <itertools.filterfalse object at 0x7f41253b6d10>
        scheme     = <pip._internal.models.scheme.Scheme object at 0x7f41253d2d90>
        script_scheme_files = <map object at 0x7f41253b7f10>
        script_scheme_paths = <filter object at 0x7f41253b79a0>
        stdout     = <pip._internal.utils.misc.StreamWrapper object at 0x7f41253f49d0>
        warn_script_location = True
        wheel_path = '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/sample-1.2.0-py2.py3-none-any.whl'
        wheel_zip  = <zipfile.ZipFile [closed]>
/opt/hostedtoolcache/Python/3.10.0-rc.1/x64/lib/python3.10/compileall.py:240: in compile_file
    ok = py_compile.compile(fullname, cfile, dfile, True,
        cfile      = '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__pycache__/__init__.cpython-310.pyc'
        ddir       = None
        dfile      = None
        force      = True
        fullname   = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py')
        hardlink_dupes = False
        head       = '__init__'
        index      = 0
        invalidation_mode = None
        legacy     = False
        limit_sl_dest = None
        name       = '__init__.py'
        opt_cfiles = {-1: '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__pycache__/__init__.cpython-310.pyc'}
        opt_level  = -1
        optimize   = [-1]
        prependdir = None
        quiet      = True
        rx         = None
        stripdir   = None
        success    = True
        tail       = '.py'
/opt/hostedtoolcache/Python/3.10.0-rc.1/x64/lib/python3.10/py_compile.py:162: in compile
    bytecode = importlib._bootstrap_external._code_to_timestamp_pyc(
        cfile      = '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__pycache__/__init__.cpython-310.pyc'
        code       = <code object <module> at 0x7f41253e3470, file "/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py", line 1>
        dfile      = None
        dirname    = '/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__pycache__'
        doraise    = True
        file       = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py')
        invalidation_mode = <PycInvalidationMode.TIMESTAMP: 1>
        loader     = <_frozen_importlib_external.SourceFileLoader object at 0x7f41253b7c40>
        optimize   = -1
        quiet      = 0
        source_bytes = b'\n__version__ = \'1.2.0\'\n\ndef main():\n    """Entry point for the application script"""\n    print("Call your main application code here")\n'
        source_stats = {'mtime': 1628852640.7727487, 'size': 135}
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

code = <code object <module> at 0x7f41253e3470, file "/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py", line 1>
mtime = 1628852640.7727487, source_size = 135

>   ???
E   ValueError: unmarshallable object

code       = <code object <module> at 0x7f41253e3470, file "/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py", line 1>
data       = bytearray(b'o\r\r\n\x00\x00\x00\x00\xa0Q\x16a\x87\x00\x00\x00')
mtime      = 1628852640.7727487
source_size = 135

<frozen importlib._bootstrap_external>:689: ValueError
```
pradyunsg: Maybe @brettcannon has an idea of what's up?

pradyunsg: > re-running the build.

Definitely not a flaky failure. :/
brettcannon: Is the code marshalled on the fly or somehow checked into the repo? If it's the former it may be a bug, if it's the latter then it's probably due to the marshal format being changed.

I know Guido and the perf team have been poking around in marshal lately but I don't know if any of their PRs landed upstream in 3.10.
pradyunsg: There's no marshal'd contented checked in to this repository. This file is generated on each test run, and compiled using `compileall.compile_file(path, force=True, quiet=True)`, where path is:

```
path       = Path('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py')
```

The contents are from the link above, and can be confirmed in the "showlocals" output from pytest, that's augumented the traceback.

The `ValueError` seems to bubble up from `importlib._bootstrap_external._code_to_timestamp_pyc`.

brettcannon: @pradyunsg if you can get a small reproducer then please open a bug at bugs.python.org.
uranusjr: So I tried compiling CPython from source (I’m not sure the exact version GitHub is using so tested with rc) and this test seems to run fine. Since all we do here is switching from distutils to sysconfig, and all that affects is potentially changing some path values pip uses to install things to, I’m suspecting this is an environment issue. Maybe GHA’s Python setup makes sysconfig point to a different location from distutils that somehow cannot be correctly pycompile-d 😟 
uranusjr: Err, the paths are the same as far as I can tell?

https://github.com/pypa/pip/pull/10358/checks?check_run_id=3373309381#step:5:3012
pradyunsg: Did a round of debugging by ssh'ing into the GHA runner. Turns out, somehow, `tests.lib.path.Path` objects are not marshallable in 3.10 + GHA?

```
(Pdb) code.co_consts[1].co_filename
Path('/tmp/pytest-of-runner/pytest-0/test_install_prefix0/some/path/lib/python3.10/site-packages/sample/__init__.py')
(Pdb) code.co_consts[1].co_filename.__class__
<class 'tests.lib.path.Path'>
(Pdb) marshal.dumps(code.co_consts[1].co_filename)
*** ValueError: unmarshallable object
```

This looks like it's super specific to pip's test suite. The fix for us, will be to change `root=tmpdir,` to `root=str(tmpdir)`.

I'd day #9578 has became a bit more important now. :)
pradyunsg: Specifically, the patch is:

```diff
diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py
index 3b39f91b97c..863b4ddb011 100644
--- a/tests/unit/test_wheel.py
+++ b/tests/unit/test_wheel.py
@@ -404,7 +404,7 @@ def test_install_prefix(self, data, tmpdir):
             self.name,
             user=False,
             home=None,
-            root=tmpdir,
+            root=str(tmpdir),
             isolated=False,
             prefix=prefix,
         )
```
uranusjr: We really should switch to `pathlib.Path`.
uranusjr: I do wonder why the class suddenly becomes unmarshallable though. Were we depending on some internals we shouldn’t (and 3.10 exposes our mistake), or is 3.10 breaking compatibility intentially? But that’s not pip-related so I’ll do it in my own free time (wait, but I’m doing pip in my free time anyway, what’s the difference).

Patch:
diff --git a/news/10358.removal.rst b/news/10358.removal.rst
new file mode 100644
index 000000000..727631cd3
--- /dev/null
+++ b/news/10358.removal.rst
@@ -0,0 +1,4 @@
+On Python 3.10 or later, the installation scheme backend has been changed to use
+``sysconfig``. This is to anticipate the deprecation of ``distutils`` in Python
+3.10, and its scheduled removal in 3.12. For compatibility considerations, pip
+installations running on Python 3.9 or lower will continue to use ``distutils``.
diff --git a/setup.cfg b/setup.cfg
index d5dfb587e..be58a126b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -57,6 +57,7 @@ follow_imports = skip
 addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes
 markers =
     network: tests that need network
+    incompatible_with_sysconfig
     incompatible_with_test_venv
     incompatible_with_venv
     no_auto_tempdir_manager
diff --git a/src/pip/_internal/locations/__init__.py b/src/pip/_internal/locations/__init__.py
index d74857b2a..4f3fdf4a9 100644
--- a/src/pip/_internal/locations/__init__.py
+++ b/src/pip/_internal/locations/__init__.py
@@ -45,6 +45,8 @@ else:
 
 _PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
 
+_USE_SYSCONFIG = sys.version_info >= (3, 10)
+
 
 def _looks_like_bpo_44860() -> bool:
     """The resolution to bpo-44860 will change this incorrect platlib.
@@ -190,7 +192,7 @@ def get_scheme(
     isolated: bool = False,
     prefix: Optional[str] = None,
 ) -> Scheme:
-    old = _distutils.get_scheme(
+    new = _sysconfig.get_scheme(
         dist_name,
         user=user,
         home=home,
@@ -198,7 +200,10 @@ def get_scheme(
         isolated=isolated,
         prefix=prefix,
     )
-    new = _sysconfig.get_scheme(
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_scheme(
         dist_name,
         user=user,
         home=home,
@@ -335,8 +340,11 @@ def get_scheme(
 
 
 def get_bin_prefix() -> str:
-    old = _distutils.get_bin_prefix()
     new = _sysconfig.get_bin_prefix()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_bin_prefix()
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
         _log_context()
     return old
@@ -365,8 +373,11 @@ def _looks_like_deb_system_dist_packages(value: str) -> bool:
 
 def get_purelib() -> str:
     """Return the default pure-Python lib location."""
-    old = _distutils.get_purelib()
     new = _sysconfig.get_purelib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_purelib()
     if _looks_like_deb_system_dist_packages(old):
         return old
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"):
@@ -376,8 +387,11 @@ def get_purelib() -> str:
 
 def get_platlib() -> str:
     """Return the default platform-shared lib location."""
-    old = _distutils.get_platlib()
     new = _sysconfig.get_platlib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_platlib()
     if _looks_like_deb_system_dist_packages(old):
         return old
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
@@ -385,10 +399,20 @@ def get_platlib() -> str:
     return old
 
 
+def _deduplicated(v1: str, v2: str) -> List[str]:
+    """Deduplicate values from a list."""
+    if v1 == v2:
+        return [v1]
+    return [v1, v2]
+
+
 def get_prefixed_libs(prefix: str) -> List[str]:
     """Return the lib locations under ``prefix``."""
-    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
     new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
+    if _USE_SYSCONFIG:
+        return _deduplicated(new_pure, new_plat)
+
+    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
 
     warned = [
         _warn_if_mismatch(
@@ -405,6 +429,4 @@ def get_prefixed_libs(prefix: str) -> List[str]:
     if any(warned):
         _log_context(prefix=prefix)
 
-    if old_pure == old_plat:
-        return [old_pure]
-    return [old_pure, old_plat]
+    return _deduplicated(old_pure, old_plat)
diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py
index 1d3c30336..29437da2a 100644
--- a/src/pip/_internal/operations/install/wheel.py
+++ b/src/pip/_internal/operations/install/wheel.py
@@ -628,10 +628,7 @@ def _install_wheel(
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore")
                 for path in pyc_source_file_paths():
-                    # Python 2's `compileall.compile_file` requires a str in
-                    # error cases, so we must convert to the native type.
-                    path_arg = ensure_str(path, encoding=sys.getfilesystemencoding())
-                    success = compileall.compile_file(path_arg, force=True, quiet=True)
+                    success = compileall.compile_file(path, force=True, quiet=True)
                     if success:
                         pyc_path = pyc_output_path(path)
                         assert os.path.exists(pyc_path)


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index af567e5f1..520d0cc72 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -15,6 +15,7 @@ import pytest
 from setuptools.wheel import Wheel
 
 from pip._internal.cli.main import main as pip_entry_point
+from pip._internal.locations import _USE_SYSCONFIG
 from pip._internal.utils.temp_dir import global_tempdir_manager
 from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData
 from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key
@@ -77,6 +78,9 @@ def pytest_collection_modifyitems(config, items):
         ):
             item.add_marker(pytest.mark.skip("Incompatible with venv"))
 
+        if item.get_closest_marker("incompatible_with_sysconfig") and _USE_SYSCONFIG:
+            item.add_marker(pytest.mark.skip("Incompatible with sysconfig"))
+
         module_path = os.path.relpath(
             item.module.__file__,
             os.path.commonprefix([__file__, item.module.__file__]),
diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py
index b605f2c6a..31308cf80 100644
--- a/tests/unit/test_locations.py
+++ b/tests/unit/test_locations.py
@@ -96,6 +96,7 @@ class TestDistutilsScheme:
             expected = os.path.join(root, path[1:])
             assert os.path.abspath(root_scheme[key]) == expected
 
+    @pytest.mark.incompatible_with_sysconfig
     @pytest.mark.incompatible_with_venv
     def test_distutils_config_file_read(self, tmpdir, monkeypatch):
         # This deals with nt/posix path differences
@@ -116,10 +117,11 @@ class TestDistutilsScheme:
         scheme = _get_scheme_dict("example")
         assert scheme["scripts"] == install_scripts
 
+    @pytest.mark.incompatible_with_sysconfig
     @pytest.mark.incompatible_with_venv
     # when we request install-lib, we should install everything (.py &
     # .so) into that path; i.e. ensure platlib & purelib are set to
-    # this path
+    # this path. sysconfig does not support this.
     def test_install_lib_takes_precedence(self, tmpdir, monkeypatch):
         # This deals with nt/posix path differences
         install_lib = os.path.normcase(
diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py
index 247e8d664..e21c0ec2e 100644
--- a/tests/unit/test_wheel.py
+++ b/tests/unit/test_wheel.py
@@ -422,7 +422,7 @@ class TestInstallUnpackedWheel:
             self.name,
             user=False,
             home=None,
-            root=tmpdir,
+            root=str(tmpdir),  # Casting needed for CPython 3.10+. See GH-10358.
             isolated=False,
             prefix=prefix,
         )