Repository: pypa/pip
Pull request #10654 (2021-11-12 12:47:14)

Allow to decide whether to use distutils or sysconfig with sysconfig._PIP_USE_SYSCONFIG
<!---
Thank you for your soon to be pull request. Before you submit this, please
double check to make sure that you've added a news file fragment. In pip we
generate our NEWS.rst from multiple news fragment files, and all pull requests
require either a news file fragment or a marker to indicate they don't require
one.

To read more about adding a news file fragment for your PR, please check out
our documentation at: https://pip.pypa.io/en/latest/development/contributing/#news-entries
-->

(description of the change, taken from 63b50517653adf8bc02faa16e524d05e4dabcd64)

> Allow Python distributors to opt-out from or opt-in to the ``sysconfig`` installation scheme backend. By default, the installation scheme backend is ``sysconfig`` on Python 3.10 or later, but by setting the ``sysconfig._PIP_USE_SYSCONFIG`` the distributors can override this default: The boolean value of this private ``sysconfig`` attribute will take precedence over the default if set. This can be used for compatibility considerations as well as for opting in for more leading-edge technology if so desired. Distributors who set ``sysconfig._PIP_USE_SYSCONFIG = False`` are strongly encouraged to ensure ``distutils`` is present, e.g. it is not recommended to set this when ``distutils`` was removed from the standard library, as it will result in failures.

Fixes https://github.com/pypa/pip/issues/10647

pradyunsg: One option if you want to fully test this would be to factor this expression out into a function, and the instrument the dependencies of this function.

```py
def _should_use_sysconfig():
   ...

# This is a function for testability, but should be constant during any one run.
_USE_SYSCONFIG = _should_use_sysconfig()
```

hroncok: OK, this works as expected. Will paly with the tests next week.

Patch:
diff --git a/news/10557.bugfix.rst b/news/10557.bugfix.rst
deleted file mode 100644
index 5a6a4ef26..000000000
--- a/news/10557.bugfix.rst
+++ /dev/null
@@ -1,1 +0,0 @@
-Optimize installation order calculation to improve performance when installing requirements that form a complex dependency graph with a large amount of edges.
diff --git a/news/10647.feature.rst b/news/10647.feature.rst
new file mode 100644
index 000000000..ad9d06fbb
--- /dev/null
+++ b/news/10647.feature.rst
@@ -0,0 +1,3 @@
+Allow Python distributors to opt-out from or opt-in to the ``sysconfig``
+installation scheme backend by setting ``sysconfig._PIP_USE_SYSCONFIG``
+to ``True`` or ``False``.
diff --git a/src/pip/_internal/locations/__init__.py b/src/pip/_internal/locations/__init__.py
index dba182d75..729c7548d 100644
--- a/src/pip/_internal/locations/__init__.py
+++ b/src/pip/_internal/locations/__init__.py
@@ -45,7 +45,22 @@ else:
 
 _PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
 
-_USE_SYSCONFIG = sys.version_info >= (3, 10)
+
+def _should_use_sysconfig() -> bool:
+    """
+    This function determines the value of _USE_SYSCONFIG.
+    By default, pip uses sysconfig on Python 3.10+.
+    But Python distributors can override this decision by setting:
+        sysconfig._PIP_USE_SYSCONFIG = True / False
+    Rationale in https://github.com/pypa/pip/issues/10647
+    """
+    if hasattr(sysconfig, "_PIP_USE_SYSCONFIG"):
+        return bool(sysconfig._PIP_USE_SYSCONFIG)  # type: ignore [attr-defined]
+    return sys.version_info >= (3, 10)
+
+
+# This is a function for testability, but should be constant during any one run.
+_USE_SYSCONFIG = _should_use_sysconfig()
 
 
 def _looks_like_bpo_44860() -> bool:
diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py
index 8ee36d377..12f967020 100644
--- a/src/pip/_internal/resolution/resolvelib/resolver.py
+++ b/src/pip/_internal/resolution/resolvelib/resolver.py
@@ -171,17 +171,12 @@ class Resolver(BaseResolver):
         get installed one-by-one.
 
         The current implementation creates a topological ordering of the
-        dependency graph, giving more weight to packages with less
-        or no dependencies, while breaking any cycles in the graph at
-        arbitrary points. We make no guarantees about where the cycle
-        would be broken, other than it *would* be broken.
+        dependency graph, while breaking any cycles in the graph at arbitrary
+        points. We make no guarantees about where the cycle would be broken,
+        other than they would be broken.
         """
         assert self._result is not None, "must call resolve() first"
 
-        if not req_set.requirements:
-            # Nothing is left to install, so we do not need an order.
-            return []
-
         graph = self._result.graph
         weights = get_topological_weights(
             graph,
@@ -204,19 +199,13 @@ def get_topological_weights(
     This implementation may change at any point in the future without prior
     notice.
 
-    We first simplify the dependency graph by pruning any leaves and giving them
-    the highest weight: a package without any dependencies should be installed
-    first. This is done again and again in the same way, giving ever less weight
-    to the newly found leaves. The loop stops when no leaves are left: all
-    remaining packages have at least one dependency left in the graph.
-
-    Then we continue with the remaining graph, by taking the length for the
-    longest path to any node from root, ignoring any paths that contain a single
-    node twice (i.e. cycles). This is done through a depth-first search through
-    the graph, while keeping track of the path to the node.
+    We take the length for the longest path to any node from root, ignoring any
+    paths that contain a single node twice (i.e. cycles). This is done through
+    a depth-first search through the graph, while keeping track of the path to
+    the node.
 
     Cycles in the graph result would result in node being revisited while also
-    being on its own path. In this case, take no action. This helps ensure we
+    being it's own path. In this case, take no action. This helps ensure we
     don't get stuck in a cycle.
 
     When assigning weight, the longer path (i.e. larger length) is preferred.
@@ -238,34 +227,6 @@ def get_topological_weights(
         last_known_parent_count = weights.get(node, 0)
         weights[node] = max(last_known_parent_count, len(path))
 
-    # Simplify the graph, pruning leaves that have no dependencies.
-    # This is needed for large graphs (say over 200 packages) because the
-    # `visit` function is exponentially slower then, taking minutes.
-    # See https://github.com/pypa/pip/issues/10557
-    # We will loop until we explicitly break the loop.
-    while True:
-        leaves = set()
-        for key in graph:
-            if key is None:
-                continue
-            for _child in graph.iter_children(key):
-                # This means we have at least one child
-                break
-            else:
-                # No child.
-                leaves.add(key)
-        if not leaves:
-            # We are done simplifying.
-            break
-        # Calculate the weight for the leaves.
-        weight = len(graph) - 1
-        for leaf in leaves:
-            weights[leaf] = weight
-        # Remove the leaves from the graph, making it simpler.
-        for leaf in leaves:
-            graph.remove(leaf)
-
-    # Visit the remaining graph.
     # `None` is guaranteed to be the root node by resolvelib.
     visit(None)
 


Test patch:
diff --git a/tests/unit/resolution_resolvelib/test_resolver.py b/tests/unit/resolution_resolvelib/test_resolver.py
index 579195b55..1fcde34a4 100644
--- a/tests/unit/resolution_resolvelib/test_resolver.py
+++ b/tests/unit/resolution_resolvelib/test_resolver.py
@@ -115,7 +115,7 @@ def test_new_resolver_get_installation_order(
                 ("three", "four"),
                 ("four", "five"),
             ],
-            {None: 0, "five": 5, "four": 4, "one": 4, "three": 2, "two": 1},
+            {None: 0, "one": 1, "two": 1, "three": 2, "four": 3, "five": 4},
         ),
         (
             "linear",
diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py
index 640be7f0d..48de5f860 100644
--- a/tests/unit/test_locations.py
+++ b/tests/unit/test_locations.py
@@ -6,13 +6,14 @@ import getpass
 import os
 import shutil
 import sys
+import sysconfig
 import tempfile
 from typing import Any, Dict
 from unittest.mock import Mock
 
 import pytest
 
-from pip._internal.locations import SCHEME_KEYS, get_scheme
+from pip._internal.locations import SCHEME_KEYS, _should_use_sysconfig, get_scheme
 from tests.lib.path import Path
 
 if sys.platform == "win32":
@@ -82,6 +83,24 @@ class TestLocations:
         result.pw_name = self.username
         return result
 
+    def test_default_should_use_sysconfig(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.delattr(sysconfig, "_PIP_USE_SYSCONFIG", raising=False)
+        if sys.version_info[:2] >= (3, 10):
+            assert _should_use_sysconfig() is True
+        else:
+            assert _should_use_sysconfig() is False
+
+    @pytest.mark.parametrize("vendor_value", [True, False, None, "", 0, 1])
+    def test_vendor_overriden_should_use_sysconfig(
+        self, monkeypatch: pytest.MonkeyPatch, vendor_value: Any
+    ) -> None:
+        monkeypatch.setattr(
+            sysconfig, "_PIP_USE_SYSCONFIG", vendor_value, raising=False
+        )
+        assert _should_use_sysconfig() is bool(vendor_value)
+
 
 class TestDistutilsScheme:
     def test_root_modifies_appropriately(self) -> None: