Repository: pypa/pip
Pull request #11257 (2022-07-14 12:47:50)

Speedup isolated environment creation
Based on my tests, on an M1 MacBook Air, this improved `pip install ../furo --no-deps` from 12-13s to 8-9s. Both things were run interlaced, 4 times, ignoring the first run of each.

I imagine this translates to full test suite runs as well, but the numbers I have are skewed since doing a test run makes my laptop heat up, resulting in signficant-enough heat-related throttling that would skew the results.

pradyunsg: @pfmoore and @sbidoul I've incorporated your suggestions and added a news file. Please take a look again! :)


Patch:
diff --git a/docs/html/topics/vcs-support.md b/docs/html/topics/vcs-support.md
index 70bb5beb9..538b4a791 100644
--- a/docs/html/topics/vcs-support.md
+++ b/docs/html/topics/vcs-support.md
@@ -73,16 +73,16 @@ The supported schemes are `svn`, `svn+svn`, `svn+http`, `svn+https` and
 `svn+ssh`. Here are some of the supported forms:
 
 ```none
-MyProject @ svn+https://svn.example.com/MyProject
-MyProject @ svn+ssh://svn.example.com/MyProject
-MyProject @ svn+ssh://user@svn.example.com/MyProject
+MyProject @svn+https://svn.example.com/MyProject
+MyProject @svn+ssh://svn.example.com/MyProject
+MyProject @svn+ssh://user@svn.example.com/MyProject
 ```
 
 You can also give specific revisions to an SVN URL, like so:
 
 ```none
--e svn+http://svn.example.com/svn/MyProject/trunk@2019#egg=MyProject
--e svn+http://svn.example.com/svn/MyProject/trunk@{20080101}#egg=MyProject
+MyProject @ -e svn+http://svn.example.com/svn/MyProject/trunk@2019
+MyProject @ -e svn+http://svn.example.com/svn/MyProject/trunk@{20080101}
 ```
 
 Note that you need to use [Editable VCS installs](#editable-vcs-installs) for
diff --git a/news/11225.feature.rst b/news/11225.feature.rst
index 2da8f5617..4e1ecec50 100644
--- a/news/11225.feature.rst
+++ b/news/11225.feature.rst
@@ -1,3 +1,3 @@
-pip's deprecation warnings now subclass the built-in ``DeprecationWarning``, and
+pip's deprecation warnings now suclass the built-in ``DeprecationWarning``, and
 can be suppressed by running the Python interpreter with
 ``-W ignore::DeprecationWarning``.
diff --git a/news/11241.removal.rst b/news/11241.removal.rst
deleted file mode 100644
index 69fb01866..000000000
--- a/news/11241.removal.rst
+++ /dev/null
@@ -1,1 +0,0 @@
-Remove ``--use-deprecated=backtrack-on-build-failures``.
diff --git a/news/11257.feature.rst b/news/11257.feature.rst
new file mode 100644
index 000000000..1cb75b228
--- /dev/null
+++ b/news/11257.feature.rst
@@ -0,0 +1,3 @@
+Significantly speed up isolated environment creation, by using the same
+sources for pip instead of creating a standalone installation for each
+environment.
diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py
index ccf2b4bc4..4554b1e4f 100644
--- a/src/pip/_internal/build_env.py
+++ b/src/pip/_internal/build_env.py
@@ -7,7 +7,6 @@ import os
 import pathlib
 import sys
 import textwrap
-import zipfile
 from collections import OrderedDict
 from sysconfig import get_paths
 from types import TracebackType
@@ -29,6 +28,29 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+PIP_RUNNER = """
+import importlib.util
+import os
+import runpy
+import sys
+
+
+class PipImportRedirectingFinder:
+
+    @classmethod
+    def find_spec(cls, fullname, path=None, target=None):
+        if not fullname.startswith("pip."):
+            return None
+
+        # Import pip from the current source directory
+        location = os.path.join({source!r}, *fullname.split("."))
+        return importlib.util.spec_from_file_location(fullname, location)
+
+
+sys.meta_path.insert(0, PipImportRedirectingFinder())
+runpy.run_module("pip", run_name="__main__")
+"""
+
 
 class _Prefix:
     def __init__(self, path: str) -> None:
@@ -42,29 +64,25 @@ class _Prefix:
 
 
 @contextlib.contextmanager
-def _create_standalone_pip() -> Generator[str, None, None]:
-    """Create a "standalone pip" zip file.
+def _create_runnable_pip() -> Generator[str, None, None]:
+    """Create a "pip runner" file.
 
-    The zip file's content is identical to the currently-running pip.
+    The runner file ensures that import for pip happens using the currently-running pip.
     It will be used to install requirements into the build environment.
     """
     source = pathlib.Path(pip_location).resolve().parent
 
-    # Return the current instance if `source` is not a directory. We can't build
-    # a zip from this, and it likely means the instance is already standalone.
+    # Return the current instance if `source` is not a directory. It likely
+    # means that this copy of pip is already standalone.
     if not source.is_dir():
         yield str(source)
         return
 
     with TempDirectory(kind="standalone-pip") as tmp_dir:
-        pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
-        kwargs = {}
-        if sys.version_info >= (3, 8):
-            kwargs["strict_timestamps"] = False
-        with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf:
-            for child in source.rglob("*"):
-                zf.write(child, child.relative_to(source.parent).as_posix())
-        yield os.path.join(pip_zip, "pip")
+        pip_runner = os.path.join(tmp_dir.path, "__pip-runner__.py")
+        with open(pip_runner, "w", encoding="utf8") as f:
+            f.write(PIP_RUNNER.format(source=os.fsdecode(source)))
+        yield pip_runner
 
 
 class BuildEnvironment:
@@ -206,7 +224,7 @@ class BuildEnvironment:
         if not requirements:
             return
         with contextlib.ExitStack() as ctx:
-            pip_runnable = ctx.enter_context(_create_standalone_pip())
+            pip_runnable = ctx.enter_context(_create_runnable_pip())
             self._install_requirements(
                 pip_runnable,
                 finder,
diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py
index 35ae1e68c..df5152a78 100644
--- a/src/pip/_internal/cli/cmdoptions.py
+++ b/src/pip/_internal/cli/cmdoptions.py
@@ -1013,6 +1013,7 @@ use_deprecated_feature: Callable[..., Option] = partial(
     default=[],
     choices=[
         "legacy-resolver",
+        "backtrack-on-build-failures",
         "html5lib",
     ],
     help=("Enable deprecated functionality, that will be removed in the future."),
diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py
index b289426cf..6db058869 100644
--- a/src/pip/_internal/cli/req_command.py
+++ b/src/pip/_internal/cli/req_command.py
@@ -34,6 +34,7 @@ from pip._internal.req.req_file import parse_requirements
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.resolution.base import BaseResolver
 from pip._internal.self_outdated_check import pip_self_version_check
+from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.temp_dir import (
     TempDirectory,
     TempDirectoryTypeRegistry,
@@ -269,6 +270,31 @@ class RequirementCommand(IndexGroupCommand):
 
         return "2020-resolver"
 
+    @staticmethod
+    def determine_build_failure_suppression(options: Values) -> bool:
+        """Determines whether build failures should be suppressed and backtracked on."""
+        if "backtrack-on-build-failures" not in options.deprecated_features_enabled:
+            return False
+
+        if "legacy-resolver" in options.deprecated_features_enabled:
+            raise CommandError("Cannot backtrack with legacy resolver.")
+
+        deprecated(
+            reason=(
+                "Backtracking on build failures can mask issues related to how "
+                "a package generates metadata or builds a wheel. This flag will "
+                "be removed in pip 22.2."
+            ),
+            gone_in=None,
+            replacement=(
+                "avoiding known-bad versions by explicitly telling pip to ignore them "
+                "(either directly as requirements, or via a constraints file)"
+            ),
+            feature_flag=None,
+            issue=10655,
+        )
+        return True
+
     @classmethod
     def make_requirement_preparer(
         cls,
@@ -345,6 +371,7 @@ class RequirementCommand(IndexGroupCommand):
             use_pep517=use_pep517,
             config_settings=getattr(options, "config_settings", None),
         )
+        suppress_build_failures = cls.determine_build_failure_suppression(options)
         resolver_variant = cls.determine_resolver_variant(options)
         # The long import name and duplicated invocation is needed to convince
         # Mypy into correctly typechecking. Otherwise it would complain the
@@ -364,6 +391,7 @@ class RequirementCommand(IndexGroupCommand):
                 force_reinstall=force_reinstall,
                 upgrade_strategy=upgrade_strategy,
                 py_version_info=py_version_info,
+                suppress_build_failures=suppress_build_failures,
             )
         import pip._internal.resolution.legacy.resolver
 
diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py
index a4c24b52a..4569033db 100644
--- a/src/pip/_internal/resolution/resolvelib/factory.py
+++ b/src/pip/_internal/resolution/resolvelib/factory.py
@@ -27,6 +27,7 @@ from pip._internal.cache import CacheEntry, WheelCache
 from pip._internal.exceptions import (
     DistributionNotFound,
     InstallationError,
+    InstallationSubprocessError,
     MetadataInconsistent,
     UnsupportedPythonVersion,
     UnsupportedWheel,
@@ -96,6 +97,7 @@ class Factory:
         force_reinstall: bool,
         ignore_installed: bool,
         ignore_requires_python: bool,
+        suppress_build_failures: bool,
         py_version_info: Optional[Tuple[int, ...]] = None,
     ) -> None:
         self._finder = finder
@@ -106,6 +108,7 @@ class Factory:
         self._use_user_site = use_user_site
         self._force_reinstall = force_reinstall
         self._ignore_requires_python = ignore_requires_python
+        self._suppress_build_failures = suppress_build_failures
 
         self._build_failures: Cache[InstallationError] = {}
         self._link_candidate_cache: Cache[LinkCandidate] = {}
@@ -198,6 +201,12 @@ class Factory:
                     )
                     self._build_failures[link] = e
                     return None
+                except InstallationSubprocessError as e:
+                    if not self._suppress_build_failures:
+                        raise
+                    logger.warning("Discarding %s due to build failure: %s", link, e)
+                    self._build_failures[link] = e
+                    return None
 
             base: BaseCandidate = self._editable_candidate_cache[link]
         else:
@@ -219,6 +228,12 @@ class Factory:
                     )
                     self._build_failures[link] = e
                     return None
+                except InstallationSubprocessError as e:
+                    if not self._suppress_build_failures:
+                        raise
+                    logger.warning("Discarding %s due to build failure: %s", link, e)
+                    self._build_failures[link] = e
+                    return None
             base = self._link_candidate_cache[link]
 
         if not extras:
diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py
index a605d6c25..32ef7899b 100644
--- a/src/pip/_internal/resolution/resolvelib/resolver.py
+++ b/src/pip/_internal/resolution/resolvelib/resolver.py
@@ -47,6 +47,7 @@ class Resolver(BaseResolver):
         ignore_requires_python: bool,
         force_reinstall: bool,
         upgrade_strategy: str,
+        suppress_build_failures: bool,
         py_version_info: Optional[Tuple[int, ...]] = None,
     ):
         super().__init__()
@@ -61,6 +62,7 @@ class Resolver(BaseResolver):
             force_reinstall=force_reinstall,
             ignore_installed=ignore_installed,
             ignore_requires_python=ignore_requires_python,
+            suppress_build_failures=suppress_build_failures,
             py_version_info=py_version_info,
         )
         self.ignore_dependencies = ignore_dependencies


Test patch:
diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py
index efcae2928..f3acdb9ca 100644
--- a/tests/functional/test_new_resolver.py
+++ b/tests/functional/test_new_resolver.py
@@ -2326,6 +2326,26 @@ def test_new_resolver_do_not_backtrack_on_build_failure(
     assert "egg_info" in result.stderr
 
 
+def test_new_resolver_flag_permits_backtracking_on_build_failure(
+    script: PipTestEnvironment,
+) -> None:
+    create_basic_sdist_for_package(script, "pkg1", "2.0", fails_egg_info=True)
+    create_basic_wheel_for_package(script, "pkg1", "1.0")
+
+    script.pip(
+        "install",
+        "--use-deprecated=backtrack-on-build-failures",
+        "--no-cache-dir",
+        "--no-index",
+        "--find-links",
+        script.scratch_path,
+        "pkg1",
+        allow_stderr_warning=True,
+    )
+
+    script.assert_installed(pkg1="1.0")
+
+
 def test_new_resolver_works_when_failing_package_builds_are_disallowed(
     script: PipTestEnvironment,
 ) -> None:
diff --git a/tests/unit/resolution_resolvelib/conftest.py b/tests/unit/resolution_resolvelib/conftest.py
index a8889293b..6cef3c17c 100644
--- a/tests/unit/resolution_resolvelib/conftest.py
+++ b/tests/unit/resolution_resolvelib/conftest.py
@@ -63,6 +63,7 @@ def factory(finder: PackageFinder, preparer: RequirementPreparer) -> Iterator[Fa
         force_reinstall=False,
         ignore_installed=False,
         ignore_requires_python=False,
+        suppress_build_failures=False,
         py_version_info=None,
     )
 
diff --git a/tests/unit/resolution_resolvelib/test_resolver.py b/tests/unit/resolution_resolvelib/test_resolver.py
index 87c2b5f35..db71f911a 100644
--- a/tests/unit/resolution_resolvelib/test_resolver.py
+++ b/tests/unit/resolution_resolvelib/test_resolver.py
@@ -29,6 +29,7 @@ def resolver(preparer: RequirementPreparer, finder: PackageFinder) -> Resolver:
         ignore_requires_python=False,
         force_reinstall=False,
         upgrade_strategy="to-satisfy-only",
+        suppress_build_failures=False,
     )
     return resolver