Repository: pypa/build
Pull request #140 (2020-10-14 08:57:57)

Borrow pip from host python if available
Instead of provisioning via ensurepip. This speeds up isolated build environment creation.

While implementing this I had to also address https://github.com/pypa/build/issues/120 to make the CI pass. They're separate commits though. 

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>


Patch:
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..de4edc8
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,6 @@
+.tox
+.*_cache
+*.egg-info
+Dockerfile
+build
+dist
diff --git a/.gitignore b/.gitignore
index 76597bc..e534689 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,4 @@ coverage.xml
 tests/integration/
 .integration-sources
 .tox
+Dockerfile
diff --git a/setup.cfg b/setup.cfg
index 7aa7e35..952a42f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -30,6 +30,7 @@ install_requires =
     toml
     importlib-metadata;python_version < "3.8"
     typing;python_version<"3"
+    virtualenv>=20.0.35;python_version < "3"
 python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
 package_dir =
     =src
diff --git a/src/build/__main__.py b/src/build/__main__.py
index cfa2c3f..9fe8d19 100644
--- a/src/build/__main__.py
+++ b/src/build/__main__.py
@@ -8,8 +8,8 @@ import warnings
 
 from typing import List, Optional, TextIO, Type
 
-from . import BuildBackendException, BuildException, ConfigSettings, ProjectBuilder
-from .env import IsolatedEnvironment
+from build import BuildBackendException, BuildException, ConfigSettings, ProjectBuilder
+from build.env import IsolatedEnvironment
 
 
 __all__ = ['build', 'main', 'main_parser']
diff --git a/src/build/env.py b/src/build/env.py
index 65d32f4..760957e 100644
--- a/src/build/env.py
+++ b/src/build/env.py
@@ -6,11 +6,17 @@ import sys
 import sysconfig
 import tempfile
 import types
+import typing
 from typing import Dict, Iterable, List, Optional, Sequence, Type
 
-if sys.version_info[0] == 2:  # pragma: no cover
+if sys.version_info[0] == 2:
     FileExistsError = OSError
 
+try:
+    import pip
+except ImportError:  # pragma: no cover
+    pip = None  # pragma: no cover
+
 _HAS_SYMLINK = None  # type: Optional[bool]
 
 
@@ -57,6 +63,7 @@ class IsolatedEnvironment(object):
         self._remove_paths = []  # type: List[str]
         self._executable = _executable
         self._old_executable = None  # type: Optional[str]
+        self._pip_executable = None  # type: Optional[str]
 
         # normalize paths so that we can compare them -- required on case insensitive systems
         for path in remove_paths:
@@ -79,16 +86,10 @@ class IsolatedEnvironment(object):
         Creates an isolated environment for the current interpreter
         """
         remove_paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
-
-        for path in cls._MANIPULATE_PATHS:
-            our_path = sysconfig.get_path(path)
-            if our_path:
-                remove_paths.append(our_path)
-
-            for scheme in sysconfig.get_scheme_names():
-                our_path = sysconfig.get_path(path, scheme)
-                if our_path:
-                    remove_paths.append(our_path)
+        # also remove the pth file adding this project (when installed via --develop install)
+        for path in sys.path:
+            if os.path.exists(os.path.join(path, 'build.egg-info')):
+                remove_paths.append(path)  # pragma: no cover
 
         return cls(remove_paths)
 
@@ -124,69 +125,10 @@ class IsolatedEnvironment(object):
         """
         return sysconfig.get_path(path, vars=self._env_vars)
 
-    def _symlink_relative(self, path):  # type: (Optional[str]) -> None
-        """
-        Symlinks a path into our environment
-
-        The original prefix will be removed and replaced with our environmenmt's
-
-        If the path is not valid, nothing will happen
-        """
-        if not path:  # pragma: no cover
-            return
-
-        prefix = sysconfig.get_config_var('prefix')
-        if prefix and path and path.startswith(prefix):
-            new_path = os.path.join(self.path, path[len(prefix + os.pathsep) :])
-            if not os.path.exists(new_path):
-                try:
-                    os.makedirs(os.path.dirname(new_path))
-                except FileExistsError:
-                    pass
-                if fs_supports_symlink():
-                    os.symlink(path, new_path)
-                else:
-                    import shutil
-
-                    shutil.copytree(path, new_path)
-
-    def _create_env_pythonhome(self):  # type: () -> None
-        sys_path = sys.path[1:]
-
-        for path in self._MANIPULATE_PATHS:
-            env_path = self._get_env_path(path)
-            if env_path:
-                sys_path.append(os.path.normcase(env_path))
-
-        for path in self._remove_paths:
-            if path in sys_path:
-                sys_path.remove(path)
-
-        env_scripts = self._get_env_path('scripts')
-        if env_scripts is None:  # pragma: no cover
-            raise RuntimeError('Missing scripts directory in sysconfig')
-
-        exe_path = [env_scripts]
-        if 'PATH' in os.environ:
-            exe_path.append(os.environ['PATH'])
-
-        self._replace_env('PATH', os.pathsep.join(exe_path))
-        self._replace_env('PYTHONPATH', os.pathsep.join(sys_path))
-
-        # Point the Python interpreter to our environment
-        self._replace_env('PYTHONHOME', self.path)
-
-        self._symlink_relative(sysconfig.get_path('include'))
-        self._symlink_relative(sysconfig.get_path('platinclude'))
-        self._symlink_relative(sysconfig.get_config_var('LIBPL'))
-
     def _create_env_venv(self):  # type: () -> None
-        if sys.version_info[0] == 2:
-            raise RuntimeError('venv not available on Python 2')
-
         import venv
 
-        venv.EnvBuilder(with_pip=True).create(self.path)
+        venv.EnvBuilder(with_pip=False).create(self.path)
 
         env_scripts = self._get_env_path('scripts')
         if not env_scripts:
@@ -199,6 +141,42 @@ class IsolatedEnvironment(object):
         if not os.path.exists(self._executable):
             raise RuntimeError('Virtual environment creation failed, executable {} missing'.format(self._executable))
 
+        # Scenario 1: pip is available (either installed or via pth file) within the python executable alongside
+        # this projects environment: in this case we should be able to import it
+        if pip is not None:
+            self._pip_executable = sys.executable
+            return
+        # Scenario 2: this project is installed into a virtual environment that has no pip, but the matching system has
+        # Scenario 3: there's a pip executable on PATH
+        # Scenario 4: no pip can be found, we might be able to provision one into the isolated build env via ensurepip
+        cmd = [self.executable, '-Im', 'ensurepip', '--upgrade', '--default-pip']
+        try:
+            subprocess.check_call(cmd, cwd=self.path)
+        except subprocess.CalledProcessError:  # pragma: no cover
+            pass  # pragma: no cover
+        # avoid the setuptools from ensurepip to break the isolation
+        subprocess.check_call([self.executable, '-Im', 'pip', 'uninstall', 'setuptools', '-y'])
+        self._pip_executable = self.executable
+
+    def _create_env_virtualenv(self):  # type: () -> None
+        from virtualenv import cli_run
+
+        cmd = [str(self.path), '--no-setuptools', '--no-wheel', '--activators', '']
+        if pip is not None:
+            cmd.append('--no-pip')
+        result = cli_run(cmd, setup_logging=False)
+        self._executable = str(result.creator.exe)
+        self._pip_executable = self._executable
+        if pip is not None:
+            self._pip_executable = sys.executable
+
+    def _create_isolated_env(self):  # type: () -> None
+        if sys.version_info[0] == 2:
+            self._create_env_virtualenv()
+
+        else:
+            self._create_env_venv()
+
     def __enter__(self):  # type: () -> IsolatedEnvironment
         """
         Set up the environment
@@ -211,10 +189,7 @@ class IsolatedEnvironment(object):
             'platbase': self.path,
         }
 
-        if sys.version_info[0] == 2:
-            self._create_env_pythonhome()
-        else:
-            self._create_env_venv()
+        self._create_isolated_env()
 
         self._pop_env('PIP_REQUIRE_VIRTUALENV')
         # address https://github.com/pypa/pep517/pull/93
@@ -245,19 +220,20 @@ class IsolatedEnvironment(object):
         if not requirements:
             return
 
-        if sys.version_info[0] == 2:
-            subprocess.check_call([self.executable, '-m', 'ensurepip'], cwd=self.path)
-
         with tempfile.NamedTemporaryFile('w+', prefix='build-reqs-', suffix='.txt', delete=False) as req_file:
             req_file.write(os.linesep.join(requirements))
             req_file.close()
             cmd = [
-                self.executable,
-                '-m',
+                typing.cast(str, self._pip_executable),
+                # on python2 if isolation is achieved via environment variables, we need to ignore those while calling
+                # host python (otherwise pip would not be available within it)
+                '-{}m'.format('E' if self._pip_executable == self._executable and sys.version_info[0] == 2 else ''),
                 'pip',
                 'install',
                 '--prefix',
                 self.path,
+                '--ignore-installed',
+                '--no-warn-script-location',
                 '-r',
                 os.path.abspath(req_file.name),
             ]
diff --git a/tox.ini b/tox.ini
index 5b72c46..e32bb0d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,9 +1,9 @@
 [tox]
 envlist =
     fix
-    {py39, py38, py37, py36, py35, py27, pypy3, pypy2}{, -path, -sdist, -wheel}
     type
     docs
+    {py39, py38, py37, py36, py35, py27, pypy3, pypy2}{, -path, -sdist, -wheel}
 isolated_build = true
 skip_missing_interpreters = true
 minversion = 3.14
@@ -16,16 +16,17 @@ description =
     path: via PYTHONPATH
     sdist: via source distribution
     wheel: via wheel
+passenv =
+    LC_ALL
+    PIP_*
+    PYTEST_*
 setenv =
-    COVERAGE_FILE = {envtmpdir}/.coverage
-    {py27, pypy2}: PYTHONWARNINGS = ignore:DEPRECATION::pip._internal.cli.base_command
+    COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
     path: TEST_MODE = path
     sdist: TEST_MODE = sdist
     wheel: TEST_MODE = wheel
-passenv =
-    PYTEST_*
-    PIP_*
-skip_install = path,sdist,wheel: false
+    {py27,pypy2}: PYTHONWARNINGS = ignore:DEPRECATION::pip._internal.cli.base_command
+skip_install = false
 deps =
     path,sdist,wheel: -rrequirements-dev.txt
 extras =
@@ -65,3 +66,34 @@ deps =
 commands =
     sphinx-build -n -W docs/source {envtmpdir}
     python -c 'print("Documentation available under file://{envtmpdir}/index.html")'
+
+[testenv:dev]
+description = generate a DEV environment
+usedevelop = true
+deps =
+    virtualenv>=20.0.34
+extras =
+    doc
+    test
+commands =
+    python -m pip list --format=columns
+    python -c 'import sys; print(sys.executable)'
+
+[testenv:coverage]
+description = combine coverage from test environments
+passenv =
+    DIFF_AGAINST
+setenv =
+    COVERAGE_FILE = {toxworkdir}/.coverage
+skip_install = true
+deps =
+    coverage>=5.1
+    diff_cover>=3
+parallel_show_output = true
+commands =
+    coverage combine
+    coverage report --skip-covered --show-missing -i
+    coverage xml -o {toxworkdir}/coverage.xml -i
+    coverage html -d {toxworkdir}/htmlcov -i
+    python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml
+depends = {py39, py38, py37, py36, py35, py27, pypy3, pypy2}{, -path, -sdist, -wheel}


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 7237655..769d4ca 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -35,6 +35,8 @@ _setup()
 
 
 def pytest_addoption(parser):
+    os.environ['PYTHONWARNINGS'] = 'ignore:DEPRECATION::pip._internal.cli.base_command'  # for when not run within tox
+    os.environ['PIP_DISABLE_PIP_VERSION_CHECK'] = '1'  # do not pollute stderr with upgrade advisory
     parser.addoption('--run-integration', action='store_true', help='run the integration tests')
     parser.addoption('--only-integration', action='store_true', help='only run the integration tests')
 
diff --git a/tests/test_env.py b/tests/test_env.py
index 7e5d08a..969eff4 100644
--- a/tests/test_env.py
+++ b/tests/test_env.py
@@ -1,65 +1,16 @@
 # SPDX-License-Identifier: MIT
-
+import json
 import os
 import os.path
 import platform
 import subprocess
 import sys
-import sysconfig
 
 import pytest
 
 import build.env
 
 
-@pytest.mark.skipif(sys.version_info[0] != 2, reason='Custom isolated environment only available on Python 2')
-def test_isolated_environment_setup():
-    old_path = os.environ['PATH']
-    with build.env.IsolatedEnvironment.for_current() as env:
-        if os.name != 'nt':
-            assert os.environ['PATH'] == os.pathsep.join([os.path.join(env.path, 'bin'), old_path])
-        assert os.environ['PYTHONHOME'] == env.path
-
-        python_path = map(os.path.normpath, os.environ['PYTHONPATH'].split(os.pathsep))
-        for path in ('purelib', 'platlib'):
-            sys_vars = {
-                'base': env.path,
-                'platbase': env.path,
-            }
-            assert os.path.normcase(sysconfig.get_path(path)) not in python_path
-            assert os.path.normcase(sysconfig.get_path(path, vars=sys_vars)) in python_path
-
-        copy_path = [
-            sysconfig.get_path('include'),
-            sysconfig.get_path('platinclude'),
-        ]
-        libpl = sysconfig.get_config_var('LIBPL')
-        if libpl is None:
-            """
-            if os.name != 'nt':
-                if sys.version_info[0] == 2:
-                    assert sys.subversion[0] == 'PyPy'  # not available in Windows CPython 3
-                else:
-                    assert sys.implementation.name == 'pypy'  # Python 3 only
-            """
-        else:
-            copy_path.append(libpl)
-
-        prefix = sysconfig.get_config_var('prefix')
-        assert prefix is not None
-
-        for path in copy_path:
-            assert path is not None
-            if path.startswith(prefix):
-                relative_path = path[len(prefix + os.pathsep) :]
-                path = os.path.join(env.path, relative_path)
-            assert os.path.exists(path)
-
-
-@pytest.mark.skipif(
-    platform.python_implementation() == 'PyPy' and sys.version_info[0] == 2,
-    reason='PyPy 2 does not honor PYTHONHOME - switch to virtualenv',
-)
 @pytest.mark.isolated
 def test_isolation():
     subprocess.check_call([sys.executable, '-c', 'import build.env'])
@@ -88,8 +39,18 @@ def test_isolated_environment_install(mocker):
         env.install(['some', 'requirements'])
         if sys.version_info[:2] != (3, 5):
             subprocess.check_call.assert_called()
-        args = subprocess.check_call.call_args[0][0]
-        assert args[:7] == [env.executable, '-m', 'pip', 'install', '--prefix', env.path, '-r']
+        args = subprocess.check_call.call_args[0][0][:-1]
+        assert args == [
+            env._pip_executable,
+            '-{}m'.format('E' if env._pip_executable == env._executable and sys.version_info[0] == 2 else ''),
+            'pip',
+            'install',
+            '--prefix',
+            env.path,
+            '--ignore-installed',
+            '--no-warn-script-location',
+            '-r',
+        ]
 
 
 def test_uninitialised_isolated_environment():
@@ -97,3 +58,31 @@ def test_uninitialised_isolated_environment():
 
     with pytest.raises(RuntimeError):
         env.path
+
+
+@pytest.mark.isolated
+def test_create_isolated_build_host_with_no_pip(tmp_path, capfd, mocker):
+    mocker.patch.object(build.env, 'pip', None)
+    expected = {'pip', 'greenlet', 'readline', 'cffi'} if platform.python_implementation() == 'PyPy' else {'pip'}
+
+    with build.env.IsolatedEnvironment.for_current() as isolated_env:
+        cmd = [isolated_env.executable, '-m', 'pip', 'list', '--format', 'json']
+        packages = {p['name'] for p in json.loads(subprocess.check_output(cmd, universal_newlines=True))}
+        assert packages == expected
+    assert isolated_env._pip_executable == isolated_env.executable
+    out, err = capfd.readouterr()
+    if sys.version_info[0] == 3:
+        assert out  # ensurepip prints onto the stdout
+    else:
+        assert not out
+    assert not err
+
+
+@pytest.mark.isolated
+def test_create_isolated_build_has_with_pip(tmp_path, capfd, mocker):
+    with build.env.IsolatedEnvironment.for_current() as isolated_env:
+        pass
+    assert isolated_env._pip_executable == sys.executable
+    out, err = capfd.readouterr()
+    assert not out
+    assert not err