Repository: pex-tool/pex

Issue:
Sporadic Pip HTTP errors in CI
This has actually been happening a while (~1 year): https://github.com/pantsbuild/pex/actions/runs/5432519810/jobs/9879610414#step:8:1734

It looks like so:
```
E       WARNING: Skipping page https://pypi.org/simple/psutil/ because the GET request got Content-Type: Unknown. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/html
E       ERROR: Could not find a version that satisfies the requirement psutil==5.9.1 (from versions: none)
E       ERROR: No matching distribution found for psutil==5.9.1
```

In particular this very much looks like a PyPI issue on the surface due to the claim of `Content-Type: Unknown` by Pip:
```
WARNING: Skipping page https://pypi.org/simple/psutil/ because the GET request got Content-Type: Unknown. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/html
```

I happened upon this though just now: https://github.com/pypa/pip/issues/11340#issuecomment-1272381581 and that certainly would explain this issue since CI tests many versions of Pip spanning 20.3.4 <-> 23.1.2 currently but Pex uses a single directory for the Pip cache here:
https://github.com/pantsbuild/pex/blob/c2fa8130d3dfb38f19f550a5bb0a3c69b7459f0d/pex/pip/tool.py#L304-L305



Discussion:

Resolving patch:
diff --git a/pex/pip/installation.py b/pex/pip/installation.py
--- a/pex/pip/installation.py
+++ b/pex/pip/installation.py
@@ -11,7 +11,7 @@
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
 from pex.pex import PEX
-from pex.pex_bootstrapper import VenvPex, ensure_venv
+from pex.pex_bootstrapper import ensure_venv
 from pex.pip.tool import Pip
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve.resolvers import Resolver
@@ -29,13 +29,14 @@
     from pex.third_party import attr
 
 
-def _pip_venv(
+def _pip_installation(
     version,  # type: PipVersionValue
     iter_distribution_locations,  # type: Callable[[], Iterator[str]]
     interpreter=None,  # type: Optional[PythonInterpreter]
 ):
-    # type: (...) -> VenvPex
-    path = os.path.join(ENV.PEX_ROOT, "pip-{version}.pex".format(version=version))
+    # type: (...) -> Pip
+    pip_root = os.path.join(ENV.PEX_ROOT, "pip", str(version))
+    path = os.path.join(pip_root, "pip.pex")
     pip_interpreter = interpreter or PythonInterpreter.get()
     pip_pex_path = os.path.join(path, isolated().pex_hash)
     with atomic_directory(pip_pex_path) as chroot:
@@ -66,13 +67,15 @@ def _pip_venv(
                 fp.close()
                 isolated_pip_builder.set_executable(fp.name, "__pex_patched_pip__.py")
             isolated_pip_builder.freeze()
-    return ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
+    pip_cache = os.path.join(pip_root, "pip_cache")
+    pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
+    return Pip(pip_pex=pip_pex, pip_cache=pip_cache)
 
 
 def _vendored_installation(interpreter=None):
-    # type: (Optional[PythonInterpreter]) -> VenvPex
+    # type: (Optional[PythonInterpreter]) -> Pip
 
-    return _pip_venv(
+    return _pip_installation(
         version=PipVersion.VENDORED,
         iter_distribution_locations=lambda: third_party.expose(("pip", "setuptools", "wheel")),
         interpreter=interpreter,
@@ -84,7 +87,7 @@ def _resolved_installation(
     resolver,  # type: Resolver
     interpreter=None,  # type: Optional[PythonInterpreter]
 ):
-    # type: (...) -> VenvPex
+    # type: (...) -> Pip
     if version is PipVersion.VENDORED:
         return _vendored_installation(interpreter=interpreter)
 
@@ -96,7 +99,7 @@ def resolve_distribution_locations():
         ).installed_distributions:
             yield installed_distribution.distribution.location
 
-    return _pip_venv(
+    return _pip_installation(
         version=version,
         iter_distribution_locations=resolve_distribution_locations,
         interpreter=interpreter,
@@ -204,7 +207,7 @@ def get_pip(
     if pip is None:
         installation.check_python_applies()
         if version is PipVersion.VENDORED:
-            pip = Pip(pip_pex=_vendored_installation(interpreter=interpreter))
+            pip = _vendored_installation(interpreter=interpreter)
         else:
             if resolver is None:
                 raise ValueError(
@@ -212,12 +215,8 @@ def get_pip(
                         requirement=version.requirement
                     )
                 )
-            pip = Pip(
-                pip_pex=_resolved_installation(
-                    version=version,
-                    resolver=resolver,
-                    interpreter=interpreter,
-                )
+            pip = _resolved_installation(
+                version=version, resolver=resolver, interpreter=interpreter
             )
         _PIP[installation] = pip
     return pip
diff --git a/pex/pip/tool.py b/pex/pip/tool.py
--- a/pex/pip/tool.py
+++ b/pex/pip/tool.py
@@ -224,6 +224,7 @@ class Pip(object):
     _PATCHES_PACKAGE_NAME = "_pex_pip_patches"
 
     _pip_pex = attr.ib()  # type: VenvPex
+    _pip_cache = attr.ib()  # type: str
 
     @staticmethod
     def _calculate_resolver_version(package_index_configuration=None):
@@ -301,8 +302,7 @@ def _spawn_pip_isolated(
         else:
             pip_args.append("-q")
 
-        pip_cache = os.path.join(ENV.PEX_ROOT, "pip_cache")
-        pip_args.extend(["--cache-dir", pip_cache])
+        pip_args.extend(["--cache-dir", self._pip_cache])
 
         command = pip_args + list(args)
 
@@ -321,7 +321,7 @@ def _spawn_pip_isolated(
         # since Pip relies upon `shutil.move` which is only atomic when `os.rename` can be used.
         # See https://github.com/pantsbuild/pex/issues/1776 for an example of the issues non-atomic
         # moves lead to in the `pip wheel` case.
-        pip_tmpdir = os.path.join(pip_cache, ".tmp")
+        pip_tmpdir = os.path.join(self._pip_cache, ".tmp")
         safe_mkdir(pip_tmpdir)
         extra_env.update(TMPDIR=pip_tmpdir)
 
diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py
--- a/pex/venv/virtualenv.py
+++ b/pex/venv/virtualenv.py
@@ -349,8 +349,7 @@ def sys_path(self):
 
     def iter_distributions(self):
         # type: () -> Iterator[Distribution]
-        """"""
-        for dist in find_distributions(search_path=self.sys_path):
+        for dist in find_distributions(search_path=self._interpreter.site_packages):
             yield dist
 
     def _rewrite_base_scripts(self, real_venv_dir):


Test patch:
diff --git a/tests/integration/cli/commands/test_issue_1667.py b/tests/integration/cli/commands/test_issue_1667.py
--- a/tests/integration/cli/commands/test_issue_1667.py
+++ b/tests/integration/cli/commands/test_issue_1667.py
@@ -77,14 +77,19 @@ def test_interpreter_constraints_range_coverage(
 
     def assert_pex_works(python):
         # type: (str) -> None
-        assert (
-            subprocess.check_output(
-                args=[python, ipython_pex, "-c", "import IPython; print(IPython.__file__)"]
-            )
-            .decode("utf-8")
-            .strip()
-            .startswith(pex_root)
+        comm_channel = os.path.join(str(tmpdir), "comm")
+        subprocess.check_call(
+            args=[
+                python,
+                ipython_pex,
+                "-c",
+                "import IPython; print(IPython.__file__, file=open({comm!r}, 'w'))".format(
+                    comm=comm_channel
+                ),
+            ]
         )
+        with open(comm_channel) as fp:
+            assert fp.read().startswith(pex_root)
 
     assert_pex_works(py38.binary)
     if (3, 7) <= sys.version_info[:2] < (3, 11):
diff --git a/tests/integration/test_issue_1726.py b/tests/integration/test_issue_1726.py
--- a/tests/integration/test_issue_1726.py
+++ b/tests/integration/test_issue_1726.py
@@ -46,6 +46,13 @@ def test_check_install_issue_1726(
             )
         )
 
+    # Via: jaraco-collections==3.5.1 -> jaraco-text -> inflect -> pydantic>=1.9.1
+    # Pydantic 2.0 can get pulled in which defeats the Pip legacy resolver and leads to a resolve
+    # conflict for PyPy; so we bound pydantic low.
+    constraints = os.path.join(str(tmpdir), "constraints.txt")
+    with open(constraints, "w") as fp:
+        fp.write("pydantic<2")
+
     pex_root = os.path.join(str(tmpdir), "pex_root")
     pex_args = [
         "--pex-root",
@@ -53,6 +60,8 @@ def test_check_install_issue_1726(
         "--runtime-pex-root",
         pex_root,
         src,
+        "--constraints",
+        constraints,
         "--",
         "-c",
         "from jaraco import collections; print(collections.__file__)",