Repository: pex-tool/pex

Issue:
Venv creation fails when PEX_PATH includes another PEX with overlapping distributions
As seen in Pants CI here: https://github.com/pantsbuild/pants/runs/5366819312?check_suite_focus=true#step:14:244
```
Traceback (most recent call last):
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 496, in execute
    exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 401, in _wrap_coverage
    return runner(*args)
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 432, in _wrap_profiling
    return runner(*args)
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 552, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 688, in execute_entry
    return self.execute_pkg_resources(entry_point)
  File "/home/runner/.cache/pants/named_caches/pex_root/unzipped_pexes/f24bc43887fbe833662f695d52015ae6960a03e0/.bootstrap/pex/pex.py", line 720, in execute_pkg_resources
    return runner()
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/bin/pex.py", line 693, in main
    env=env,
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/bin/pex.py", line 733, in do_main
    seed_info = seed_cache(options, pex, verbose=options.seed == Seed.VERBOSE)
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/bin/pex.py", line 771, in seed_cache
    venv_pex = ensure_venv(pex)
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/pex_bootstrapper.py", line 469, in ensure_venv
    symlink=not pex_info.venv_site_packages_copies,
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/venv/pex.py", line 120, in populate_venv
    record_provenance(_populate_deps(venv, pex, venv_python, symlink))
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/venv/pex.py", line 116, in record_provenance
    for src, dst in src_to_dst:
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/venv/pex.py", line 216, in _populate_deps
    for src, dst in record.reinstall(venv, symlink=symlink, rel_extra_path=rel_extra_path):
  File "/home/runner/.cache/pants/named_caches/pex_root/installed_wheels/9c753f809ca916c7792319971e4b0a1289669f9f/pex-2.1.68-py2.py3-none-any.whl/pex/pep_376.py", line 430, in reinstall
    os.symlink(rel_src, dst)
FileExistsError: [Errno 17] File exists: '../../../../../../installed_wheels/344f2d140a5da7db8bc9a1815c4ef21a7acf78df/typing_extensions-4.0.1-py3-none-any.whl/typing_extensions-4.0.1.dist-info' -> '/home/runner/.cache/pants/named_caches/pex_root/venvs/803bfa7a4272661bf61d0f74b7f68fec85cce03e/869423fe37ea5ac6ac5d56346aa5c96907919ef8.d22ba668af3c4c6f847d6d591b493a71/lib/python3.7/site-packages/typing_extensions-4.0.1.dist-info'
```

The issue here is a PEX_PATH is in play (`pytest.pex:requirements.pex:local_dists.pex`) and more than one PEX provides typing_extensions-4.0.1. Since PEX.resolve does no de-dup:
https://github.com/pantsbuild/pex/blob/6cc144e2d45d1320fda0e1ff9961989dff20580f/pex/pex.py#L115-L119

And since #1630 removed the failure guard from symlink (but only symlink) creation:
https://github.com/pantsbuild/pex/blob/70a2caded62e12ff8ecd97b67d0ea2f89c16b60d/pex/pep_376.py#L423-L430
https://github.com/pantsbuild/pex/blob/70a2caded62e12ff8ecd97b67d0ea2f89c16b60d/pex/pep_376.py#L405-L421

... this long-standing lack of de-duping came to the fore.


Discussion:
Aha. This is pretty narrow. The issue only occurs for distributuons that ship a top-level _module_. Top-level packages work fine since they trigger namespace-package .pth treatment and each gets its own unique site-packages subdir. Easy to fix, hard to write a failing test!

Resolving patch:
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -663,6 +663,10 @@ def _activate(self):
 
         resolved = self.resolve()
         for dist in resolved:
+            # N.B.: Since there can be more than one PEXEnvironment on the PEX_PATH we take care to
+            # avoid re-installing duplicate distributions we have in common with them.
+            if dist.location in sys.path:
+                continue
             with TRACER.timed("Activating %s" % dist, V=2):
                 if self._pex_info.inherit_path == InheritPath.FALLBACK:
                     # Prepend location to sys.path.
diff --git a/pex/pep_376.py b/pex/pep_376.py
--- a/pex/pep_376.py
+++ b/pex/pep_376.py
@@ -342,6 +342,10 @@ def reinstall(
         N.B.: A record of reinstalled files is returned in the form of an iterator that must be
         consumed to drive the installation to completion.
 
+        If there is an error re-installing a file due to it already existing in the destination
+        venv, the error is suppressed, and it's expected that the caller detects this by comparing
+        the record of installed files against those installed previously.
+
         :return: An iterator over src -> dst pairs.
         """
 
@@ -427,4 +431,8 @@ def reinstall(
                 yield src, dst
             rel_src = os.path.relpath(src, site_packages_dir)
             safe_mkdir(site_packages_dir)
-            os.symlink(rel_src, dst)
+            try:
+                os.symlink(rel_src, dst)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise e
diff --git a/pex/pex.py b/pex/pex.py
--- a/pex/pex.py
+++ b/pex/pex.py
@@ -114,8 +114,15 @@ def _loaded_envs(self):
 
     def resolve(self):
         # type: () -> Iterator[Distribution]
+        """Resolves all distributions loadable from this PEX by the current interpreter."""
+        seen = set()
         for env in self._loaded_envs:
             for dist in env.resolve():
+                # N.B.: Since there can be more than one PEX env on the PEX_PATH we take care to
+                # de-dup distributions they have in common.
+                if dist in seen:
+                    continue
+                seen.add(dist)
                 yield dist
 
     def _activate(self):


Test patch:
diff --git a/tests/integration/venv/test_issue_1630.py b/tests/integration/venv_ITs/test_issue_1630.py
similarity index 100%
rename from tests/integration/venv/test_issue_1630.py
rename to tests/integration/venv_ITs/test_issue_1630.py
diff --git a/tests/integration/venv_ITs/test_issue_1637.py b/tests/integration/venv_ITs/test_issue_1637.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/venv_ITs/test_issue_1637.py
@@ -0,0 +1,177 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os
+import subprocess
+from textwrap import dedent
+
+import pytest
+from colors import yellow
+
+from pex.common import safe_open, touch
+from pex.testing import IS_PYPY3, make_env, run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any, Iterable, List, Text
+
+
+def create_pex(
+    tmpdir,  # type: Any
+    include_srcs=True,  # type: bool
+    include_deps=True,  # type: bool
+    extra_args=(),  # type: Iterable[str]
+):
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    args = ["--pex-root", pex_root, "--runtime-pex-root", pex_root]
+
+    if include_deps and include_srcs:
+        name = "app.pex"
+    elif include_deps:
+        name = "deps.pex"
+    elif include_srcs:
+        name = "srcs.pex"
+    else:
+        name = "pex.file"
+    pex_file = os.path.join(str(tmpdir), name)
+    args.extend(["-o", pex_file])
+
+    if include_srcs:
+        src = os.path.join(str(tmpdir), "src")
+        with safe_open(os.path.join(src, "app.py"), "w") as fp:
+            fp.write(
+                dedent(
+                    """\
+                    from __future__ import print_function
+
+                    import sys
+
+                    from colors import yellow
+
+                    print(yellow("*** Flashy UI ***"), file=sys.stderr)
+                    for entry in sys.path:
+                        if not entry.startswith({unzipped_pexes_dir!r}):
+                            print(entry)
+                    """.format(
+                        unzipped_pexes_dir=os.path.join(pex_root, "unzipped_pexes")
+                    )
+                )
+            )
+        args.extend(["-D", src, "-m", "app"])
+
+    if include_deps:
+        # N.B.: We use this particular version of ansicolors since it contains a top-level module:
+        # colors.py. Some of the --venv symlink mode tests below only trigger for the exact case
+        # of a distribution with no top level packages and just a top-level modules.
+        args.append("ansicolors==1.0.2")
+
+    run_pex_command(args=args + list(extra_args)).assert_success()
+    return pex_file
+
+
+def execute_app(
+    pex_file,  # type: str
+    **env  # type: str
+):
+    # type: (...) -> List[Text]
+
+    process = subprocess.Popen(
+        args=[pex_file], env=make_env(**env), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    stdout, stderr = process.communicate()
+    stripped_stderr = stderr.decode("utf-8").strip()
+    assert 0 == process.returncode, stripped_stderr
+    assert yellow("*** Flashy UI ***") in stripped_stderr
+    return stdout.decode("utf-8").splitlines()
+
+
+def test_pex_path_dedup(tmpdir):
+    # type: (Any) -> None
+
+    app = create_pex(tmpdir)
+    expected_sys_path = execute_app(app)
+
+    deps = create_pex(tmpdir, include_srcs=False)
+    assert expected_sys_path == execute_app(app, PEX_PATH=deps)
+
+    srcs = create_pex(tmpdir, include_deps=False)
+    assert expected_sys_path == execute_app(app, PEX_PATH=os.pathsep.join((deps, srcs)))
+
+
+@pytest.mark.parametrize(
+    ["execution_mode_args"],
+    [
+        pytest.param([], id="zipapp"),
+        pytest.param(["--venv", "--venv-site-packages-copies"], id="venv (site-packages copies)"),
+        pytest.param(
+            ["--venv", "--no-venv-site-packages-copies"],
+            id="venv (site-packages symlinks)",
+        ),
+    ],
+)
+def test_pex_path_collision_non_conflicting(
+    tmpdir,  # type: Any
+    execution_mode_args,  # type: List[str]
+):
+    # type: (...) -> None
+
+    app = create_pex(tmpdir, extra_args=execution_mode_args)
+    execute_app(app)
+
+    # Test a non-conflicting duplicate dep collision.
+    deps = create_pex(tmpdir, include_srcs=False)
+    execute_app(app, PEX_PATH=deps)
+
+    # Test a non-conflicting duplicate src collision.
+    srcs = create_pex(tmpdir, include_deps=False)
+    execute_app(app, PEX_PATH=os.pathsep.join((deps, srcs)))
+
+
+@pytest.mark.skipif(IS_PYPY3, reason="Warnings are not always flushed to stderr under PyPy3.")
+def test_pex_path_collision_conflicting(tmpdir):
+    # Test a conflicting duplicate dep top-level module collision.
+
+    conflicting_colors = os.path.join(str(tmpdir), "conflicting_colors")
+    touch(os.path.join(conflicting_colors, "colors.py"))
+    with safe_open(os.path.join(conflicting_colors, "setup.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                from setuptools import setup
+
+                setup(
+                    name="conflicting-colors",
+                    version="0.0.1",
+                    py_modules=["colors"]
+                )
+                """
+            )
+        )
+    alternate_colors = create_pex(
+        tmpdir, include_srcs=False, include_deps=False, extra_args=[conflicting_colors]
+    )
+    env = make_env(PEX_PATH=alternate_colors)
+
+    # A zipapp with a collision should always work, 1st on PEX_PATH wins.
+    app = create_pex(tmpdir)
+    execute_app(app, **env)
+
+    def assert_venv_collision(*extra_args):
+        # type: (*str) -> None
+
+        app = create_pex(tmpdir, extra_args=["--venv"] + list(extra_args))
+        process = subprocess.Popen(
+            args=[app], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        _, stderr = process.communicate()
+        decoded_stderr = stderr.decode("utf-8")
+
+        # The --venv mode should warn about collisions but succeed.
+        assert 0 == process.returncode, decoded_stderr
+        assert "PEXWarning: Encountered collision building venv at " in decoded_stderr
+        assert "site-packages/colors.py was provided by:" in decoded_stderr
+        assert "sha1:17772af8295ffb7f4d6c3353665b5c542be332a2 -> " in decoded_stderr
+        assert "sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709 -> " in decoded_stderr
+
+    assert_venv_collision("--venv-site-packages-copies")
+    assert_venv_collision("--no-venv-site-packages-copies")
diff --git a/tests/integration/venv/test_virtualenv.py b/tests/integration/venv_ITs/test_virtualenv.py
similarity index 100%
rename from tests/integration/venv/test_virtualenv.py
rename to tests/integration/venv_ITs/test_virtualenv.py