Repository: pex-tool/pex

Issue:
Some python tools and libraries don't work with symlinked venv site-packages contents.
Notably, pylint, which is used by Pants in a `--venv` PEX: https://github.com/PyCQA/pylint/issues/1470

Pex should gain a knob to work around this as needed.


Discussion:

Resolving patch:
diff --git a/pex/bin/pex.py b/pex/bin/pex.py
--- a/pex/bin/pex.py
+++ b/pex/bin/pex.py
@@ -184,7 +184,21 @@ def configure_clp_pex_options(parser):
         action=HandleBoolAction,
         help=(
             "If --venv is specified, create the venv using copies of base interpreter files "
-            "instead of symlinks."
+            "instead of symlinks. This allows --venv mode PEXes to work across interpreter "
+            "upgrades without being forced to remove the PEX_ROOT to allow the venv to re-build "
+            "using the upgraded interpreter."
+        ),
+    )
+    group.add_argument(
+        "--venv-site-packages-copies",
+        "--no-venv-site-packages-copies",
+        dest="venv_site_packages_copies",
+        default=False,
+        action=HandleBoolAction,
+        help=(
+            "If --venv is specified, populate the venv site packages using hard links or copies of "
+            "resolved PEX dependencies instead of symlinks. This can be used to work around "
+            "problems with tools or libraries that are confused by symlinked source files."
         ),
     )
 
@@ -529,6 +543,7 @@ def build_pex(
     pex_info.venv = bool(options.venv)
     pex_info.venv_bin_path = options.venv or BinPath.FALSE
     pex_info.venv_copies = options.venv_copies
+    pex_info.venv_site_packages_copies = options.venv_site_packages_copies
     pex_info.includes_tools = options.include_tools or options.venv
     pex_info.pex_path = options.pex_path
     pex_info.ignore_errors = options.ignore_errors
diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py
--- a/pex/pex_bootstrapper.py
+++ b/pex/pex_bootstrapper.py
@@ -473,7 +473,7 @@ def ensure_venv(
                             short_venv_dir, "venv", "bin", os.path.basename(pex.interpreter.binary)
                         ),
                         collisions_ok=collisions_ok,
-                        symlink=True,
+                        symlink=not pex_info.venv_site_packages_copies,
                     )
 
                     # There are popular Linux distributions with shebang length limits
diff --git a/pex/pex_info.py b/pex/pex_info.py
--- a/pex/pex_info.py
+++ b/pex/pex_info.py
@@ -200,6 +200,16 @@ def venv_copies(self, value):
         # type: (bool) -> None
         self._pex_info["venv_copies"] = value
 
+    @property
+    def venv_site_packages_copies(self):
+        # type: () -> bool
+        return self._pex_info.get("venv_site_packages_copies", False)
+
+    @venv_site_packages_copies.setter
+    def venv_site_packages_copies(self, value):
+        # type: (bool) -> None
+        self._pex_info["venv_site_packages_copies"] = value
+
     def venv_dir(
         self,
         pex_file,  # type: str


Test patch:
diff --git a/tests/integration/test_pex_bootstrapper.py b/tests/integration/test_pex_bootstrapper.py
--- a/tests/integration/test_pex_bootstrapper.py
+++ b/tests/integration/test_pex_bootstrapper.py
@@ -18,7 +18,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Callable
 
 
 def test_ensure_venv_short_link(
@@ -201,3 +201,40 @@ def test_ensure_venv_namespace_packages(tmpdir):
     } == {
         os.path.basename(d) for d in package_file_installed_wheel_dirs
     }, "Expected 5 unique contributing wheels."
+
+
+def test_ensure_venv_site_packages_copies(
+    pex_bdist,  # type: str
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    pex_file = os.path.join(str(tmpdir), "pex")
+
+    def assert_venv_site_packages_copies(copies):
+        # type: (bool) -> None
+        run_pex_command(
+            args=[
+                pex_bdist,
+                "-o",
+                pex_file,
+                "--pex-root",
+                pex_root,
+                "--runtime-pex-root",
+                pex_root,
+                "--venv",
+                "--venv-site-packages-copies" if copies else "--no-venv-site-packages-copies",
+                "--seed",
+            ]
+        ).assert_success()
+
+        venv_dir = PexInfo.from_pex(pex_file).venv_dir(pex_file)
+        assert venv_dir is not None
+        venv = Virtualenv(venv_dir=venv_dir)
+        pex_package = os.path.join(venv.site_packages_dir, "pex")
+        assert os.path.isdir(pex_package)
+        assert copies != os.path.islink(pex_package)
+
+    assert_venv_site_packages_copies(copies=True)
+    assert_venv_site_packages_copies(copies=False)