Repository: pex-tool/pex

Issue:
The Pex CLI does not work when run from a read-only venv.
Works fine in a typical venv:
```
$ python3.10 -mvenv pex.venv
$ pex.venv/bin/pip -q install -U pip
$ pex.venv/bin/pip -q install -U pex
$ rm -rf ~/.pex
$ pex.venv/bin/pex3 lock create cowsay --indent 2 -o lock.json
running egg_info
writing cowsay.egg-info/PKG-INFO
writing dependency_links to cowsay.egg-info/dependency_links.txt
writing entry points to cowsay.egg-info/entry_points.txt
writing top-level names to cowsay.egg-info/top_level.txt
reading manifest file 'cowsay.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'cowsay.egg-info/SOURCES.txt'
running dist_info
creating /tmp/tmplrzt483o/build/cowsay.egg-info
writing /tmp/tmplrzt483o/build/cowsay.egg-info/PKG-INFO
writing dependency_links to /tmp/tmplrzt483o/build/cowsay.egg-info/dependency_links.txt
writing entry points to /tmp/tmplrzt483o/build/cowsay.egg-info/entry_points.txt
writing top-level names to /tmp/tmplrzt483o/build/cowsay.egg-info/top_level.txt
writing manifest file '/tmp/tmplrzt483o/build/cowsay.egg-info/SOURCES.txt'
reading manifest file '/tmp/tmplrzt483o/build/cowsay.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file '/tmp/tmplrzt483o/build/cowsay.egg-info/SOURCES.txt'
creating '/tmp/tmplrzt483o/build/cowsay.dist-info'
adding license file "LICENSE.txt" (matched pattern "LICEN[CS]E*")
$ pex.venv/bin/pex --lock lock.json cowsay -c cowsay -- 'Moo!'
  ____
| Moo! |
  ====
    \
     \
       ^__^
       (oo)\_______
       (__)\       )\/\
           ||----w |
           ||     ||
```

But in a read-only venv Pex commands fail like so:
```
$ rm -rf ~/.pex
$ chmod -R -w pex.venv/lib/python3.10/site-packages
$ pex.venv/bin/pex3 lock create cowsay --indent 2 -o lock.json
Failed to spawn a job for /home/jsirois/support/pex/phaer/pex.venv/bin/python3.10: [Errno 13] Permission denied: '/home/jsirois/.pex/pip/20.3.4-patched/pip.pex/d391aba428cc037d796ca2e7c83909a1db40f322.lck.work/.bootstrap/pex/__init__.py'
```

The issue here is that Pex trips over itself when constructing the `.bootstrap` by attempting to copy `pex/__init__.py` and other root `pex.*` package `__init__.py` files multiple times. The 1st copy works, but since the source file is read only, the destination file is as well, and subsequent copies (over-writes), fail.

This issue was brought to light by https://github.com/NixOS/nixpkgs/issues/246448 where read-only venvs - which are not typically a thing, are used by Nix.


Discussion:

Resolving patch:
diff --git a/pex/pex_builder.py b/pex/pex_builder.py
--- a/pex/pex_builder.py
+++ b/pex/pex_builder.py
@@ -562,25 +562,25 @@ def _copy_or_link(self, src, dst, label=None):
     def _prepare_bootstrap(self):
         from . import vendor
 
-        vendor.vendor_runtime(
+        # NB: We use pip here in the builder, but that's only at build time, and
+        # although we don't use pyparsing directly, packaging.markers, which we
+        # do use at runtime, does.
+        root_module_names = ["attr", "packaging", "pkg_resources", "pyparsing"]
+        include_dist_info = set()
+        if self._pex_info.includes_tools:
+            # The `repository extract` tool needs setuptools and wheel to build sdists and wheels
+            # and distutils needs .dist-info to discover setuptools (and wheel).
+            for project in "setuptools", "wheel":
+                root_module_names.append(project)
+                include_dist_info.add(project)
+
+        prepared_sources = vendor.vendor_runtime(
             chroot=self._chroot,
             dest_basedir=self._pex_info.bootstrap,
             label="bootstrap",
-            # NB: We use pip here in the builder, but that's only at buildtime and
-            # although we don't use pyparsing directly, packaging.markers, which we
-            # do use at runtime, does.
-            root_module_names=["attr", "packaging", "pkg_resources", "pyparsing"],
+            root_module_names=root_module_names,
+            include_dist_info=include_dist_info,
         )
-        if self._pex_info.includes_tools:
-            # The `repository extract` tool needs setuptools and wheel to build sdists and wheels
-            # and distutils needs .dist-info to discover setuptools (and wheel).
-            vendor.vendor_runtime(
-                chroot=self._chroot,
-                dest_basedir=self._pex_info.bootstrap,
-                label="bootstrap",
-                root_module_names=["setuptools", "wheel"],
-                include_dist_info=True,
-            )
 
         bootstrap_digest = hashlib.sha1()
         bootstrap_packages = ["third_party", "venv"]
@@ -591,9 +591,11 @@ def _prepare_bootstrap(self):
                 dirs[:] = bootstrap_packages
 
             for f in filter_pyc_files(files):
-                if f.endswith("testing.py"):
-                    continue
                 abs_src = os.path.join(root, f)
+                # N.B.: Some of the `pex.*` package files (__init__.py) will already have been
+                # prepared when vendoring the runtime above; so we skip them here.
+                if abs_src in prepared_sources:
+                    continue
                 with open(abs_src, "rb") as fp:
                     data = fp.read()
                 self._chroot.write(
diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py
--- a/pex/vendor/__init__.py
+++ b/pex/vendor/__init__.py
@@ -9,12 +9,12 @@
 import sys
 from textwrap import dedent
 
-from pex.common import filter_pyc_dirs, filter_pyc_files, safe_mkdtemp, touch
+from pex.common import Chroot, filter_pyc_dirs, filter_pyc_files, safe_mkdtemp, touch
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, Iterator, Optional, Sequence, Text, Tuple
+    from typing import Iterable, Iterator, Optional, Sequence, Set, Text, Tuple
 
 _PACKAGE_COMPONENTS = __name__.split(".")
 
@@ -254,7 +254,14 @@ def iter_vendor_specs(filter_requires_python=False):
     yield VendorSpec.pinned("wheel", "0.37.1", rewrite=False)
 
 
-def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_info=False):
+def vendor_runtime(
+    chroot,  # type: Chroot
+    dest_basedir,  # type: str
+    label,  # type: str
+    root_module_names,  # type: Iterable[str]
+    include_dist_info=(),  # type: Iterable[str]
+):
+    # type: (...) -> Set[str]
     """Includes portions of vendored distributions in a chroot.
 
     The portion to include is selected by root module name. If the module is a file, just it is
@@ -262,18 +269,17 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_
     recursively.
 
     :param chroot: The chroot to add vendored code to.
-    :type chroot: :class:`pex.common.Chroot`
-    :param str dest_basedir: The prefix to store the vendored code under in the ``chroot``.
-    :param str label: The chroot label for the vendored code fileset.
+    :param dest_basedir: The prefix to store the vendored code under in the ``chroot``.
+    :param label: The chroot label for the vendored code fileset.
     :param root_module_names: The names of the root vendored modules to include in the chroot.
-    :type root_module_names: :class:`collections.Iterable` of str
-    :param bool include_dist_info: Include the .dist-info dirs associated with the root module
-                                   names.
+    :param include_dist_info: Include the .dist-info dirs associated with these root module names.
+    :returns: The set of absolute paths of the source files that were vendored.
     :raise: :class:`ValueError` if any of the given ``root_module_names`` could not be found amongst
             the vendored code and added to the chroot.
     """
     vendor_module_names = {root_module_name: False for root_module_name in root_module_names}
 
+    vendored_sources = set()  # type: Set[str]
     for spec in iter_vendor_specs():
         for root, dirs, files in os.walk(spec.target_dir):
             if root == spec.target_dir:
@@ -290,6 +296,8 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_
                     pkg_path = os.path.join(pkg_path, pkg)
                     pkg_file = os.path.join(pkg_path, "__init__.py")
                     src = os.path.join(VendorSpec.ROOT, pkg_file)
+                    if src in vendored_sources:
+                        continue
                     dest = os.path.join(dest_basedir, pkg_file)
                     if os.path.exists(src):
                         chroot.copy(src, dest, label)
@@ -297,11 +305,17 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_
                         # We delete `pex/vendor/_vendored/<dist>/__init__.py` when isolating
                         # third_party.
                         chroot.touch(dest, label)
+                    vendored_sources.add(src)
+
                 for name in vendored_names:
                     vendor_module_names[name] = True
                     TRACER.log("Vendoring {} from {} @ {}".format(name, spec, spec.target_dir), V=3)
+
                 dirs[:] = packages + [
-                    d for d in dirs if include_dist_info and d.endswith(".dist-info")
+                    d
+                    for project in include_dist_info
+                    for d in dirs
+                    if d.startswith(project) and d.endswith(".dist-info")
                 ]
                 files[:] = modules
 
@@ -309,10 +323,13 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_
             dirs[:] = filter_pyc_dirs(dirs)
             for filename in filter_pyc_files(files):
                 src = os.path.join(root, filename)
+                if src in vendored_sources:
+                    continue
                 dest = os.path.join(
                     dest_basedir, spec.relpath, os.path.relpath(src, spec.target_dir)
                 )
                 chroot.copy(src, dest, label)
+                vendored_sources.add(src)
 
     if not all(vendor_module_names.values()):
         raise ValueError(
@@ -325,3 +342,5 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names, include_dist_
                 ),
             )
         )
+
+    return vendored_sources


Test patch:
diff --git a/tests/integration/test_issue_2203.py b/tests/integration/test_issue_2203.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_2203.py
@@ -0,0 +1,57 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+import stat
+import subprocess
+
+from pex.common import safe_rmtree
+from pex.typing import TYPE_CHECKING
+from pex.venv.virtualenv import Virtualenv
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_read_only_venv(
+    tmpdir,  # type: Any
+    pex_project_dir,  # type: str
+):
+    # type: (...) -> None
+
+    venv_dir = os.path.join(str(tmpdir), "venv")
+    venv = Virtualenv.create(venv_dir)
+    venv.install_pip()
+    subprocess.check_call(args=[venv.bin_path("pip"), "install", pex_project_dir])
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+
+    def assert_pex_works():
+        safe_rmtree(pex_root)
+        assert (
+            "Moo!"
+            in subprocess.check_output(
+                args=[
+                    venv.bin_path("pex"),
+                    "--pex-root",
+                    pex_root,
+                    "--runtime-pex-root",
+                    pex_root,
+                    "cowsay",
+                    "-c",
+                    "cowsay",
+                    "--",
+                    "Moo!",
+                ]
+            ).decode("utf-8")
+        )
+
+    assert_pex_works()
+
+    write_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
+    for root, dirs, files in os.walk(venv.site_packages_dir, topdown=False):
+        for path in files + dirs:
+            abs_path = os.path.join(root, path)
+            os.chmod(abs_path, os.stat(abs_path).st_mode & ~write_mask)
+
+    assert_pex_works()