Repository: pex-tool/pex

Issue:
PEXBuilder symlinked wheel install chroots can observe compilation.
Seen in CI - looks like so, where `/tmp/tmpo358ye79/.deps/urllib3-1.26.4-py2.py3-none-any.whl` is a symlink pointing out to `~/.pex/installed_wheels/...`:
```
"/home/runner/work/pex/pex/pex/pex_builder.py", line 626, in build
E self._chroot.zip(tmp_zip, mode="a", deterministic_timestamp=deterministic_timestamp)
E File "/home/runner/work/pex/pex/pex/common.py", line 697, in zip
E write_entry(filename, arcname)
E File "/home/runner/work/pex/pex/pex/common.py", line 658, in write_entry
E zip_entry = zf.zip_entry_from_file(
E File "/home/runner/work/pex/pex/pex/common.py", line 165, in zip_entry_from_file
E st = os.stat(filename)
E FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpo358ye79/.deps/urllib3-1.26.4-py2.py3-none-any.whl/urllib3/packages/__pycache__/six.cpython-39.pyc.140328414463456'
```



Discussion:

Resolving patch:
diff --git a/pex/common.py b/pex/common.py
--- a/pex/common.py
+++ b/pex/common.py
@@ -24,7 +24,17 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Any, DefaultDict, Iterable, Iterator, NoReturn, Optional, Set, Sized
+    from typing import (
+        Any,
+        Callable,
+        DefaultDict,
+        Iterable,
+        Iterator,
+        NoReturn,
+        Optional,
+        Set,
+        Sized,
+    )
 
 # We use the start of MS-DOS time, which is what zipfiles use (see section 4.4.6 of
 # https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT).
@@ -43,17 +53,23 @@ def filter_pyc_dirs(dirs):
 
 def filter_pyc_files(files):
     # type: (Iterable[str]) -> Iterator[str]
-    """Return an iterator over the input `files` filtering out any Python bytecode files."""
+    """Iterate the input `files` filtering out any Python bytecode files."""
     for f in files:
         # For Python 2.7, `.pyc` files are compiled as siblings to `.py` files (there is no
-        # __pycache__ dir). We rely on the fact that the temporary files created by CPython
-        # have object id (integer) suffixes to avoid picking up either finished `.pyc` files
-        # or files where Python bytecode compilation is in-flight; i.e.:
-        # `.pyc.0123456789`-style files.
-        if not re.search(r"\.pyc(?:\.[0-9]+)?$", f):
+        # __pycache__ dir).
+        if not f.endswith(".pyc") and not is_pyc_temporary_file(f):
             yield f
 
 
+def is_pyc_temporary_file(file_path):
+    # type: (str) -> bool
+    """Check if `file` is a temporary Python bytecode file."""
+    # We rely on the fact that the temporary files created by CPython have object id (integer)
+    # suffixes to avoid picking up files where Python bytecode compilation is in-flight; i.e.:
+    # `.pyc.0123456789`-style files.
+    return re.search(r"\.pyc\.[0-9]+$", file_path) is not None
+
+
 def die(msg, exit_code=1):
     # type: (str, int) -> NoReturn
     print(msg, file=sys.stderr)
@@ -651,7 +667,14 @@ def __str__(self):
     def delete(self):
         shutil.rmtree(self.chroot)
 
-    def zip(self, filename, mode="w", deterministic_timestamp=False):
+    def zip(
+        self,
+        filename,  # type: str
+        mode="w",  # type: str
+        deterministic_timestamp=False,  # type: bool
+        exclude_file=lambda _: False,  # type: Callable[[str], bool]
+    ):
+        # type: (...) -> None
         with open_zip(filename, mode) as zf:
 
             def write_entry(filename, arcname):
@@ -684,10 +707,15 @@ def iter_files():
                 for path in sorted(self.files()):
                     full_path = os.path.join(self.chroot, path)
                     if os.path.isfile(full_path):
+                        if exclude_file(full_path):
+                            continue
                         yield full_path, path
                         continue
+
                     for root, _, files in os.walk(full_path):
                         for f in sorted(files):
+                            if exclude_file(f):
+                                continue
                             abs_path = os.path.join(root, f)
                             rel_path = os.path.join(path, os.path.relpath(abs_path, full_path))
                             yield abs_path, rel_path
diff --git a/pex/pex_builder.py b/pex/pex_builder.py
--- a/pex/pex_builder.py
+++ b/pex/pex_builder.py
@@ -8,7 +8,15 @@
 import os
 
 from pex import pex_warnings
-from pex.common import Chroot, chmod_plus_x, open_zip, safe_mkdtemp, safe_open, temporary_dir
+from pex.common import (
+    Chroot,
+    chmod_plus_x,
+    is_pyc_temporary_file,
+    open_zip,
+    safe_mkdtemp,
+    safe_open,
+    temporary_dir,
+)
 from pex.compatibility import to_bytes
 from pex.compiler import Compiler
 from pex.distribution_target import DistributionTarget
@@ -623,7 +631,19 @@ def build(self, filename, bytecode_compile=True, deterministic_timestamp=False):
             assert os.path.getsize(pexfile.name) == 0
             pexfile.write(to_bytes("{}\n".format(self._shebang)))
         with TRACER.timed("Zipping PEX file."):
-            self._chroot.zip(tmp_zip, mode="a", deterministic_timestamp=deterministic_timestamp)
+            self._chroot.zip(
+                tmp_zip,
+                mode="a",
+                deterministic_timestamp=deterministic_timestamp,
+                # When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions
+                # as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since
+                # those installed wheels reside in a shared cache, they can be in-use by other
+                # processes and so their code may be in the process of being bytecode compiled as we
+                # attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary
+                # pyc files that we should avoid copying since they are unuseful and inherently
+                # racy.
+                exclude_file=is_pyc_temporary_file,
+            )
         if os.path.exists(filename):
             os.unlink(filename)
         os.rename(tmp_zip, filename)


Test patch:
diff --git a/tests/test_common.py b/tests/test_common.py
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -233,6 +233,7 @@ def test_chroot_zip_symlink():
     with temporary_dir() as tmp:
         chroot = Chroot(os.path.join(tmp, "chroot"))
         chroot.write(b"data", "directory/subdirectory/file")
+        chroot.write(b"data", "directory/subdirectory/file.foo")
         chroot.symlink(
             os.path.join(chroot.path(), "directory/subdirectory/file"),
             "directory/subdirectory/symlinked",
@@ -250,7 +251,7 @@ def test_chroot_zip_symlink():
 
         chroot.symlink(os.path.join(chroot.path(), "directory"), "symlinked")
         zip_dst = os.path.join(tmp, "chroot.zip")
-        chroot.zip(zip_dst)
+        chroot.zip(zip_dst, exclude_file=lambda path: path.endswith(".foo"))
         with open_zip(zip_dst) as zip:
             assert [
                 "directory/",