Repository: linkedin/shiv

Issue:
Environments created with symlinked folders (by PDM with cache=True) are not correctly packed
Currently shiv uses `rglob('*')` to find all files in the site-packages directory; rglob does not follow symlinks (https://github.com/python/cpython/issues/77609). PDM uses folder symlinks when cache is enabled, to optimize performance and save space when managing multiple environments. Such environments cannot currently be used to generate shiv, because shiv will not pack folders which are symlinks.

Comments:

Patch:
diff --git a/setup.cfg b/setup.cfg
index 36d394a..f0e3dca 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -24,7 +24,7 @@ sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
 
 [metadata]
 name = shiv
-version = 1.0.3
+version = 1.0.4
 description = A command line utility for building fully self contained Python zipapps.
 long_description = file: README.md
 long_description_content_type = text/markdown
diff --git a/src/shiv/builder.py b/src/shiv/builder.py
index e4c14aa..67a7753 100644
--- a/src/shiv/builder.py
+++ b/src/shiv/builder.py
@@ -12,9 +12,10 @@ import zipapp
 import zipfile
 
 from datetime import datetime, timezone
+from itertools import chain
 from pathlib import Path
 from stat import S_IFMT, S_IMODE, S_IXGRP, S_IXOTH, S_IXUSR
-from typing import IO, Any, List, Optional, Tuple
+from typing import Generator, IO, Any, List, Optional, Tuple
 
 from . import bootstrap
 from .bootstrap.environment import Environment
@@ -69,6 +70,15 @@ def write_to_zipapp(
     archive.writestr(zinfo, data)
 
 
+def rglob_follow_symlinks(path: Path, glob: str) -> Generator[Path, None, None]:
+    """Path.rglob extended to follow symlinks, while we wait for Python 3.13."""
+    for p in path.rglob('*'):
+        if p.is_symlink() and p.is_dir():
+            yield from chain([p], rglob_follow_symlinks(p, glob))
+        else:
+            yield p
+
+
 def create_archive(
     sources: List[Path], target: Path, interpreter: str, main: str, env: Environment, compressed: bool = True
 ) -> None:
@@ -110,7 +120,11 @@ def create_archive(
                 # Glob is known to return results in non-deterministic order.
                 # We need to sort them by in-archive paths to ensure
                 # that archive contents are reproducible.
-                for path in sorted(source.rglob("*"), key=str):
+                #
+                # NOTE: https://github.com/linkedin/shiv/issues/236
+                # this special rglob function can be replaced with "rglob('*', follow_symlinks=True)"
+                # when Python 3.13 becomes the lowest supported version
+                for path in sorted(rglob_follow_symlinks(source, "*"), key=str):
 
                     # Skip compiled files and directories (as they are not required to be present in the zip).
                     if path.suffix == ".pyc" or path.is_dir():


Test patch:
diff --git a/test/test_builder.py b/test/test_builder.py
index 4d8a357..ec5af35 100644
--- a/test/test_builder.py
+++ b/test/test_builder.py
@@ -9,7 +9,7 @@ from zipapp import ZipAppError
 
 import pytest
 
-from shiv.builder import create_archive, write_file_prefix
+from shiv.builder import create_archive, rglob_follow_symlinks, write_file_prefix
 
 UGOX = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 
@@ -39,6 +39,16 @@ class TestBuilder:
         with pytest.raises(SystemExit):
             tmp_write_prefix(f"/{'c' * 200}/python")
 
+    def test_rglob_follow_symlinks(self, tmp_path):
+        real_dir = tmp_path / 'real_dir'
+        real_dir.mkdir()
+        real_file = real_dir / 'real_file'
+        real_file.touch()
+        sym_dir = tmp_path / 'sym_dir'
+        sym_dir.symlink_to(real_dir)
+        sym_file = sym_dir / real_file.name
+        assert sorted(rglob_follow_symlinks(tmp_path, '*'), key=str) == [real_dir, real_file, sym_dir, sym_file]
+
     def test_create_archive(self, sp, env):
         with tempfile.TemporaryDirectory() as tmpdir:
             target = Path(tmpdir, "test.zip")