Repository: pex-tool/pex

Issue:
Venv `pex` and bin scripts can run afoul of shebang length limits.
These limits can be as low as 128 bytes for the whole shebang line on Linux depending on how the kernel is configured: https://www.in-ulm.de/~mascheck/various/shebang/.

On my Arch Linux machine I find:
```
$ grep BINPRM_BUF_SIZE /usr/include/linux/binfmts.h 
#define BINPRM_BUF_SIZE 256
``` 

But on Debian 10.7 Buster you get:
```
$ docker run --rm -it debian:10.7
root@f5735d61a8c7:/# apt update && apt install --yes linux-source
root@f5735d61a8c7:/# grep BINPRM_BUF_SIZE /usr/include/linux/binfmts.h
#define BINPRM_BUF_SIZE 128
```


Discussion:

Resolving patch:
diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py
--- a/pex/pex_bootstrapper.py
+++ b/pex/pex_bootstrapper.py
@@ -3,11 +3,13 @@
 
 from __future__ import absolute_import
 
+import errno
+import hashlib
 import os
 import sys
 
 from pex import pex_warnings
-from pex.common import atomic_directory, die
+from pex.common import atomic_directory, die, pluralize, safe_mkdir
 from pex.inherit_path import InheritPath
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError
@@ -377,13 +379,72 @@ def ensure_venv(pex):
                 interpreter=pex.interpreter,
                 copies=pex_info.venv_copies,
             )
-            populate_venv_with_pex(
+
+            pex_path = os.path.abspath(pex.path())
+
+            short_venv_dir = os.path.join(pex_info.pex_root, "venvs", "short")
+            safe_mkdir(short_venv_dir)
+
+            # A sha1 hash is 160 bits -> 20 bytes -> 40 hex characters. We start with 8 characters
+            # (32 bits) of entropy since that is short and _very_ unlikely to collide with another
+            # PEX venv on this machine. If we still collide after using the whole sha1 (for a total
+            # of 33 collisions), then the universe is broken and we raise. It's the least we can do.
+            venv_hash = hashlib.sha1(venv_dir.encode("utf-8")).hexdigest()
+            collisions = []
+            for chars in range(8, len(venv_hash) + 1):
+                entropy = venv_hash[:chars]
+                short_venv_path = os.path.join(short_venv_dir, entropy)
+                try:
+                    os.symlink(venv_dir, short_venv_path)
+                    break
+                except OSError as e:
+                    if e.errno != errno.EEXIST:
+                        raise e
+                    collisions.append(short_venv_path)
+                    if entropy == venv_hash:
+                        raise RuntimeError(
+                            "The venv for {pex} at {venv} has hash collisions with {count} other "
+                            "{venvs}!\n{collisions}".format(
+                                pex=pex_path,
+                                venv=venv_dir,
+                                count=len(collisions),
+                                venvs=pluralize(collisions, "venv"),
+                                collisions="\n".join(
+                                    "{index}.) {venv_path}".format(
+                                        index=index, venv_path=os.path.realpath(path)
+                                    )
+                                    for index, path in enumerate(collisions, start=1)
+                                ),
+                            )
+                        )
+
+            shenbang = populate_venv_with_pex(
                 virtualenv,
                 pex,
                 bin_path=pex_info.venv_bin_path,
-                python=os.path.join(venv_dir, "bin", os.path.basename(pex.interpreter.binary)),
+                python=os.path.join(
+                    short_venv_path, "bin", os.path.basename(pex.interpreter.binary)
+                ),
                 collisions_ok=True,
             )
+
+            # There are popular Linux distributions with shebang length limits (BINPRM_BUF_SIZE
+            # in /usr/include/linux/binfmts.h) set at 128 characters, so we warn in the _very_
+            # unlikely case that our shortened shebang is longer than this.
+            if len(shenbang) > 128:
+                pex_warnings.warn(
+                    "The venv for {pex} at {venv} has script shebangs of {shebang!r} with {count} "
+                    "characters. On some systems this may be too long and cause problems running "
+                    "the venv scripts. You may be able adjust PEX_ROOT from {pex_root} to a "
+                    "shorter path as a work-around.".format(
+                        pex=pex_path,
+                        venv=venv_dir,
+                        shebang=shenbang,
+                        count=len(shenbang),
+                        pex_root=pex_info.pex_root,
+                    )
+                )
+
     return os.path.join(venv_dir, "pex")
 
 
diff --git a/pex/tools/commands/venv.py b/pex/tools/commands/venv.py
--- a/pex/tools/commands/venv.py
+++ b/pex/tools/commands/venv.py
@@ -81,7 +81,7 @@ def populate_venv_with_pex(
     python=None,  # type: Optional[str]
     collisions_ok=True,  # type: bool
 ):
-    # type: (...) -> None
+    # type: (...) -> str
 
     venv_python = python or venv.interpreter.binary
     venv_bin_dir = os.path.dirname(python) if python else venv.bin_dir
@@ -139,9 +139,10 @@ def record_provenance(src_to_dst):
 
     # 2. Add a __main__ to the root of the venv for running the venv dir like a loose PEX dir
     # and a main.py for running as a script.
+    shebang = "#!{} -sE".format(venv_python)
     main_contents = dedent(
         """\
-        #!{venv_python} -sE
+        {shebang}
 
         if __name__ == "__main__":
             import os
@@ -149,8 +150,9 @@ def record_provenance(src_to_dst):
 
             venv_dir = os.path.abspath(os.path.dirname(__file__))
             venv_bin_dir = os.path.join(venv_dir, "bin")
-            python = os.path.join(venv_bin_dir, os.path.basename({venv_python!r}))
-            if sys.executable != python:
+            shebang_python = {shebang_python!r}
+            python = os.path.join(venv_bin_dir, os.path.basename(shebang_python))
+            if sys.executable not in (python, shebang_python):
                 sys.stderr.write("Re-execing from {{}}\\n".format(sys.executable))
                 os.execv(python, [python, "-sE"] + sys.argv)
 
@@ -248,7 +250,8 @@ def record_provenance(src_to_dst):
                     func = namespace = getattr(namespace, attr)
                 sys.exit(func())
         """.format(
-            venv_python=venv_python,
+            shebang=shebang,
+            shebang_python=venv_python,
             bin_path=bin_path,
             entry_point=pex_info.entry_point,
             exec_ast=(
@@ -264,9 +267,11 @@ def record_provenance(src_to_dst):
     os.symlink(os.path.basename(fp.name), venv.join_path("pex"))
 
     # 3. Re-write any (console) scripts to use the venv Python.
-    for script in venv.rewrite_scripts(python=python, python_args="-sE"):
+    for script in venv.rewrite_scripts(python=venv_python, python_args="-sE"):
         TRACER.log("Re-writing {}".format(script))
 
+    return shebang
+
 
 class Venv(Command):
     """Creates a venv from the PEX file."""


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -2688,19 +2688,25 @@ def test_venv_mode(
     def run_isort_pex(**env):
         # type: (**Any) -> str
         pex_root = str(tmpdir)
-        stdout, returncode = run_simple_pex(
-            pex_file,
-            args=["-c", "import sys; print(sys.executable)"],
+        stdout = subprocess.check_output(
+            args=[pex_file, "-c", "import sys; print(sys.executable); print(sys.prefix)"],
             env=make_env(PEX_ROOT=pex_root, PEX_INTERPRETER=1, **env),
         )
-        assert returncode == 0, stdout
-        pex_interpreter = cast(str, stdout.decode("utf-8").strip())
+        pex_interpreter, venv_home = cast(
+            "Tuple[str, str]", stdout.decode("utf-8").strip().splitlines()
+        )
+        actual_venv_home = os.path.realpath(venv_home)
+        assert venv_home != actual_venv_home, "Expected the venv home to be a symlink"
+        assert len(venv_home) < len(
+            actual_venv_home
+        ), "Expected the venv home symlink path length to be shorter than the actual path length"
+
         with ENV.patch(**env):
             pex_info = PexInfo.from_pex(pex_file)
             pex_hash = pex_info.pex_hash
             assert pex_hash is not None
             expected_venv_home = venv_dir(pex_root, pex_hash, interpreter_constraints=[])
-        assert expected_venv_home == os.path.commonprefix([pex_interpreter, expected_venv_home])
+        assert expected_venv_home == os.path.commonprefix([actual_venv_home, expected_venv_home])
         return pex_interpreter
 
     isort_pex_interpreter1 = run_isort_pex()