Repository: pex-tool/pex

Issue:
Pex fails to generate hermetic wheel installs on Linux hosts when the install target path is >127 characters long
For example:

1. Hermetic:
```
$ rm -rf /tmp/pex_root && pex --pex-root /tmp/pex_root mypy_protobuf==2.4 --intransitive -o mypy_protobuf.pex
$ find /tmp/pex_root/installed_wheels/ -name "protoc-gen-mypy" | while read script; do echo "$script:" && head -3 $script; done 
/tmp/pex_root/installed_wheels/60974b0bc64909a12b2201d5d0093478d955e90a/mypy_protobuf-2.4-py3-none-any.whl/bin/protoc-gen-mypy:
#!python
# -*- coding: utf-8 -*-
import re
```

2. Not hermetic:
```
$ pex_root="/tmp/$(echo | sha512sum | cut -d' ' -f1)"
$ echo $pex_root
/tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09
$ echo $pex_root | wc -c
134
$ rm -rf $pex_root && pex --pex-root $pex_root mypy_protobuf==2.4 --intransitive -o mypy_protobuf.pex
/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/pex_bootstrapper.py:483: PEXWarning: The venv for /tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09/pip.pex/8781476b36a7a6b86d240e88ab26d74d7bc81315 at /tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09/venvs/6d965e35637dae05966e8439822c63a427c2a522/108a3ddc84230ab282ea6312e06cb68f51008ce5 has script shebangs of '#!/tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09/venvs/s/adaddbff/venv/bin/python3 -sE' with 173 characters. On some systems this may be too long and cause problems running the venv scripts. You may be able adjust PEX_ROOT from /tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09 to a shorter path as a work-around.
  pex_warnings.warn(
$ find $pex_root/installed_wheels/ -name "protoc-gen-mypy" | while read script; do echo "$script:" && head -3 $script; done 
/tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09/installed_wheels/60974b0bc64909a12b2201d5d0093478d955e90a/mypy_protobuf-2.4-py3-none-any.whl/bin/protoc-gen-mypy:
#!/bin/sh
'''exec' /tmp/be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09/venvs/s/adaddbff/venv/bin/python3 "$0" "$@"
' '''
```



Discussion:
The code doing this is here:
https://github.com/pantsbuild/pex/blob/196b4cd5b8dd4b4af2586460530e9a777262be7d/pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py#L127-L156
I think Pex will have to become console_script entry-point aware in its post-processing of installed wheel chroots. The ~spec is here: https://packaging.python.org/specifications/entry-points/ Using that, Pex could single out console_scripts and more aggressively re-write them to have `#!python` shebangs.
I'm almost certainly missing something...could we fork the `distutils` code to always use the "simple_shebang"? I think I'm not understanding the `executable` variable, whether it will always be `python` or not like the example shows.
Yeah - you're missing something, you need to read more of the pip / distlib vendored code.

Simply though, every fork makes upgrading vendored Pip harder. My strategy has been 0 forks and instead using fixups after running `pip download`, `pip wheel` or `pip install`. I already do `pip install` fixups here so this is just more of that:
https://github.com/pantsbuild/pex/blob/da25bae605407b12461180fc8701cd0e78934b8e/pex/pip.py#L931-L1065

Resolving patch:
diff --git a/pex/pip.py b/pex/pip.py
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -44,7 +44,7 @@
 )
 from pex.resolve.resolver_configuration import ResolverVersion
 from pex.third_party import isolated
-from pex.third_party.pkg_resources import Requirement
+from pex.third_party.pkg_resources import EntryPoint, Requirement
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, Generic, cast
 from pex.util import CacheHelper, named_temporary_file
@@ -935,16 +935,6 @@ def _fixup_install(
         install_dir,  # type: str
     ):
         # type: (...) -> None
-        modified_scripts = list(cls._fixup_scripts(install_dir))
-        cls._fixup_record(dist, install_dir, modified_scripts=modified_scripts)
-
-    @staticmethod
-    def _fixup_record(
-        dist,  # type: str
-        install_dir,  # type: str
-        modified_scripts=None,  # type: Optional[Iterable[str]]
-    ):
-        # type: (...) -> None
 
         # The dist-info metadata directory is named as specifed in:
         #   https://www.python.org/dev/peps/pep-0427/
@@ -969,23 +959,45 @@ def _fixup_record(
             for root, _, files in os.walk(install_dir)
             for f in files
         ]
-        record_relpath = dist_metadata.find_dist_info_file(
-            project_name=project_name_and_version.project_name,
-            version=project_name_and_version.version,
-            filename="RECORD",
-            listing=listing,
+
+        def find_dist_info_file(filename):
+            # type: (str) -> Optional[str]
+            return dist_metadata.find_dist_info_file(
+                project_name=project_name_and_version.project_name,
+                version=project_name_and_version.version,
+                filename=filename,
+                listing=listing,
+            )
+
+        entry_points_relpath = find_dist_info_file("entry_points.txt")
+        modified_scripts = list(
+            cls._fixup_scripts(install_dir, entry_points_relpath=entry_points_relpath)
         )
-        if not record_relpath:
-            return
+        record_relpath = find_dist_info_file("RECORD")
+        if record_relpath:
+            direct_url_relpath = find_dist_info_file("direct_url.json")
+            cls._fixup_record(
+                install_dir,
+                record_relpath,
+                direct_url_relpath=direct_url_relpath,
+                modified_scripts=modified_scripts,
+            )
+
+    @staticmethod
+    def _fixup_record(
+        install_dir,  # type: str
+        record_relpath,  # type: str
+        direct_url_relpath,  # type: Optional[str]
+        modified_scripts=None,  # type: Optional[Iterable[str]]
+    ):
+        # type: (...) -> None
 
         exclude_relpaths = []
 
         record_abspath = os.path.join(install_dir, record_relpath)
-        dist_info_dir = os.path.dirname(record_abspath)
 
-        direct_url_abspath = os.path.join(dist_info_dir, "direct_url.json")
-        direct_url_relpath = os.path.relpath(direct_url_abspath, install_dir)
-        if os.path.exists(direct_url_abspath):
+        if direct_url_relpath:
+            direct_url_abspath = os.path.join(install_dir, direct_url_relpath)
             with open(direct_url_abspath) as fp:
                 if urlparse.urlparse(json.load(fp)["url"]).scheme == "file":
                     exclude_relpaths.append(os.path.relpath(direct_url_abspath, install_dir))
@@ -994,8 +1006,8 @@ def _fixup_record(
         to_rehash = {}
         if modified_scripts:
             for modified_script in modified_scripts:
-                # N.B.: Pip installs wheels with RECORD entries like `../../bin/script` even when it's
-                # called in `--target <dir>` mode which installs the script in `bin/script`.
+                # N.B.: Pip installs wheels with RECORD entries like `../../bin/script` even when
+                # it's called in `--target <dir>` mode which installs the script in `bin/script`.
                 record_relpath = os.path.join(os.pardir, os.pardir, modified_script)
                 modified_script_abspath = os.path.join(install_dir, modified_script)
                 to_rehash[record_relpath] = modified_script_abspath
@@ -1037,32 +1049,74 @@ def _fixup_record(
     @staticmethod
     def _fixup_scripts(
         install_dir,  # type: str
+        entry_points_relpath,  # type: Optional[str]
     ):
         # type: (...) -> Iterator[str]
         bin_dir = os.path.join(install_dir, "bin")
         if not os.path.isdir(bin_dir):
             return
 
-        scripts = []
+        console_scripts = {}  # type: Dict[str, EntryPoint]
+        if entry_points_relpath:
+            entry_points_abspath = os.path.join(install_dir, entry_points_relpath)
+            with open(entry_points_abspath) as fp:
+                console_scripts.update(EntryPoint.parse_map(fp.read()).get("console_scripts", {}))
+
+        scripts = {}  # type: Dict[str, Optional[bytes]]
         for script_name in os.listdir(bin_dir):
             script_path = os.path.join(bin_dir, script_name)
             if is_python_script(script_path):
-                scripts.append(script_path)
+                scripts[script_path] = None
+            elif script_name in console_scripts:
+                # When a wheel is installed by Pip and that wheel contains console_scripts, they are
+                # normally written with a faux-shebang of:
+                # #!python
+                #
+                # Pex relies on this hermetic shebang and only ever reifies it when creating venvs.
+                #
+                # If Pip is being run under a Python executable with a path length >127 characters
+                # on Linux though, it writes a shebang / header of:
+                # #!/bin/sh
+                # '''exec' <too long path to Pip venv python> "$0" "$@"'
+                # ' '''
+                #
+                # That header is immediately followed by the expected console_script shim contents:
+                # # -*- coding: utf-8 -*-
+                # import re
+                # import sys
+                # from <ep_module> import <ep_func>
+                # if __name__ == '__main__':
+                #     sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+                #     sys.exit(main())
+                #
+                # Instead of guessing that 127 characters is the shebang length limit and using
+                # Pip's safety-hatch `/bin/sh` trick, we forcibly re-write the header to be just the
+                # expected `#!python` shebang. We detect the end of the header with the known 1st
+                # line of console_script shim ~code defined in
+                # pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py on line 41:
+                # https://github.com/pantsbuild/pex/blob/196b4cd5b8dd4b4af2586460530e9a777262be7d/pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py#L41
+                scripts[script_path] = b"# -*- coding: utf-8 -*-"
         if not scripts:
             return
 
-        with closing(fileinput.input(files=scripts, inplace=True, mode="rb")) as script_fi:
+        with closing(fileinput.input(files=scripts.keys(), inplace=True, mode="rb")) as script_fi:
+            first_non_shebang_line = None  # type: Optional[bytes]
             for line in script_fi:
                 buffer = get_stdout_bytes_buffer()
                 if script_fi.isfirstline():
+                    first_non_shebang_line = scripts[script_fi.filename()]
                     # Ensure python shebangs are reproducible. The only place these can be used is
                     # in venv mode PEXes where the `#!python` placeholder shebang will be re-written
                     # to use the venv's python interpreter.
                     buffer.write(b"#!python\n")
                     yield os.path.relpath(script_fi.filename(), install_dir)
-                else:
+                elif (
+                    not first_non_shebang_line
+                    or cast(bytes, line).strip() == first_non_shebang_line
+                ):
                     # N.B.: These lines include the newline already.
                     buffer.write(cast(bytes, line))
+                    first_non_shebang_line = None
 
     def spawn_install_wheel(
         self,


Test patch:
diff --git a/tests/integration/test_issue_1520.py b/tests/integration/test_issue_1520.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_1520.py
@@ -0,0 +1,70 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os
+import shutil
+import subprocess
+
+import pytest
+
+from pex.testing import IS_LINUX, IS_MAC, PY_VER, run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@pytest.mark.skipif(
+    PY_VER < (3, 6), reason="The mypy_protobuf 2.4 distribution is only available for Python 3.6+"
+)
+@pytest.mark.xfail(
+    IS_MAC,
+    reason=(
+        "On modern Linux (starting with the 5.1 kernel shipped on May 19th 2019), the default max "
+        "shebang length limit is 256 but the hardcoded limit in Pip that #1520 fixes is 127; so "
+        "the work-around here should test green on Linux. On Mac, however, the hardcoded limit in "
+        "Pip that #1520 fixes is 512 and that limit has been stable on macOS; so we expect the PEX "
+        "creation to fail with something like: [Errno 63] File name too long: '/tmp/"
+        "pytest-of-runner/pytest-0/popen-gw2/test_hermetic_console_scripts0/<512 of `_`>/pex_root/"
+        "isolated/.488310d43ea7ca80b559c306f2db44914a184e37.atomic_directory.lck'."
+    ),
+)
+def test_hermetic_console_scripts(tmpdir):
+    # type: (Any) -> None
+
+    # N.B.: See pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py lines 127-156.
+    # https://github.com/pantsbuild/pex/blob/196b4cd5b8dd4b4af2586460530e9a777262be7d/pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py#L127-L156
+    length_pad = 127 if IS_LINUX else 512
+    pex_root = os.path.join(str(tmpdir), "_" * length_pad, "pex_root")
+    assert len(pex_root) > length_pad
+
+    mypy_protobuf_pex = os.path.join(str(tmpdir), "mypy_protobuf.pex")
+
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "mypy_protobuf==2.4",
+            "-o",
+            mypy_protobuf_pex,
+            "--venv",
+            "prepend",
+        ],
+    ).assert_success()
+
+    scripts = [
+        os.path.join(root, f)
+        for root, dirs, files in os.walk(os.path.join(pex_root, "installed_wheels"))
+        for f in files
+        if "protoc-gen-mypy" == f
+    ]
+    assert 1 == len(scripts)
+    with open(scripts[0]) as fp:
+        assert "#!python" == fp.readline().strip()
+        assert "# -*- coding: utf-8 -*-" == fp.readline().strip()
+
+    shutil.rmtree(pex_root)
+    # This should no-op (since there is no proto sent on stdin) and exit success.
+    subprocess.check_call(
+        [mypy_protobuf_pex, "-c", "import subprocess; subprocess.check_call(['protoc-gen-mypy'])"],
+    )