Repository: pex-tool/pex

Issue:
Pex third_party isolation fails for packed PEX containing Pex 2.1.48+ as a dep.
Seen over in Pants:
```console
$ git describe
release_2.8.0.dev1
$ PANTS_PEX_RELEASE=STABLE ./build-support/bin/release.sh build-universal-pex
...
[=== 05:36 Checking prebuilt wheels for 2.8.0.dev1+git11bacb90 ===]
All 2 pantsbuild.pants packages were fetched and are valid.
/tmp/tmpb7qudv1z/pex/.deps/pex-2.1.48-py2.py3-none-any.whl/pex/bin/pex.py:1168: PEXWarning: The pex root has been set via PEX_ROOT but --disable-cache is also set. Ignoring PEX_ROOT and disabling caches.
Failed to spawn a job for DistributionTarget(platform=Platform(platform='linux_x86_64', impl='cp', version='38', abi='cp38')): Expected the `pex` module to be available via an installed distribution or else via an installed or loose PEX. Loaded the `pex` module from /tmp/tmpb7qudv1z/pex/.deps/pex-2.1.48-py2.py3-none-any.whl/pex and argv0 is /tmp/tmpb7qudv1z/pex.
Traceback (most recent call last):
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 476, in execute
    exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 401, in _wrap_coverage
    return runner(*args)
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 432, in _wrap_profiling
    return runner(*args)
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 532, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 679, in execute_entry
    return self.execute_module(entry_point, alter_sys)
  File "/home/jsirois/.cache/pants/named_caches/pex_root/unzipped_pexes/28f4c740b4fd904afa532678dde9783c8d7487c8/.bootstrap/pex/pex.py", line 691, in execute_module
    runpy.run_module(module_name, run_name="__main__", alter_sys=alter_sys)
  File "/home/jsirois/.pyenv/versions/3.7.11/lib/python3.7/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/home/jsirois/.pyenv/versions/3.7.11/lib/python3.7/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/home/jsirois/.pyenv/versions/3.7.11/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jsirois/dev/pantsbuild/pants/.pants.d/tmp7dbk0pvm/build-support/bin/_release_helper.py", line 977, in <module>
    main()
  File "/home/jsirois/dev/pantsbuild/pants/.pants.d/tmp7dbk0pvm/build-support/bin/_release_helper.py", line 965, in main
    build_pex(fetch=True)
  File "/home/jsirois/dev/pantsbuild/pants/.pants.d/tmp7dbk0pvm/build-support/bin/_release_helper.py", line 604, in build_pex
    check=True,
  File "/home/jsirois/.pyenv/versions/3.7.11/lib/python3.7/subprocess.py", line 512, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['/home/jsirois/.pyenv/versions/3.7.11/bin/python3.7', '/tmp/tmpb7qudv1z/pex', '-o', 'dist/pants.2.8.0.dev1+git11bacb90.pex', '--no-build', '--no-pypi', '--disable-cache', '-f', '/home/jsirois/dev/pantsbuild/pants/dist/deploy/wheels/pantsbuild.pants/11bacb90b483018c9169dce47901102ac82bcd91/2.8.0.dev1+git11bacb90', '-f', '/home/jsirois/dev/pantsbuild/pants/dist/deploy/wheels/3rdparty/11bacb90b483018c9169dce47901102ac82bcd91/2.8.0.dev1+git11bacb90', '--no-strip-pex-env', '--console-script=pants', '--platform=linux_x86_64-cp-37-m', '--platform=linux_x86_64-cp-38-cp38', '--platform=linux_x86_64-cp-39-cp39', '--platform=macosx_10.15_x86_64-cp-37-m', '--platform=macosx_10.15_x86_64-cp-38-cp38', '--platform=macosx_10.15_x86_64-cp-39-cp39', 'pantsbuild.pants==2.8.0.dev1+git11bacb90']' returned non-zero exit status 1.
```


Discussion:
The issue is in PEX layout identification here:
https://github.com/pantsbuild/pex/blob/e0ad91f7542e174fe878113cbad39823eff4f2a9/pex/layout.py#L277-L282
The `is_python_script` checks the executable bit is set:
https://github.com/pantsbuild/pex/blob/e0ad91f7542e174fe878113cbad39823eff4f2a9/pex/common.py#L514-L516
https://github.com/pantsbuild/pex/blob/e0ad91f7542e174fe878113cbad39823eff4f2a9/pex/common.py#L491-L505

The Pants code in the OP downloads an official Pex PEX and runs it via `python pex`; so it never needs to (re)establish the execute mode bit on the Pex PEX file.

Resolving patch:
diff --git a/pex/common.py b/pex/common.py
--- a/pex/common.py
+++ b/pex/common.py
@@ -491,17 +491,19 @@ def is_exe(path):
 def is_script(
     path,  # type: str
     pattern=None,  # type: Optional[str]
+    check_executable=True,  # type: bool
 ):
     # type: (...) -> bool
-    """Determines if the given path is a script executable by the current user.
+    """Determines if the given path is a script.
 
-    A script is an executable (`is_exe` is True) that starts with a shebang (#!...) line.
+    A script is a file that starts with a shebang (#!...) line.
 
     :param path: The path to check.
     :param pattern: An optional pattern to match against the shebang (excluding the leading #!).
-    :return: `True if the given path is a script executable by the current user.
+    :param check_executable: Check that the script is executable by the current user.
+    :return: True if the given path is a script.
     """
-    if not is_exe(path):
+    if check_executable and not is_exe(path):
         return False
     with open(path, "rb") as fp:
         if b"#!" != fp.read(2):
@@ -511,9 +513,12 @@ def is_script(
         return bool(re.match(pattern, fp.readline().decode("utf-8")))
 
 
-def is_python_script(path):
-    # type: (str) -> bool
-    return is_script(path, pattern=r"(?i)^.*(?:python|pypy)")
+def is_python_script(
+    path,  # type: str
+    check_executable=True,  # type: bool
+):
+    # type: (...) -> bool
+    return is_script(path, pattern=r"(?i)^.*(?:python|pypy)", check_executable=check_executable)
 
 
 def can_write_dir(path):
diff --git a/pex/layout.py b/pex/layout.py
--- a/pex/layout.py
+++ b/pex/layout.py
@@ -277,7 +277,11 @@ def __str__(self):
 @contextmanager
 def _identify_layout(pex):
     # type: (str) -> Iterator[Optional[_Layout]]
-    if zipfile.is_zipfile(pex) and is_python_script(pex):
+    if zipfile.is_zipfile(pex) and is_python_script(
+        pex,
+        # N.B.: A PEX file need not be executable since it can always be run via `python a.pex`.
+        check_executable=False,
+    ):
         with open_zip(pex) as zfp:
             yield _ZipAppPEX(pex, zfp)
     elif os.path.isdir(pex) and zipfile.is_zipfile(os.path.join(pex, BOOTSTRAP_DIR)):


Test patch:
diff --git a/tests/integration/test_issue_1447.py b/tests/integration/test_issue_1447.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_1447.py
@@ -0,0 +1,38 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import os
+import shutil
+import subprocess
+import sys
+
+from pex.pex_info import PexInfo
+from pex.testing import run_pex_command
+from pex.typing import TYPE_CHECKING
+from pex.variables import unzip_dir
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_layout_identification(tmpdir):
+    # type: (Any) -> None
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    pex_file = os.path.join(str(tmpdir), "a.pex")
+    run_pex_command(
+        args=["-o", pex_file, "--pex-root", pex_root, "--runtime-pex-root", pex_root]
+    ).assert_success()
+
+    pex_hash = PexInfo.from_pex(pex_file).pex_hash
+    assert pex_hash is not None
+
+    expected_unzip_dir = unzip_dir(pex_root, pex_hash)
+    assert not os.path.exists(expected_unzip_dir)
+
+    subprocess.check_call(args=[pex_file, "-c", ""])
+    assert os.path.isdir(expected_unzip_dir)
+
+    shutil.rmtree(expected_unzip_dir)
+    os.chmod(pex_file, 0o644)
+    subprocess.check_call(args=[sys.executable, pex_file, "-c", ""])
+    assert os.path.isdir(expected_unzip_dir)
diff --git a/tests/test_common.py b/tests/test_common.py
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -383,5 +383,6 @@ def test_is_script(tmpdir):
     assert not is_script(exe, pattern=r"^python")
 
     os.chmod(exe, 0o665)
+    assert is_script(exe, check_executable=False)
     assert not is_script(exe)
     assert not is_exe(exe)