Repository: pex-tool/pex

Issue:
pex stops interpreter search if even one intepreter fails to identify itself
A user on Pants Slack [reported a pex error](https://pantsbuild.slack.com/archives/C046T6T9U/p1635515621287700) when upgrading to macOS Monterrey:

```
15:46:09.41 [DEBUG] spawned local process as Some(99409) for Process { argv: ["/Users/sinisaderasevic/.pyenv/versions/3.9.7/bin/python", "./pex", "--python-path", "/Users/sinisaderasevic/.pyenv/versions/3.9.7/bin:/Users/sinisaderasevic/.virtualenvs/apollo/bin:/Users/sinisaderasevic/.pyenv/shims:/Users/sinisaderasevic/.pyenv/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "--tmpdir", ".tmp", "--interpreter-constraint", "CPython==3.9.*", "--", "-c", "import hashlib, os, sys\n\npython = os.path.realpath(sys.executable)\nprint(python)\n\nhasher = hashlib.sha256()\nwith open(python, \"rb\") as fp:\n  for chunk in iter(lambda: fp.read(8192), b\"\"):\n      hasher.update(chunk)\nprint(hasher.hexdigest())\n"], env: {"CPPFLAGS": "", "LC_CTYPE": "en_US.UTF-8", "LDFLAGS": "", "PATH": "/Users/sinisaderasevic/.virtualenvs/apollo/bin:/Users/sinisaderasevic/.pyenv/shims:/Users/sinisaderasevic/.pyenv/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "PEX_IGNORE_RCFILES": "true", "PEX_PYTHON_PATH": "/Users/sinisaderasevic/.pyenv/versions/3.9.7/bin:/Users/sinisaderasevic/.virtualenvs/apollo/bin:/Users/sinisaderasevic/.pyenv/shims:/Users/sinisaderasevic/.pyenv/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "PEX_ROOT": ".cache/pex_root"}, working_directory: None, input_files: Digest { hash: Fingerprint<26cf32db81511eab7e4996e09d5a21b162c66212333573648332b89025965ab8>, size_bytes: 158 }, output_files: {}, output_directories: {}, timeout: None, execution_slot_variable: None, description: "Find interpreter for constraints: CPython==3.9.*", level: Debug, append_only_caches: {CacheName("pex_root"): CacheDest(".cache/pex_root")}, jdk_home: None, platform_constraint: None, is_nailgunnable: false, cache_scope: PerRestartSuccessful }
15:46:09.80 [DEBUG] Completed: Find interpreter for constraints: CPython==3.9.*
15:46:09.80 [DEBUG] Completed: Scheduling: Find interpreter for constraints: CPython==3.9.*
15:46:09.80 [DEBUG] Completed: Find Python interpreter for constraints
15:46:09.80 [DEBUG] Completed: pants.backend.python.util_rules.pex.build_pex_component
15:46:09.80 [DEBUG] Completed: pants.backend.python.util_rules.pex.build_pex
15:46:09.80 [DEBUG] Completed: pants.backend.python.lint.black.rules.setup_black
15:46:09.80 [DEBUG] Completed: Format with Black
15:46:09.80 [DEBUG] Completed: `fmt` goal
15:46:09.80 [DEBUG] computed 1 nodes in 0.422410 seconds. there are 1501 total nodes.
15:46:09.80 [ERROR] Exception caught: (pants.engine.internals.scheduler.ExecutionError)
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/bin/local_pants_runner.py", line 236, in _run_inner
    return self._perform_run(goals)
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/bin/local_pants_runner.py", line 175, in _perform_run
    return self._perform_run_body(goals, poll=False)
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/bin/local_pants_runner.py", line 192, in _perform_run_body
    return self.graph_session.run_goal_rules(
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/init/engine_initializer.py", line 135, in run_goal_rules
    exit_code = self.scheduler_session.run_goal_rule(
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/engine/internals/scheduler.py", line 538, in run_goal_rule
    self._raise_on_error([t for _, t in throws])
  File "/Users/sinisaderasevic/.cache/pants/setup/bootstrap-Darwin-arm64/2.7.0_py39/lib/python3.9/site-packages/pants/engine/internals/scheduler.py", line 506, in _raise_on_error
    raise ExecutionError(

Exception message: 1 Exception encountered:

  ProcessExecutionFailure: Process 'Find interpreter for constraints: CPython==3.9.*' failed with exit code 1.
stdout:

stderr:
Traceback (most recent call last):
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 483, in execute
    exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 401, in _wrap_coverage
    return runner(*args)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 432, in _wrap_profiling
    return runner(*args)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 539, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 655, in execute_entry
    return self.execute_pkg_resources(entry_point)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.bootstrap/pex/pex.py", line 687, in execute_pkg_resources
    return runner()
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/bin/pex.py", line 1149, in main
    pex_builder = build_pex(options.requirements, options, cache=ENV.PEX_ROOT)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/bin/pex.py", line 895, in build_pex
    interpreters = list(
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/pex_bootstrapper.py", line 139, in iter_compatible_interpreters
    for interpreter_or_error in _iter_interpreters():
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/pex_bootstrapper.py", line 104, in _iter_interpreters
    for interp in PythonInterpreter.iter_candidates(
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/interpreter.py", line 570, in iter_candidates
    for interpreter in cls._filter(iter_interpreters()):
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/interpreter.py", line 922, in _filter
    for interp in pythons:
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/interpreter.py", line 551, in iter_interpreters
    for candidate in cls._find(
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/jobs.py", line 431, in execute_parallel
    yield spawn_result.spawned_job.await_result()
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/jobs.py", line 220, in await_result
    return result_func(stdout)
  File "/Users/sinisaderasevic/bull/apollo/.cache/pants/named_caches/pex_root/unzipped_pexes/592827b562c4aa09c1e9af6b3ed7b797eef1a4dc/.deps/pex-2.1.44-py2.py3-none-any.whl/pex/interpreter.py", line 667, in create_interpreter
    raise cls.IdentificationError("Could not establish identity of {}.".format(binary))
pex.interpreter.IdentificationError: Could not establish identity of /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/pythonwrapper.app/Contents/MacOS/pythonwrapper.
```

The user's system had /usr/bin/python symlinked to this interpreter `/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/pythonwrapper.app/Contents/MacOS/pythonwrapper`.

The user worked around this error by modifying `interpreter_search_paths` to not include `/usr/bin/python` in the search.

Maybe the interpreter search process should warn about an interpreter failing to identify itself and just invalidate any cached state for that intepreter instead of raising an exception? Especially since the interpreter constraint in this case CPython==3.9.* would not have matched a Python 2.7 interpreter any way.


Discussion:
The exception is raised at https://github.com/pantsbuild/pex/blob/a2eb72e4f627ce0630cd120b5411b2fda7974ce9/pex/interpreter.py#L663-L666 if the candidate interpreter returns no output when asked to run a script to dump the                         from `pex.interpreter.PythonIdentity`.
> Maybe the interpreter search process should warn about an interpreter failing to identify itself and just invalidate any cached state for that intepreter instead of raising an exception?

Yes. Great pains are gone through in the `iter_compatible_interpreters` code path to do exactly that. This is a hole in all that than needs fixing.

> Especially since the interpreter constraint in this case CPython==3.9.* would not have matched a Python 2.7 interpreter any way.

Well, that's not known to this code. You and I scan that path and see `.../Versions/2.7/...` but the code has no heuristic that looks at the containing path to guess partial constraints.

Resolving patch:
diff --git a/pex/interpreter.py b/pex/interpreter.py
--- a/pex/interpreter.py
+++ b/pex/interpreter.py
@@ -728,7 +728,6 @@ def create_interpreter(
 
 
                         encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
-                        sys.stdout.write(encoded_identity)
                         with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
                             if not cache_dir.is_finalized:
                                 with safe_open(
@@ -749,7 +748,7 @@ def create_interpreter(
                 cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
             )
             job = Job(command=cmd, process=process, finalizer=lambda: safe_rmtree(cwd))
-            return SpawnedJob.stdout(job, result_func=create_interpreter)
+            return SpawnedJob.file(job, output_file=cache_file, result_func=create_interpreter)
 
     @classmethod
     def _expand_path(cls, path):
diff --git a/pex/jobs.py b/pex/jobs.py
--- a/pex/jobs.py
+++ b/pex/jobs.py
@@ -8,7 +8,6 @@
 from abc import abstractmethod
 from threading import BoundedSemaphore, Event, Thread
 
-from pex.attrs import str_tuple_from_iterable
 from pex.compatibility import AbstractClass, Queue, cpu_count
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, Generic
@@ -105,6 +104,30 @@ def kill(self):
         finally:
             self._finalize_job()
 
+    def create_error(
+        self,
+        msg,  # type: str
+        stderr=None,  # type: Optional[bytes]
+    ):
+        # type: (...) -> Job.Error
+        """Creates an error with this Job's details.
+
+        :param msg: The message for the error.
+        :param stderr: Any stderr output captured from the job.
+        :return: A job error.
+        """
+        err = None
+        if stderr:
+            err = stderr.decode("utf-8")
+            msg += "\nSTDERR:\n{}".format(err)
+        raise self.Error(
+            pid=self._process.pid,
+            command=self._command,
+            exitcode=self._process.returncode,
+            stderr=err,
+            message=msg,
+        )
+
     def _finalize_job(self):
         if self._finalizer is not None:
             self._finalizer()
@@ -113,20 +136,10 @@ def _finalize_job(self):
     def _check_returncode(self, stderr=None):
         # type: (Optional[bytes]) -> None
         if self._process.returncode != 0:
-            err = None
             msg = "Executing {} failed with {}".format(
                 " ".join(self._command), self._process.returncode
             )
-            if stderr:
-                err = stderr.decode("utf-8")
-                msg += "\nSTDERR:\n{}".format(err)
-            raise self.Error(
-                pid=self._process.pid,
-                command=self._command,
-                exitcode=self._process.returncode,
-                stderr=err,
-                message=msg,
-            )
+            raise self.create_error(msg, stderr=stderr)
 
     def __str__(self):
         # type: () -> str
@@ -245,6 +258,56 @@ def __repr__(self):
 
         return Stdout()
 
+    @classmethod
+    def file(
+        cls,
+        job,  # type: Job
+        output_file,  # type: str
+        result_func,  # type: Callable[[bytes], _T]
+        input=None,  # type: Optional[bytes]
+    ):
+        # type: (...) -> SpawnedJob[_T]
+        """Wait for the job to complete and return a result derived from a file the job creates.
+
+        :param job: The spawned job.
+        :param output_file: The path of the file the job will create.
+        :param result_func: A function taking the byte contents of the file the spawned job
+                            created and returning the desired result.
+        :param input: Optional input stream data to pass to the process as per the
+                      `subprocess.Popen.communicate` API.
+        :return: A spawned job whose result is derived from the contents of a file it creates.
+        """
+
+        def _read_file(stderr=None):
+            # type: (Optional[bytes]) -> bytes
+            try:
+                with open(output_file, "rb") as fp:
+                    return fp.read()
+            except (OSError, IOError) as e:
+                raise job.create_error(
+                    "Expected job to create file {output_file!r} but it did not exist or could not "
+                    "be read: {err}".format(output_file=output_file, err=e),
+                    stderr=stderr,
+                )
+
+        class File(SpawnedJob):
+            def await_result(self):
+                # type: () -> _T
+                _, stderr = job.communicate(input=input)
+                return result_func(_read_file(stderr=stderr))
+
+            def kill(self):
+                # type: () -> None
+                job.kill()
+
+            def __repr__(self):
+                # type: () -> str
+                return "SpawnedJob.file({job!r}, output_file={output_file!r})".format(
+                    job=job, output_file=output_file
+                )
+
+        return File()
+
     def await_result(self):
         # type: () -> _T
         """Waits for the spawned job to complete and returns its result."""


Test patch:
diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py
--- a/tests/test_interpreter.py
+++ b/tests/test_interpreter.py
@@ -4,17 +4,21 @@
 import glob
 import json
 import os
+import re
 import subprocess
+import sys
 from collections import defaultdict
 from contextlib import contextmanager
+from textwrap import dedent
 
 import pytest
 
 from pex import interpreter
-from pex.common import safe_mkdir, safe_mkdtemp, temporary_dir, touch
+from pex.common import chmod_plus_x, safe_mkdir, safe_mkdtemp, temporary_dir, touch
 from pex.compatibility import PY3
 from pex.executor import Executor
 from pex.interpreter import PythonInterpreter
+from pex.jobs import Job
 from pex.pyenv import Pyenv
 from pex.testing import (
     PY27,
@@ -417,3 +421,66 @@ def test_identify_cwd_isolation_issues_1231(tmpdir):
     assert 1 == len(interp_info_files)
     with open(interp_info_files.pop()) as fp:
         assert interp.binary == json.load(fp)["binary"]
+
+
+@pytest.fixture(scope="module")
+def macos_monterey_interpeter(tmpdir_factory):
+    # type: (Any) -> str
+    pythonwrapper = os.path.join(str(tmpdir_factory.mktemp("bin")), "pythonwrapper")
+    with open(pythonwrapper, "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                #!/usr/bin/env bash
+                echo >&2 "pythonwrapper[3129:20922] \\
+                pythonwrapper is not supposed to be executed directly. Exiting."
+                exit 0
+                """
+            )
+        )
+    chmod_plus_x(pythonwrapper)
+
+    python = os.path.join(str(tmpdir_factory.mktemp("bin")), "python")
+    os.symlink(pythonwrapper, python)
+    return python
+
+
+def test_issue_1494_job_error_not_identification_error(
+    macos_monterey_interpeter,  # type: str
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    with ENV.patch(PEX_ROOT=pex_root):
+        spawned_job = PythonInterpreter._spawn_from_binary(macos_monterey_interpeter)
+        with pytest.raises(Job.Error) as exc_info:
+            spawned_job.await_result()
+        exc_info.match(
+            r"^Expected job to create file '{}/interpreters/[0-9a-f/]+/INTERP-INFO' "
+            r"but it did not exist or could not be read: ".format(re.escape(pex_root))
+        )
+        exc_info.match(
+            r"\n"
+            r"STDERR:\n"
+            r"pythonwrapper\[3129:20922\] "
+            r"pythonwrapper is not supposed to be executed directly. Exiting.\n$"
+        )
+
+
+def test_issue_1494_iter(macos_monterey_interpeter):
+    # type: (str) -> None
+    assert [PythonInterpreter.get()] == list(
+        PythonInterpreter.iter(paths=[sys.executable, macos_monterey_interpeter])
+    )
+
+
+def test_issue_1494_iter_candidates(macos_monterey_interpeter):
+    # type: (str) -> None
+    assert [
+        PythonInterpreter.get(),
+        (
+            macos_monterey_interpeter,
+            "pythonwrapper[3129:20922] pythonwrapper is not supposed to be executed directly. "
+            "Exiting.",
+        ),
+    ] == list(PythonInterpreter.iter_candidates(paths=[sys.executable, macos_monterey_interpeter]))