Repository: pex-tool/pex

Issue:
Kill off PEX_PYTHON
The pex runtime currently supports a configuration for a PEX_PYTHON variable in a pexrc file. This variable allows a user to define a Python Interpreter to override the default interpreter when executing the pex. 

This has been replaced by a PEX_PYTHON_PATH variable, that allows the user to specify a colon-separated list of paths to Python interpreters with the aim of being able support interpreter selection against a list of interpreter override candidates.

The goal of this issue is to completely kill off PEX_PYTHON support in the pex codebase in favor of the new PEX_PYTHON_PATH pexrc variable. 


Discussion:
See https://github.com/pantsbuild/pex/issues/53 - isn't there sometimes a use case for "please use exactly this python"? Or is there an alternative way to get that functionality?
The idea here is that PEX_PYTHON_PATH is a graduated variant of PEX_PYTHON that says "Use user-supplied constraints and exec with the minimum compatible interpreter from this list of Python interpreters". As of right now, PEX_PYTHON_PATH with only one interpreter will effectively accomplish the same thing as PEX_PYTHON.
Although what @CMLivingston says is True, this is ingrained enough in scripts and a useful enough distinction to meet reasonable user expectations in practice (PATH variables don't normally support the full path to a binary, just to the parent dir), that I think its worth keeping.

Resolving patch:
diff --git a/pex/interpreter.py b/pex/interpreter.py
--- a/pex/interpreter.py
+++ b/pex/interpreter.py
@@ -43,7 +43,6 @@
         Mapping,
         MutableMapping,
         Optional,
-        Sequence,
         Text,
         Tuple,
         Union,
@@ -496,7 +495,7 @@ class InterpreterNotFound(Error):
 
     @staticmethod
     def latest_release_of_min_compatible_version(interps):
-        # type: (Sequence[PythonInterpreter]) -> PythonInterpreter
+        # type: (Iterable[PythonInterpreter]) -> PythonInterpreter
         """Find the minimum major version, but use the most recent micro version within that minor
         version.
 
diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py
--- a/pex/pex_bootstrapper.py
+++ b/pex/pex_bootstrapper.py
@@ -160,25 +160,30 @@ def _select_path_interpreter(
     path=None,  # type: Optional[str]
     valid_basenames=None,  # type: Optional[Tuple[str, ...]]
     interpreter_constraints=None,  # type: Optional[Iterable[str]]
+    preferred_interpreter=None,  # type: Optional[PythonInterpreter]
 ):
     # type: (...) -> Optional[PythonInterpreter]
     candidate_interpreters_iter = iter_compatible_interpreters(
         path=path,
         valid_basenames=valid_basenames,
         interpreter_constraints=interpreter_constraints,
+        preferred_interpreter=preferred_interpreter,
     )
     current_interpreter = PythonInterpreter.get()  # type: PythonInterpreter
-    candidate_interpreters = []
+    preferred_interpreter = preferred_interpreter or current_interpreter
+    candidate_interpreters = OrderedSet()  # type: OrderedSet[PythonInterpreter]
     for interpreter in candidate_interpreters_iter:
-        if current_interpreter == interpreter:
-            # Always prefer continuing with the current interpreter when possible to avoid re-exec
-            # overhead.
-            return current_interpreter
+        if preferred_interpreter == interpreter:
+            # Always respect the preferred interpreter if it doesn't violate other constraints.
+            return preferred_interpreter
         else:
-            candidate_interpreters.append(interpreter)
+            candidate_interpreters.add(interpreter)
     if not candidate_interpreters:
         return None
-
+    if current_interpreter in candidate_interpreters:
+        # Always prefer continuing with the current interpreter when possible to avoid re-exec
+        # overhead.
+        return current_interpreter
     # TODO: Allow the selection strategy to be parameterized:
     #   https://github.com/pantsbuild/pex/issues/430
     return PythonInterpreter.latest_release_of_min_compatible_version(candidate_interpreters)
@@ -186,13 +191,37 @@ def _select_path_interpreter(
 
 def find_compatible_interpreter(interpreter_constraints=None):
     # type: (Optional[Iterable[str]]) -> PythonInterpreter
+
+    def gather_constraints():
+        # type: () -> Iterable[str]
+        constraints = []
+        if ENV.PEX_PYTHON:
+            constraints.append("PEX_PYTHON={}".format(ENV.PEX_PYTHON))
+        if ENV.PEX_PYTHON_PATH:
+            constraints.append("PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH))
+        if interpreter_constraints:
+            constraints.append("Version matches {}".format(" or ".join(interpreter_constraints)))
+        return constraints
+
+    preferred_interpreter = None  # type: Optional[PythonInterpreter]
+    if ENV.PEX_PYTHON and os.path.isabs(ENV.PEX_PYTHON):
+        try:
+            preferred_interpreter = PythonInterpreter.from_binary(ENV.PEX_PYTHON)
+        except PythonInterpreter.Error as e:
+            raise UnsatisfiableInterpreterConstraintsError(
+                constraints=gather_constraints(),
+                candidates=[],
+                failures=[(ENV.PEX_PYTHON, str(e))],
+                preamble=(
+                    "The specified PEX_PYTHON={pex_python} could not be identified as a "
+                    "valid Python interpreter.".format(pex_python=ENV.PEX_PYTHON)
+                ),
+            )
+
     current_interpreter = PythonInterpreter.get()
     target = current_interpreter  # type: Optional[PythonInterpreter]
     with TRACER.timed("Selecting runtime interpreter", V=3):
         if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH:
-            # preserve PEX_PYTHON re-exec for backwards compatibility
-            # TODO: Kill this off completely in favor of PEX_PYTHON_PATH
-            # https://github.com/pantsbuild/pex/issues/431
             TRACER.log(
                 "Using PEX_PYTHON={} constrained by {}".format(
                     ENV.PEX_PYTHON, interpreter_constraints
@@ -204,6 +233,7 @@ def find_compatible_interpreter(interpreter_constraints=None):
                     target = _select_path_interpreter(
                         path=ENV.PEX_PYTHON,
                         interpreter_constraints=interpreter_constraints,
+                        preferred_interpreter=preferred_interpreter,
                     )
                 else:
                     target = _select_path_interpreter(
@@ -228,7 +258,9 @@ def find_compatible_interpreter(interpreter_constraints=None):
             )
             try:
                 target = _select_path_interpreter(
-                    path=ENV.PEX_PYTHON_PATH, interpreter_constraints=interpreter_constraints
+                    path=ENV.PEX_PYTHON_PATH,
+                    interpreter_constraints=interpreter_constraints,
+                    preferred_interpreter=preferred_interpreter,
                 )
             except UnsatisfiableInterpreterConstraintsError as e:
                 raise e.with_preamble(
@@ -237,24 +269,28 @@ def find_compatible_interpreter(interpreter_constraints=None):
                     )
                 )
 
+        if preferred_interpreter and target != preferred_interpreter:
+            candidates = [preferred_interpreter, target] if target else [preferred_interpreter]
+            raise UnsatisfiableInterpreterConstraintsError(
+                constraints=gather_constraints(),
+                candidates=candidates,
+                failures=[],
+                preamble=(
+                    "The specified PEX_PYTHON={pex_python} did not meet other "
+                    "constraints.".format(pex_python=ENV.PEX_PYTHON)
+                ),
+            )
+
         if target is None:
             # N.B.: This can only happen when PEX_PYTHON_PATH is set and interpreter_constraints
             # is empty, but we handle all constraints generally for sanity sake.
-            constraints = []
-            if ENV.PEX_PYTHON:
-                constraints.append("PEX_PYTHON={}".format(ENV.PEX_PYTHON))
-            if ENV.PEX_PYTHON_PATH:
-                constraints.append("PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH))
-            if interpreter_constraints:
-                constraints.append(
-                    "Version matches {}".format(" or ".join(interpreter_constraints))
-                )
             raise UnsatisfiableInterpreterConstraintsError(
-                constraints=constraints,
+                constraints=gather_constraints(),
                 candidates=[current_interpreter],
                 failures=[],
                 preamble="Could not find a compatible interpreter.",
             )
+
         return target
 
 


Test patch:
diff --git a/tests/integration/test_interpreter_selection.py b/tests/integration/test_interpreter_selection.py
--- a/tests/integration/test_interpreter_selection.py
+++ b/tests/integration/test_interpreter_selection.py
@@ -17,6 +17,10 @@
     run_simple_pex,
 )
 from pex.third_party import pkg_resources
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
 
 
 def test_interpreter_constraints_to_pex_info_py2():
@@ -124,70 +128,100 @@ def test_interpreter_resolution_with_pex_python_path():
             assert str(pex_python_path.split(":")[0]).encode() in stdout
 
 
-def test_interpreter_constraints_honored_without_ppp_or_pp():
-    # type: () -> None
+def test_interpreter_constraints_honored_without_ppp_or_pp(tmpdir):
+    # type: (Any) -> None
     # Create a pex with interpreter constraints, but for not the default interpreter in the path.
-    with temporary_dir() as td:
-        py38_path = ensure_python_interpreter(PY38)
-        py37_path = ensure_python_interpreter(PY37)
-
-        pex_out_path = os.path.join(td, "pex.pex")
-        env = make_env(
-            PEX_IGNORE_RCFILES="1",
-            PATH=os.pathsep.join(
-                [
-                    os.path.dirname(py37_path),
-                    os.path.dirname(py38_path),
-                ]
-            ),
-        )
-        res = run_pex_command(
-            ["--disable-cache", "--interpreter-constraint===%s" % PY38, "-o", pex_out_path], env=env
-        )
-        res.assert_success()
-
-        # We want to try to run that pex with no environment variables set
-        stdin_payload = b"import sys; print(sys.executable); sys.exit(0)"
-
-        stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env)
-        assert rc == 0
-
-        # If the constraints are honored, it will have run python3.8 and not python3.7
-        # Without constraints, we would expect it to use python3.7 as it is the minimum interpreter
-        # in the PATH.
-        assert str(py38_path).encode() in stdout
 
+    py38_path = ensure_python_interpreter(PY38)
+    py37_path = ensure_python_interpreter(PY37)
 
-def test_interpreter_resolution_pex_python_path_precedence_over_pex_python():
-    # type: () -> None
-    with temporary_dir() as td:
-        pexrc_path = os.path.join(td, ".pexrc")
-        with open(pexrc_path, "w") as pexrc:
-            # set both PPP and PP
-            pex_python_path = ":".join(
-                [ensure_python_interpreter(PY27), ensure_python_interpreter(PY37)]
-            )
-            pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path)
-            pex_python = "/path/to/some/python"
-            pexrc.write("PEX_PYTHON=%s" % pex_python)
-
-        pex_out_path = os.path.join(td, "pex.pex")
-        res = run_pex_command(
+    pex_out_path = os.path.join(str(tmpdir), "pex.pex")
+    env = make_env(
+        PEX_IGNORE_RCFILES="1",
+        PATH=os.pathsep.join(
             [
-                "--disable-cache",
-                "--rcfile=%s" % pexrc_path,
-                "--interpreter-constraint=>3,<3.8",
-                "-o",
-                pex_out_path,
+                os.path.dirname(py37_path),
+                os.path.dirname(py38_path),
             ]
+        ),
+    )
+    res = run_pex_command(
+        ["--disable-cache", "--interpreter-constraint===%s" % PY38, "-o", pex_out_path], env=env
+    )
+    res.assert_success()
+
+    # We want to try to run that pex with no environment variables set
+    stdin_payload = b"import sys; print(sys.executable); sys.exit(0)"
+
+    stdout, rc = run_simple_pex(
+        pex_out_path,
+        args=["-c", "import sys; print('.'.join(map(str, sys.version_info[:2])))"],
+        env=env,
+    )
+    assert rc == 0
+
+    # If the constraints are honored, it will have run python3.8 and not python3.7
+    # Without constraints, we would expect it to use python3.7 as it is the minimum interpreter
+    # in the PATH.
+    assert b"3.8\n" == stdout
+
+
+def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(tmpdir):
+    # type: (Any) -> None
+
+    pexrc_path = os.path.join(str(tmpdir), ".pexrc")
+    ppp = ":".join(os.path.dirname(ensure_python_interpreter(py)) for py in (PY27, PY37))
+    with open(pexrc_path, "w") as pexrc:
+        # set both PPP and PP
+        pexrc.write(
+            dedent(
+                """\
+                PEX_PYTHON_PATH={ppp}
+                PEX_PYTHON={pp}
+                """.format(
+                    ppp=ppp, pp=ensure_python_interpreter(PY38)
+                )
+            )
         )
-        res.assert_success()
 
-        stdin_payload = b"import sys; print(sys.executable); sys.exit(0)"
-        stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
-        assert rc == 0
-        correct_interpreter_path = pex_python_path.split(":")[1].encode()
-        assert correct_interpreter_path in stdout
+    pex_out_path = os.path.join(str(tmpdir), "pex.pex")
+    run_pex_command(
+        [
+            "--disable-cache",
+            "--rcfile",
+            pexrc_path,
+            "--interpreter-constraint=>3,<3.8",
+            "-o",
+            pex_out_path,
+        ]
+    ).assert_success()
+
+    print_python_version_command = [
+        "-c",
+        "import sys; print('.'.join(map(str, sys.version_info[:2])))",
+    ]
+
+    _, rc = run_simple_pex(pex_out_path, print_python_version_command)
+    assert rc != 0, (
+        "PEX_PYTHON_PATH should trump PEX_PYTHON when PEX_PYTHON is an explicit path to a Python "
+        "interpreter that is not on the PEX_PYTHON_PATH and this should lead to failure to select "
+        "an interpreter"
+    )
+
+    with open(pexrc_path, "w") as pexrc:
+        pexrc.write(
+            dedent(
+                """\
+                PEX_PYTHON_PATH={ppp}
+                PEX_PYTHON=python
+                """.format(
+                    ppp=ppp
+                )
+            )
+        )
+    stdout, rc = run_simple_pex(pex_out_path, print_python_version_command)
+    assert rc == 0
+    assert b"3.7\n" == stdout
 
 
 def test_plain_pex_exec_no_ppp_no_pp_no_constraints():
@@ -229,29 +263,33 @@ def test_pex_exec_with_pex_python_path_only():
         assert str(pex_python_path.split(":")[0]).encode() in stdout
 
 
-def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints():
-    # type: () -> None
-    with temporary_dir() as td:
-        pexrc_path = os.path.join(td, ".pexrc")
-        with open(pexrc_path, "w") as pexrc:
-            # set both PPP and PP
-            pex_python_path = ":".join(
-                [ensure_python_interpreter(PY27), ensure_python_interpreter(PY38)]
+def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(tmpdir):
+    # type: (Any) -> None
+    pexrc_path = os.path.join(str(tmpdir), ".pexrc")
+    with open(pexrc_path, "w") as pexrc:
+        # set both PPP and PP
+        pexrc.write(
+            dedent(
+                """\
+                PEX_PYTHON_PATH={}
+                PEX_PYTHON=python
+                """.format(
+                    ":".join(os.path.dirname(ensure_python_interpreter(py)) for py in (PY38, PY27))
+                )
             )
-            pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path)
-            pex_python = "/path/to/some/python"
-            pexrc.write("PEX_PYTHON=%s" % pex_python)
-
-        pex_out_path = os.path.join(td, "pex.pex")
-        res = run_pex_command(["--disable-cache", "--rcfile=%s" % pexrc_path, "-o", pex_out_path])
-        res.assert_success()
+        )
 
-        # test that pex bootstrapper selects lowest version interpreter
-        # in pex python path (python2.7)
-        stdin_payload = b"import sys; print(sys.executable); sys.exit(0)"
-        stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
-        assert rc == 0
-        assert str(pex_python_path.split(":")[0]).encode() in stdout
+    pex_out_path = os.path.join(str(tmpdir), "pex.pex")
+    res = run_pex_command(["--disable-cache", "--rcfile", pexrc_path, "-o", pex_out_path])
+    res.assert_success()
+
+    # test that pex bootstrapper selects lowest version interpreter
+    # in pex python path (python2.7)
+    stdout, rc = run_simple_pex(
+        pex_out_path, args=["-c", "import sys; print('.'.join(map(str, sys.version_info[:2])))"]
+    )
+    assert rc == 0
+    assert b"2.7\n" == stdout
 
 
 def test_pex_python():
diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py
--- a/tests/test_pex_bootstrapper.py
+++ b/tests/test_pex_bootstrapper.py
@@ -13,10 +13,15 @@
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError
 from pex.pex import PEX
-from pex.pex_bootstrapper import ensure_venv, iter_compatible_interpreters
+from pex.pex_bootstrapper import (
+    ensure_venv,
+    find_compatible_interpreter,
+    iter_compatible_interpreters,
+)
 from pex.pex_builder import PEXBuilder
 from pex.testing import PY27, PY37, PY38, ensure_python_interpreter
 from pex.typing import TYPE_CHECKING
+from pex.variables import ENV
 
 if TYPE_CHECKING:
     from typing import Any, Iterable, List, Optional
@@ -234,3 +239,77 @@ def test_ensure_venv_activate_issues_1276(tmpdir):
             ),
         ]
     )
+
+
+def test_pp_invalid():
+    # type: () -> None
+    with ENV.patch(PEX_PYTHON="/invalid/abs/python"):
+        with pytest.raises(
+            UnsatisfiableInterpreterConstraintsError,
+            match=(
+                r"The specified PEX_PYTHON=/invalid/abs/python could not be identified as a "
+                r"valid Python interpreter."
+            ),
+        ):
+            find_compatible_interpreter()
+
+
+def test_pp_exact():
+    # type: () -> None
+    py38 = ensure_python_interpreter(PY38)
+    with ENV.patch(PEX_PYTHON=py38):
+        assert PythonInterpreter.from_binary(py38) == find_compatible_interpreter()
+
+
+def test_pp_exact_on_ppp():
+    # type: () -> None
+
+    py27 = ensure_python_interpreter(PY27)
+    py37 = ensure_python_interpreter(PY37)
+    py38 = ensure_python_interpreter(PY38)
+
+    with ENV.patch(
+        PEX_PYTHON=py38, PEX_PYTHON_PATH=":".join(os.path.dirname(py) for py in (py27, py37, py38))
+    ):
+        assert PythonInterpreter.from_binary(py38) == find_compatible_interpreter()
+
+
+def test_pp_exact_satisfies_constraints():
+    # type: () -> None
+
+    py38 = ensure_python_interpreter(PY38)
+
+    with ENV.patch(PEX_PYTHON=py38):
+        assert PythonInterpreter.from_binary(py38) == find_compatible_interpreter(
+            interpreter_constraints=[">=3.7"]
+        )
+
+
+def test_pp_exact_does_not_satisfy_constraints():
+    # type: () -> None
+
+    py38 = ensure_python_interpreter(PY38)
+
+    with ENV.patch(PEX_PYTHON=py38):
+        with pytest.raises(
+            UnsatisfiableInterpreterConstraintsError,
+            match=r"Failed to find a compatible PEX_PYTHON={pp}.".format(pp=py38),
+        ):
+            find_compatible_interpreter(interpreter_constraints=["<=3.7"])
+
+
+def test_pp_exact_not_on_ppp():
+    # type: () -> None
+
+    py27 = ensure_python_interpreter(PY27)
+    py37 = ensure_python_interpreter(PY37)
+    py38 = ensure_python_interpreter(PY38)
+
+    with ENV.patch(
+        PEX_PYTHON=py38, PEX_PYTHON_PATH=":".join(os.path.dirname(py) for py in (py27, py37))
+    ):
+        with pytest.raises(
+            UnsatisfiableInterpreterConstraintsError,
+            match=r"The specified PEX_PYTHON={pp} did not meet other constraints.".format(pp=py38),
+        ):
+            find_compatible_interpreter()