Repository: pex-tool/pex

Issue:
Pex discards the current interpreter's PATH entry when it is a directory entry.
This means no other interpreters in the current interpreter's directory (sibling interpreters) can be found. This is an egregious problem since interpreters are commonly installed to /usr/bin; i.e.: /usr/bin/python2.7, /usr/bin/python3.6, ... etc will all be siblings and if a PEX is run with any one of them, the others won't be discoverable.

N.B.: Pex only encounters this condition when `PEX_PYTHON_PATH` / `--python-path` are used, populating the `path` function parameter below:
https://github.com/pantsbuild/pex/blob/93b0442c9574a0bba018249d842dc2ac29dc1e27/pex/pex_bootstrapper.py#L40-L45

And the bug is here where we remove both the interpreter binary path (which is ok) and the interpreter binary directory (which is the bug):
https://github.com/pantsbuild/pex/blob/93b0442c9574a0bba018249d842dc2ac29dc1e27/pex/pex_bootstrapper.py#L83-L89


Discussion:

Resolving patch:
diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py
--- a/pex/pex_bootstrapper.py
+++ b/pex/pex_bootstrapper.py
@@ -41,6 +41,7 @@ def iter_compatible_interpreters(
     path=None,  # type: Optional[str]
     valid_basenames=None,  # type: Optional[Iterable[str]]
     interpreter_constraints=None,  # type: Optional[Iterable[str]]
+    preferred_interpreter=None,  # type: Optional[PythonInterpreter]
 ):
     # type: (...) -> Iterator[PythonInterpreter]
     """Find all compatible interpreters on the system within the supplied constraints.
@@ -51,6 +52,8 @@ def iter_compatible_interpreters(
                             pypy, etc.).
     :param interpreter_constraints: Interpreter type and version constraint strings as described in
                                     `--interpreter-constraint`.
+    :param preferred_interpreter: For testing - an interpreter to prefer amongst all others.
+                                  Defaults to the current running interpreter.
 
     Interpreters are searched for in `path` if specified and $PATH if not.
 
@@ -77,7 +80,7 @@ def _iter_interpreters():
         )
 
         # Prefer the current interpreter, if valid.
-        current_interpreter = PythonInterpreter.get()
+        current_interpreter = preferred_interpreter or PythonInterpreter.get()
         if not _valid_path or _valid_path(current_interpreter.binary):
             if normalized_paths:
                 candidate_paths = frozenset(
@@ -85,8 +88,10 @@ def _iter_interpreters():
                 )
                 candidate_paths_in_path = candidate_paths.intersection(normalized_paths)
                 if candidate_paths_in_path:
-                    for p in candidate_paths_in_path:
-                        normalized_paths.remove(p)
+                    # In case the full path of the current interpreter binary was in the
+                    # `normalized_paths` we're searching, remove it to prevent identifying it again
+                    # just to then skip it as `seen`.
+                    normalized_paths.discard(current_interpreter.binary)
                     seen.add(current_interpreter)
                     yield current_interpreter
             else:


Test patch:
diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py
--- a/tests/test_pex_bootstrapper.py
+++ b/tests/test_pex_bootstrapper.py
@@ -2,11 +2,13 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import shutil
 import sys
 from textwrap import dedent
 
 import pytest
 
+from pex.common import temporary_dir
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError
 from pex.pex_bootstrapper import iter_compatible_interpreters
@@ -22,14 +24,20 @@ def basenames(*paths):
     return [os.path.basename(p) for p in paths]
 
 
-def find_interpreters(path, valid_basenames=None, constraints=None):
-    # type: (Iterable[str], Optional[Iterable[str]], Optional[Iterable[str]]) -> List[AnyStr]
+def find_interpreters(
+    path,  # type: Iterable[str]
+    valid_basenames=None,  # type: Optional[Iterable[str]]
+    constraints=None,  # type: Optional[Iterable[str]]
+    preferred_interpreter=None,  # type: Optional[PythonInterpreter]
+):
+    # type: (...) -> List[AnyStr]
     return [
         interp.binary
         for interp in iter_compatible_interpreters(
             path=os.pathsep.join(path),
             valid_basenames=valid_basenames,
             interpreter_constraints=constraints,
+            preferred_interpreter=preferred_interpreter,
         )
     ]
 
@@ -157,3 +165,19 @@ def test_find_compatible_interpreters_bias_current():
     py36 = ensure_python_interpreter(PY36)
     assert [os.path.realpath(sys.executable), py36] == find_interpreters([py36, sys.executable])
     assert [os.path.realpath(sys.executable), py36] == find_interpreters([sys.executable, py36])
+
+
+def test_find_compatible_interpreters_siblings_of_current_issues_1109():
+    py27 = ensure_python_interpreter(PY27)
+    py36 = ensure_python_interpreter(PY36)
+
+    with temporary_dir() as path_entry:
+        python27 = os.path.join(path_entry, "python2.7")
+        shutil.copy(py27, python27)
+
+        python36 = os.path.join(path_entry, "python3.6")
+        shutil.copy(py36, python36)
+
+        assert [os.path.realpath(p) for p in (python36, python27)] == find_interpreters(
+            path=[path_entry], preferred_interpreter=PythonInterpreter.from_binary(python36)
+        )