Repository: pantsbuild/pants

Issue:
PythonSetup interpreter_search_path claims file paths work, but they do not for the bootstrap interpreter search.
For an interpreter_search_path that contains just `/usr/bin/python3`, Looks like so:
```console
08:28:54.05 [WARN] Completed: Find Python interpreter to bootstrap PEX - No bootstrap Python executable could be found from the option `interpreter_search_paths` in the `[python-setup]` scope. Will attempt to run PEXes directly.
```

That contradicts our docs here: https://www.pantsbuild.org/docs/reference-python-setup#section-interpreter-search-paths where we say "You can specify absolute paths to interpreter binaries and/or to directories containing interpreter binaries.". That's only true for some uses, its not true for the use in finding a bootstrap Python. That use only works for tradition PATH (directory) entries.


Discussion:
It seems there are 3 remedies in the bootstrap search context. We'd need to test each entry in the (expanded) interpreter_search_paths and if the entry is a file:
1. Just use it.
2. Warn its not being used.
3. Use its parent dir as a PATH entry for the search

If 1 is implemented, that would make the existing doc not-a-lie and comport with how Pex handles this and generally meet expectations.

Resolving patch:
diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py
--- a/src/python/pants/engine/process.py
+++ b/src/python/pants/engine/process.py
@@ -380,6 +380,9 @@ class SearchPath(DeduplicatedCollection[str]):
 class BinaryPathRequest:
     """Request to find a binary of a given name.
 
+    If `check_file_entries` is `True` a BinaryPathRequest will consider any entries in the
+    `search_path` that are file paths in addition to traditional directory paths.
+
     If a `test` is specified all binaries that are found will be executed with the test args and
     only those binaries whose test executions exit with return code 0 will be retained.
     Additionally, if test execution includes stdout content, that will be used to fingerprint the
@@ -391,6 +394,7 @@ class BinaryPathRequest:
 
     search_path: SearchPath
     binary_name: str
+    check_file_entries: bool
     test: BinaryPathTest | None
 
     def __init__(
@@ -398,10 +402,12 @@ def __init__(
         *,
         search_path: Iterable[str],
         binary_name: str,
+        check_file_entries: bool = False,
         test: BinaryPathTest | None = None,
     ) -> None:
         self.search_path = SearchPath(search_path)
         self.binary_name = binary_name
+        self.check_file_entries = check_file_entries
         self.test = test
 
 
@@ -529,39 +535,50 @@ async def find_binary(request: BinaryPathRequest) -> BinaryPaths:
         bash = await Get(BashBinary, BashBinaryRequest())
         shebang = f"#!{bash.path}"
 
-    # Some subtle notes with this script:
-    #
-    #  - The backslash after the `"""` ensures that the shebang is at the start of the script file.
-    #       Many OSs will not see the shebang if there is intervening whitespace.
-    #  - We run the script with `ProcessResult` instead of `FallibleProcessResult` so that we
-    #      can catch bugs in the script itself, given an earlier silent failure.
-    #  - We do not use `set -e` like normal because it causes the line
-    #      `command which -a <bin> || true` to fail the script when using Bash 3, which macOS
-    #      uses by default.
-    #  - We set `ProcessCacheScope.PER_RESTART_SUCCESSFUL` to force re-run since any binary found
-    #      on the host system today could be gone tomorrow. Ideally we'd only do this for local
-    #      processes since all known remoting configurations include a static container image as
-    #      part of their cache key which automatically avoids this problem. See #10769 for a
-    #      solution that is less of a tradeoff.
     script_path = "./find_binary.sh"
-    script_content = dedent(
+    script_header = dedent(
         f"""\
         {shebang}
 
-        set -uox pipefail
+        set -euox pipefail
 
-        if command -v which > /dev/null; then
-            command which -a $1 || true
-        else
-            command -v $1 || true
-        fi
+        CHECK_FILE_ENTRIES={'1' if request.check_file_entries else ''}
         """
     )
+    script_body = dedent(
+        """\
+        for path in ${PATH//:/ }; do
+            if [[ -d "${path}" ]]; then
+                # Handle traditional directory PATH element.
+                maybe_exe="${path}/$1"
+            elif [[ -n "${CHECK_FILE_ENTRIES}" ]]; then
+                # Handle PATH elements that are filenames to allow for precise selection.
+                maybe_exe="${path}"
+            else
+                maybe_exe=
+            fi
+            if [[ "$1" == "${maybe_exe##*/}" && -f "${maybe_exe}" && -x "${maybe_exe}" ]]
+            then
+                echo "${maybe_exe}"
+            fi
+        done
+        """
+    )
+    script_content = script_header + script_body
     script_digest = await Get(
         Digest,
         CreateDigest([FileContent(script_path, script_content.encode(), is_executable=True)]),
     )
 
+    # Some subtle notes about executing this script:
+    #
+    #  - We run the script with `ProcessResult` instead of `FallibleProcessResult` so that we
+    #      can catch bugs in the script itself, given an earlier silent failure.
+    #  - We set `ProcessCacheScope.PER_RESTART_SUCCESSFUL` to force re-run since any binary found
+    #      on the host system today could be gone tomorrow. Ideally we'd only do this for local
+    #      processes since all known remoting configurations include a static container image as
+    #      part of their cache key which automatically avoids this problem. See #10769 for a
+    #      solution that is less of a tradeoff.
     search_path = create_path_env_var(request.search_path)
     result = await Get(
         ProcessResult,
diff --git a/src/python/pants/python/binaries.py b/src/python/pants/python/binaries.py
--- a/src/python/pants/python/binaries.py
+++ b/src/python/pants/python/binaries.py
@@ -382,6 +382,7 @@ async def find_python(python_bootstrap: PythonBootstrap) -> PythonBinary:
             BinaryPathRequest(
                 search_path=interpreter_search_paths,
                 binary_name=binary_name,
+                check_file_entries=True,
                 test=BinaryPathTest(
                     args=[
                         "-c",


Test patch:
diff --git a/src/python/pants/engine/process_test.py b/src/python/pants/engine/process_test.py
--- a/src/python/pants/engine/process_test.py
+++ b/src/python/pants/engine/process_test.py
@@ -3,8 +3,7 @@
 
 from __future__ import annotations
 
-import os
-import shutil
+from pathlib import Path
 
 import pytest
 
@@ -27,8 +26,7 @@
     ProcessResult,
 )
 from pants.testutil.rule_runner import QueryRule, RuleRunner
-from pants.util.contextutil import environment_as, temporary_dir
-from pants.util.dirutil import safe_mkdir, touch
+from pants.util.contextutil import environment_as
 
 
 def new_rule_runner() -> RuleRunner:
@@ -263,39 +261,87 @@ def test_interactive_process_cannot_have_input_files_and_workspace() -> None:
         InteractiveProcess(argv=["/bin/echo"], input_digest=mock_digest, run_in_workspace=True)
 
 
-def which_binary_path() -> str:
-    path = shutil.which("which")
-    if not path:
-        raise OSError("`which` not discoverable on your $PATH.")
-    return path
+def test_find_binary_non_existent(rule_runner: RuleRunner, tmp_path: Path) -> None:
+    binary_paths = rule_runner.request(
+        BinaryPaths, [BinaryPathRequest(binary_name="nonexistent-bin", search_path=[str(tmp_path)])]
+    )
+    assert binary_paths.first_path is None
 
 
-@pytest.mark.parametrize("which_path", (None, which_binary_path()))
-def test_find_binary_non_existent(rule_runner: RuleRunner, which_path: str | None) -> None:
-    with temporary_dir() as tmpdir:
-        if which_path:
-            os.symlink(which_path, os.path.join(tmpdir, "which"))
-        binary_paths = rule_runner.request(
-            BinaryPaths, [BinaryPathRequest(binary_name="nonexistent-bin", search_path=[tmpdir])]
-        )
-        assert binary_paths.first_path is None
+class MyBin:
+    binary_name = "mybin"
 
+    @classmethod
+    def create(cls, directory: Path) -> Path:
+        directory.mkdir(parents=True, exist_ok=True)
+        exe = directory / cls.binary_name
+        exe.touch(mode=0o755)
+        return exe
 
-def test_find_binary_on_path_without_bash(rule_runner: RuleRunner) -> None:
+
+def test_find_binary_on_path_without_bash(rule_runner: RuleRunner, tmp_path: Path) -> None:
     # Test that locating a binary on a PATH which does not include bash works (by recursing to
     # locate bash first).
-    binary_name = "mybin"
-    binary_dir = "bin"
-    with temporary_dir() as tmpdir:
-        binary_dir_abs = os.path.join(tmpdir, binary_dir)
-        binary_path_abs = os.path.join(binary_dir_abs, binary_name)
-        safe_mkdir(binary_dir_abs)
-        touch(binary_path_abs)
-
-        search_path = [binary_dir_abs]
-        binary_paths = rule_runner.request(
-            BinaryPaths, [BinaryPathRequest(binary_name=binary_name, search_path=search_path)]
-        )
-        assert os.path.exists(os.path.join(binary_dir_abs, binary_name))
-        assert binary_paths.first_path is not None
-        assert binary_paths.first_path.path == binary_path_abs
+    binary_dir_abs = tmp_path / "bin"
+    binary_path_abs = MyBin.create(binary_dir_abs)
+
+    binary_paths = rule_runner.request(
+        BinaryPaths,
+        [BinaryPathRequest(binary_name=MyBin.binary_name, search_path=[str(binary_dir_abs)])],
+    )
+    assert binary_paths.first_path is not None
+    assert binary_paths.first_path.path == str(binary_path_abs)
+
+
+def test_find_binary_file_path(rule_runner: RuleRunner, tmp_path: Path) -> None:
+    binary_path_abs = MyBin.create(tmp_path)
+
+    binary_paths = rule_runner.request(
+        BinaryPaths,
+        [
+            BinaryPathRequest(
+                binary_name=MyBin.binary_name,
+                search_path=[str(binary_path_abs)],
+            )
+        ],
+    )
+    assert binary_paths.first_path is None, "By default, PATH file entries should not be checked."
+
+    binary_paths = rule_runner.request(
+        BinaryPaths,
+        [
+            BinaryPathRequest(
+                binary_name=MyBin.binary_name,
+                search_path=[str(binary_path_abs)],
+                check_file_entries=True,
+            )
+        ],
+    )
+    assert binary_paths.first_path is not None
+    assert binary_paths.first_path.path == str(binary_path_abs)
+
+
+def test_find_binary_respects_search_path_order(rule_runner: RuleRunner, tmp_path: Path) -> None:
+    binary_path_abs1 = MyBin.create(tmp_path / "bin1")
+    binary_path_abs2 = MyBin.create(tmp_path / "bin2")
+    binary_path_abs3 = MyBin.create(tmp_path / "bin3")
+
+    binary_paths = rule_runner.request(
+        BinaryPaths,
+        [
+            BinaryPathRequest(
+                binary_name=MyBin.binary_name,
+                search_path=[
+                    str(binary_path_abs1.parent),
+                    str(binary_path_abs2),
+                    str(binary_path_abs3.parent),
+                ],
+                check_file_entries=True,
+            )
+        ],
+    )
+    assert binary_paths.first_path is not None
+    assert binary_paths.first_path.path == str(binary_path_abs1)
+    assert [str(p) for p in (binary_path_abs1, binary_path_abs2, binary_path_abs3)] == [
+        binary_path.path for binary_path in binary_paths.paths
+    ]
diff --git a/src/python/pants/python/binaries_test.py b/src/python/pants/python/binaries_test.py
--- a/src/python/pants/python/binaries_test.py
+++ b/src/python/pants/python/binaries_test.py
@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import sys
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path, PurePath
@@ -295,3 +296,18 @@ def test_expand_interpreter_search_paths(rule_runner: RuleRunner) -> None:
         "/qux",
     ]
     assert expected == expanded_paths
+
+
+def test_interpreter_search_path_file_entries() -> None:
+    rule_runner = RuleRunner(
+        rules=[*python_binaries.rules(), QueryRule(PythonBinary, input_types=())]
+    )
+    current_python = os.path.realpath(sys.executable)
+    rule_runner.set_options(
+        args=[
+            f"--python-bootstrap-search-path=[{current_python!r}]",
+            f"--python-bootstrap-names=[{os.path.basename(current_python)!r}]",
+        ]
+    )
+    python_binary = rule_runner.request(PythonBinary, inputs=())
+    assert current_python == python_binary.path