Repository: pantsbuild/pants

Issue:
Unexplained invalidation of graph nodes causing re-runs of binary discovery
On `main` (and in `example-python`) with `-ldebug`, re-running tests with no changes shows that binary discovery is re-running when it shouldn't be (`"Find binary for $x"`). It seems to be somewhat non-deterministic, and occasionally causes other things to re-run as well.


Discussion:
Found at least two issues here... have fixed the first one, am trying to track down the second.

Resolving patch:
diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py
--- a/src/python/pants/backend/python/goals/pytest_runner.py
+++ b/src/python/pants/backend/python/goals/pytest_runner.py
@@ -33,7 +33,7 @@
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.addresses import Address
 from pants.engine.collection import Collection
-from pants.engine.environment import CompleteEnvironment
+from pants.engine.environment import Environment, EnvironmentRequest
 from pants.engine.fs import (
     EMPTY_DIGEST,
     AddPrefix,
@@ -159,7 +159,6 @@ async def setup_pytest_for_target(
     coverage_subsystem: CoverageSubsystem,
     test_extra_env: TestExtraEnv,
     global_options: GlobalOptions,
-    complete_env: CompleteEnvironment,
 ) -> TestSetup:
     transitive_targets, plugin_setups = await MultiGet(
         Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])),
@@ -195,17 +194,23 @@ async def setup_pytest_for_target(
     # to test, rather than using auto-discovery.
     field_set_source_files_get = Get(SourceFiles, SourceFilesRequest([request.field_set.sources]))
 
+    field_set_extra_env_get = Get(
+        Environment, EnvironmentRequest(request.field_set.extra_env_vars.value or ())
+    )
+
     (
         pytest_pex,
         requirements_pex,
         prepared_sources,
         field_set_source_files,
+        field_set_extra_env,
         extra_output_directory_digest,
     ) = await MultiGet(
         pytest_pex_get,
         requirements_pex_get,
         prepared_sources_get,
         field_set_source_files_get,
+        field_set_extra_env_get,
         extra_output_directory_digest_get,
     )
 
@@ -275,9 +280,9 @@ async def setup_pytest_for_target(
         "PYTEST_ADDOPTS": " ".join(add_opts),
         "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
         **test_extra_env.env,
-        # NOTE: `complete_env` intentionally after `test_extra_env` to allow overriding within
-        # `python_tests`
-        **complete_env.get_subset(request.field_set.extra_env_vars.value or ()),
+        # NOTE: field_set_extra_env intentionally after `test_extra_env` to allow overriding within
+        # `python_tests`.
+        **field_set_extra_env,
     }
 
     # Cache test runs only if they are successful, or not at all if `--test-force`.
diff --git a/src/python/pants/engine/environment.py b/src/python/pants/engine/environment.py
--- a/src/python/pants/engine/environment.py
+++ b/src/python/pants/engine/environment.py
@@ -17,12 +17,21 @@
 name_value_re = re.compile(r"([A-Za-z_]\w*)=(.*)")
 shorthand_re = re.compile(r"([A-Za-z_]\w*)")
 
+"""
+Accesses to `os.environ` cannot be accurately tracked, so @rules that need access to the
+environment should use APIs from this module instead.
+
+Wherever possible, the `Environment` type should be consumed rather than the
+`CompleteEnvironment`, as it represents a filtered/relevant subset of the environment, rather
+than the entire unfiltered environment.
+"""
+
 
 class CompleteEnvironment(FrozenDict):
     """CompleteEnvironment contains all environment variables from the current Pants process.
 
-    Accesses to `os.environ` cannot be accurately tracked, so @rules that need access to the
-    environment should request this type instead.
+    NB: Consumers should almost always prefer to consume the `Environment` type, which is
+    filtered to a relevant subset of the environment.
     """
 
     def get_subset(
diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py
--- a/src/python/pants/engine/process.py
+++ b/src/python/pants/engine/process.py
@@ -571,7 +571,9 @@ async def find_binary(request: BinaryPathRequest) -> BinaryPaths:
                 description=f"Test binary {path}.",
                 level=LogLevel.DEBUG,
                 argv=[path, *request.test.args],
-                cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
+                # NB: Since a failure is a valid result for this script, we always cache it for
+                # `pantsd`'s lifetime, regardless of success or failure.
+                cache_scope=ProcessCacheScope.PER_RESTART_ALWAYS,
             ),
         )
         for path in found_paths


Test patch:
diff --git a/src/python/pants/core/goals/test.py b/src/python/pants/core/goals/test.py
--- a/src/python/pants/core/goals/test.py
+++ b/src/python/pants/core/goals/test.py
@@ -22,7 +22,7 @@
 from pants.engine.console import Console
 from pants.engine.desktop import OpenFiles, OpenFilesRequest
 from pants.engine.engine_aware import EngineAwareReturnType
-from pants.engine.environment import CompleteEnvironment
+from pants.engine.environment import Environment, EnvironmentRequest
 from pants.engine.fs import Digest, FileDigest, MergeDigests, Snapshot, Workspace
 from pants.engine.goal import Goal, GoalSubsystem
 from pants.engine.process import FallibleProcessResult, InteractiveProcess, InteractiveRunner
@@ -39,7 +39,6 @@
     Targets,
 )
 from pants.engine.unions import UnionMembership, union
-from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -449,19 +448,12 @@ async def run_tests(
 
 @dataclass(frozen=True)
 class TestExtraEnv:
-    env: FrozenDict[str, str]
+    env: Environment
 
 
 @rule
-def get_filtered_environment(
-    test_subsystem: TestSubsystem, complete_env: CompleteEnvironment
-) -> TestExtraEnv:
-    env = (
-        complete_env.get_subset(test_subsystem.extra_env_vars)
-        if test_subsystem.extra_env_vars
-        else FrozenDict({})
-    )
-    return TestExtraEnv(env)
+async def get_filtered_environment(test_subsystem: TestSubsystem) -> TestExtraEnv:
+    return TestExtraEnv(await Get(Environment, EnvironmentRequest(test_subsystem.extra_env_vars)))
 
 
 # NB: We mark this uncachable to ensure that the results are always streamed, even if the