Repository: pantsbuild/pants

Issue:
`./pants run` crashes when `remote_cache_eager_fetch = false`
**Describe the bug**

After setting `remote_cache_eager_fetch = false`, a CI step that `./pants run`s a `pex_binary` has started (intermittently) failing with errors like:
```
 Exception: Was not present in either the local or remote store: Digest { hash: Fingerprint<46683dec8706b7ac7c4f6011f68b4b8c10ad423ae8ba57745a6f5e01ba5b2f7b>, size_bytes: 11827 }
```

**Pants version**

`PANTS_SHA=5d8a328d72209863986c8959b20305505bc068ba`

**OS**

Linux

**Additional info**

Some BuildSense links where we've seen the failure:
* https://app.toolchain.com/organizations/color/repos/color/builds/pants_run_2022_06_27_19_29_46_827_37a3a5f2d8e440cca373a7fb4a5d3b51/
* https://app.toolchain.com/organizations/color/repos/color/builds/pants_run_2022_06_27_19_31_09_277_3793f53b54914135885f2ac951faf210/
* https://app.toolchain.com/organizations/color/repos/color/builds/pants_run_2022_06_27_19_29_46_827_37a3a5f2d8e440cca373a7fb4a5d3b51/



Discussion:
cc @stuhood maybe another race condition with lazy remote fetches
Mm, thanks: I see what this is. Yet another TODO that I should have assumed would be a blocker!

Sorry for the trouble. We're prioritizing getting the `2.12.x` release out early this week, but I should be able to get a patch into `2.13.x` for this by EOD Wednesday-ish.

Resolving patch:
diff --git a/src/python/pants/base/exceptions.py b/src/python/pants/base/exceptions.py
--- a/src/python/pants/base/exceptions.py
+++ b/src/python/pants/base/exceptions.py
@@ -3,6 +3,11 @@
 
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pants.engine.internals.native_engine import PyFailure
+
 
 class TargetDefinitionException(Exception):
     """Indicates an invalid target definition.
@@ -28,3 +33,15 @@ class BackendConfigurationError(BuildConfigurationError):
 
 class MappingError(Exception):
     """Indicates an error mapping addressable objects."""
+
+
+class NativeEngineFailure(Exception):
+    """A wrapper around a `Failure` instance.
+
+    TODO: This type is defined in Python because pyo3 doesn't support declaring Exceptions with
+    additional fields. See https://github.com/PyO3/pyo3/issues/295
+    """
+
+    def __init__(self, msg: str, failure: PyFailure) -> None:
+        super().__init__(msg)
+        self.failure = failure


Test patch:
diff --git a/tests/python/pants_test/integration/remote_cache_integration_test.py b/tests/python/pants_test/integration/remote_cache_integration_test.py
--- a/tests/python/pants_test/integration/remote_cache_integration_test.py
+++ b/tests/python/pants_test/integration/remote_cache_integration_test.py
@@ -5,13 +5,16 @@
 
 import time
 
-from pants.engine.fs import Digest, DigestContents, DigestEntries, FileDigest, FileEntry
+from pants.core.util_rules import distdir
+from pants.core.util_rules.distdir import DistDir
+from pants.engine.fs import Digest, DigestContents, DigestEntries, FileDigest, FileEntry, Workspace
+from pants.engine.goal import Goal, GoalSubsystem
 from pants.engine.internals.native_engine import PyExecutor, PyStubCAS
 from pants.engine.process import Process, ProcessResult
-from pants.engine.rules import Get, rule
+from pants.engine.rules import Get, SubsystemRule, goal_rule, rule
 from pants.option.global_options import RemoteCacheWarningsBehavior
 from pants.testutil.pants_integration_test import run_pants
-from pants.testutil.rule_runner import QueryRule, RuleRunner, logging
+from pants.testutil.rule_runner import QueryRule, RuleRunner
 from pants.util.logging import LogLevel
 
 
@@ -99,8 +102,8 @@ async def entries_from_process(process_result: ProcessResult) -> ProcessOutputEn
     return ProcessOutputEntries(await Get(DigestEntries, Digest, process_result.output_digest))
 
 
-@logging
-def test_lazy_fetch_backtracking() -> None:
+def test_async_backtracking() -> None:
+    """Tests that we backtrack when a MissingDigest error occurs at a `@rule` (async) boundary."""
     executor = PyExecutor(core_threads=2, max_threads=4)
     cas = PyStubCAS.builder().build(executor)
 
@@ -149,3 +152,72 @@ def run() -> tuple[FileDigest, dict[str, int]]:
     assert metrics2["remote_cache_requests"] == 1
     assert metrics2["remote_cache_requests_cached"] == 1
     assert metrics2["backtrack_attempts"] == 1
+
+
+class MockRunSubsystem(GoalSubsystem):
+    name = "mock-run"
+    help = "Run a simple process and write its output to the dist dir."
+
+
+class MockRun(Goal):
+    subsystem_cls = MockRunSubsystem
+
+
+@goal_rule
+async def mock_run(workspace: Workspace, dist_dir: DistDir, mock_run: MockRunSubsystem) -> MockRun:
+    result = await Get(
+        ProcessResult,
+        Process(
+            ["/bin/bash", "-c", "sleep 1 ; echo content > file.txt"],
+            description="Create file.txt",
+            output_files=["file.txt"],
+            level=LogLevel.INFO,
+        ),
+    )
+    workspace.write_digest(
+        result.output_digest,
+        path_prefix=str(dist_dir.relpath),
+        side_effecting=False,
+    )
+    return MockRun(exit_code=0)
+
+
+def test_sync_backtracking() -> None:
+    """Tests that we backtrack when a MissingDigest error occurs synchronously (in write_digest)."""
+    executor = PyExecutor(core_threads=2, max_threads=4)
+    cas = PyStubCAS.builder().build(executor)
+
+    def run() -> dict[str, int]:
+        # Use an isolated store to ensure that the only content is in the remote/stub cache.
+        rule_runner = RuleRunner(
+            rules=[mock_run, *distdir.rules(), SubsystemRule(MockRunSubsystem)],
+            isolated_local_store=True,
+            bootstrap_args=[
+                "--no-remote-cache-eager-fetch",
+                "--no-local-cache",
+                *remote_cache_args(cas.address),
+            ],
+        )
+        result = rule_runner.run_goal_rule(MockRun, args=[])
+        assert result.exit_code == 0
+
+        # Wait for any async cache writes to complete.
+        time.sleep(1)
+        return rule_runner.scheduler.get_metrics()
+
+    # Run once to populate the remote cache, and validate that there is one entry afterwards.
+    assert cas.action_cache_len() == 0
+    metrics1 = run()
+    assert cas.action_cache_len() == 1
+    assert metrics1["remote_cache_requests"] == 1
+    assert metrics1["remote_cache_requests_uncached"] == 1
+
+    # Then, remove the content from the remote store and run again.
+    assert cas.remove(
+        FileDigest("434728a410a78f56fc1b5899c3593436e61ab0c731e9072d95e96db290205e53", 8)
+    )
+    metrics2 = run()
+    # Validate both that we hit the cache, and that we backtracked to actually run the process.
+    assert metrics2["remote_cache_requests"] == 1
+    assert metrics2["remote_cache_requests_cached"] == 1
+    assert metrics2["backtrack_attempts"] == 1