Repository: pantsbuild/pants

Issue:
Add an "eager fetch/validate" mode to remote caching
When using only remote caching (and not remote execution), the [single `Store` instance](https://github.com/pantsbuild/pants/blob/0bf6ac6685aa1200716da640c0b3c562d7bc100b/src/rust/engine/src/context.rs#L107-L140) is configured with a remote backend as well as a local backend. This means that all usage of the `Store` (by design) will potentially result in remote lookups if a `Digest` cannot be found locally.

Having the `Store` backed with a remote is currently necessary, because when the `remote_cache::Runner` [hits the cache](https://github.com/pantsbuild/pants/blob/0bf6ac6685aa1200716da640c0b3c562d7bc100b/src/rust/engine/process_execution/src/remote_cache.rs#L410-L445), it does not eagerly download all of the content represented by the cache entry. This is generally good for performance: similar to with remote execution, if you hit the cache a few times in sequence, you can skip ever downloading the actual cache entries to disk. And if you do need to download the content later, you can do so lazily.

But fetching artifacts lazily from the cache can be problematic if any of the cache content is missing from the remote store: the `remote_cache` module recovers from errors if data is missing, and converts the error into a cache miss. But elsewhere in the codebase, usages of the `Store` will not have any way (currently: longer term ticket forthcoming) to recover from a missing `Digest`.

To resolve this in the short term, we should add either an "eager validate" or "eager fetch" toggle for remote caching.
* To "eagerly fetch", the `remote_cache::Runner` would be adjusted to eagerly download all of the cache content when it hits (very similar to [what the local `cache` does](https://github.com/pantsbuild/pants/blob/0bf6ac6685aa1200716da640c0b3c562d7bc100b/src/rust/engine/process_execution/src/cache.rs#L158-L174)).
    * Additionally, when constructing the `Store` instance for remote caching, we would give the `remote_cache::Runner` a `Store` _with_ a remote backend configured, but we would give the wrapped/inner `Runner` a clone of the `Store` _without_ the remote backend via the `Store::into_local_only` method: `let local_store = remote_and_local_store.clone().into_local_only();`.
* To "eagerly validate", the `remote_cache::Runner` would make network calls to validate that all content existed, _without_ fetching all of it. This would reduce the risk of lazily fetching cache content, because errors due to missing content could only happen if content disappeared from the cache between the cache hit and when it was used later.
    * This approach is not as foolproof as the "eagerly fetch" mode: 1) there is still a chance that the content drops out before it can be lazily fetched, 2) the `Store` would still need to be configured with a remote backend, and infrastructure errors in that backend elsewhere in the code could still cause issues. The latter issue could be ameliorated with retry, but the former error would require deeper changes.


Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py
--- a/src/python/pants/engine/internals/native.py
+++ b/src/python/pants/engine/internals/native.py
@@ -29,6 +29,7 @@
 from pants.engine.internals.session import SessionValues
 from pants.engine.rules import Get
 from pants.engine.unions import union
+from pants.option.global_options import ExecutionOptions
 from pants.util.logging import LogLevel
 from pants.util.memo import memoized_property
 from pants.util.meta import SingletonMetaclass
@@ -243,7 +244,7 @@ def new_scheduler(
         ignore_patterns: List[str],
         use_gitignore: bool,
         executor: PyExecutor,
-        execution_options,
+        execution_options: ExecutionOptions,
         types: PyTypes,
     ) -> PyScheduler:
         """Create and return a native Scheduler."""
@@ -264,6 +265,7 @@ def new_scheduler(
             store_initial_timeout=execution_options.remote_store_initial_timeout,
             store_timeout_multiplier=execution_options.remote_store_timeout_multiplier,
             store_maximum_timeout=execution_options.remote_store_maximum_timeout,
+            cache_eager_fetch=execution_options.remote_cache_eager_fetch,
             execution_extra_platform_properties=tuple(
                 tuple(pair.split("=", 1))
                 for pair in execution_options.remote_execution_extra_platform_properties
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -90,6 +90,7 @@ class ExecutionOptions:
     remote_store_initial_timeout: int
     remote_store_timeout_multiplier: float
     remote_store_maximum_timeout: int
+    remote_cache_eager_fetch: bool
 
     @classmethod
     def from_bootstrap_options(cls, bootstrap_options):
@@ -118,6 +119,7 @@ def from_bootstrap_options(cls, bootstrap_options):
             process_execution_local_enable_nailgun=bootstrap_options.process_execution_local_enable_nailgun,
             remote_cache_read=bootstrap_options.remote_cache_read,
             remote_cache_write=bootstrap_options.remote_cache_write,
+            remote_cache_eager_fetch=bootstrap_options.remote_cache_eager_fetch,
             remote_store_initial_timeout=bootstrap_options.remote_store_initial_timeout,
             remote_store_timeout_multiplier=bootstrap_options.remote_store_timeout_multiplier,
             remote_store_maximum_timeout=bootstrap_options.remote_store_maximum_timeout,
@@ -152,6 +154,7 @@ def from_bootstrap_options(cls, bootstrap_options):
     process_execution_local_enable_nailgun=False,
     remote_cache_read=False,
     remote_cache_write=False,
+    remote_cache_eager_fetch=True,
     remote_store_initial_timeout=10,
     remote_store_timeout_multiplier=2.0,
     remote_store_maximum_timeout=10,
@@ -692,21 +695,42 @@ def register_bootstrap_options(cls, register):
             advanced=True,
             type=bool,
             default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
-            help="Enables remote workers for increased parallelism. (Alpha)",
+            help=(
+                "Enables remote workers for increased parallelism. (Alpha)\n\nAlternatively, you "
+                "can use `--remote-cache-read` and `--remote-cache-write` to still run everything "
+                "locally, but to use a remote cache."
+            ),
         )
         register(
             "--remote-cache-read",
             type=bool,
             default=DEFAULT_EXECUTION_OPTIONS.remote_cache_read,
             advanced=True,
-            help="Whether to enable reading from a remote cache",
+            help=(
+                "Whether to enable reading from a remote cache.\n\nThis cannot be used at the same "
+                "time as `--remote-execution`."
+            ),
         )
         register(
             "--remote-cache-write",
             type=bool,
             default=DEFAULT_EXECUTION_OPTIONS.remote_cache_write,
             advanced=True,
-            help="Whether to enable writing results to a remote cache",
+            help=(
+                "Whether to enable writing results to a remote cache.\n\nThis cannot be used at "
+                "the same time as `--remote-execution`."
+            ),
+        )
+        register(
+            "--remote-cache-eager-fetch",
+            type=bool,
+            advanced=True,
+            default=DEFAULT_EXECUTION_OPTIONS.remote_cache_eager_fetch,
+            help=(
+                "Eagerly fetch relevant content from the remote store instead of lazily fetching."
+                "\n\nThis may result in worse performance, but reduce the frequency of errors "
+                "encountered by reducing the surface area of when remote caching is used."
+            ),
         )
 
         register(
@@ -990,16 +1014,26 @@ def validate_instance(cls, opts):
                 "`--remote-store-server` to work properly."
             )
 
+        if opts.remote_execution and (opts.remote_cache_read or opts.remote_cache_write):
+            raise OptionsError(
+                "`--remote-execution` cannot be set at the same time as either "
+                "`--remote-cache-read` or `--remote-cache-write`.\n\nIf remote execution is "
+                "enabled, it will already use remote caching."
+            )
+
         # Ensure that timeout values are non-zero.
         if opts.remote_store_initial_timeout <= 0:
             raise OptionsError(
-                "The --remote-store-initial-timeout option requires a positive number of milliseconds."
+                "The --remote-store-initial-timeout option requires a positive number of "
+                "milliseconds."
             )
         if opts.remote_store_timeout_multiplier <= 0.0:
             raise OptionsError(
-                "The --remote-store-timeout-multiplier option requires a positive number for the multiplier."
+                "The --remote-store-timeout-multiplier option requires a positive number for the "
+                "multiplier."
             )
         if opts.remote_store_maximum_timeout <= 0:
             raise OptionsError(
-                "The --remote-store-initial-timeout option requires a positive number of milliseconds."
+                "The --remote-store-initial-timeout option requires a positive number of "
+                "milliseconds."
             )


Test patch:
diff --git a/src/rust/engine/process_execution/src/remote_cache_tests.rs b/src/rust/engine/process_execution/src/remote_cache_tests.rs
--- a/src/rust/engine/process_execution/src/remote_cache_tests.rs
+++ b/src/rust/engine/process_execution/src/remote_cache_tests.rs
@@ -12,12 +12,14 @@ use fs::RelativePath;
 use hashing::Digest;
 use maplit::hashset;
 use mock::{StubActionCache, StubCAS};
+use remexec::ActionResult;
 use store::{BackoffConfig, Store};
 use tempfile::TempDir;
 use testutil::data::{TestData, TestDirectory, TestTree};
 use testutil::relative_paths;
 use workunit_store::WorkunitStore;
 
+use crate::remote::{ensure_action_stored_locally, make_execute_request};
 use crate::{
   CommandRunner as CommandRunnerTrait, Context, FallibleProcessResultWithPlatform,
   MultiPlatformProcess, NamedCaches, Platform, Process, ProcessMetadata,
@@ -62,6 +64,7 @@ fn create_local_runner() -> (Box<dyn CommandRunnerTrait>, Store, TempDir, StubCA
 fn create_cached_runner(
   local: Box<dyn CommandRunnerTrait>,
   store: Store,
+  eager_fetch: bool,
 ) -> (Box<dyn CommandRunnerTrait>, TempDir, StubActionCache) {
   let cache_dir = TempDir::new().unwrap();
   let action_cache = StubActionCache::new().unwrap();
@@ -77,6 +80,7 @@ fn create_cached_runner(
       Platform::current().unwrap(),
       true,
       true,
+      eager_fetch,
     )
     .expect("caching command runner"),
   );
@@ -112,7 +116,7 @@ async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
 
   let local_result = local.run(process.clone().into(), Context::default()).await;
 
-  let (caching, _cache_dir, _stub_action_cache) = create_cached_runner(local, store.clone());
+  let (caching, _cache_dir, _stub_action_cache) = create_cached_runner(local, store.clone(), false);
 
   let uncached_result = caching
     .run(process.clone().into(), Context::default())
@@ -158,7 +162,7 @@ async fn skip_cache_on_error() {
   workunit_store.init_thread_state(None);
 
   let (local, store, _local_runner_dir, _stub_cas) = create_local_runner();
-  let (caching, _cache_dir, stub_action_cache) = create_cached_runner(local, store.clone());
+  let (caching, _cache_dir, stub_action_cache) = create_cached_runner(local, store.clone(), false);
   let (process, _script_path, _script_dir) = create_script(0);
 
   stub_action_cache
@@ -174,6 +178,58 @@ async fn skip_cache_on_error() {
   assert_eq!(result.exit_code, 0);
 }
 
+/// With eager_fetch enabled, we should skip the remote cache if any of the process result's
+/// digests are invalid. This will force rerunning the process locally. Otherwise, we should use
+/// the cached result with its non-existent digests.
+#[tokio::test]
+async fn eager_fetch() {
+  let workunit_store = WorkunitStore::new(false);
+  workunit_store.init_thread_state(None);
+
+  async fn run_process(eager_fetch: bool) -> FallibleProcessResultWithPlatform {
+    let (local, store, _local_runner_dir, _stub_cas) = create_local_runner();
+    let (caching, _cache_dir, stub_action_cache) =
+      create_cached_runner(local, store.clone(), eager_fetch);
+
+    // Get the `action_digest` for the Process that we're going to run. This will allow us to
+    // insert a bogus value into the `stub_action_cache`.
+    let (process, _script_path, _script_dir) = create_script(1);
+    let (action, command, _exec_request) =
+      make_execute_request(&process, ProcessMetadata::default()).unwrap();
+    let (_command_digest, action_digest) = ensure_action_stored_locally(&store, &command, &action)
+      .await
+      .unwrap();
+
+    // Insert an ActionResult with missing digests and a return code of 0 (instead of 1).
+    let bogus_action_result = ActionResult {
+      exit_code: 0,
+      stdout_digest: Some(TestData::roland().digest().into()),
+      stderr_digest: Some(TestData::roland().digest().into()),
+      ..ActionResult::default()
+    };
+    stub_action_cache
+      .action_map
+      .lock()
+      .insert(action_digest.0, bogus_action_result);
+
+    // Run the process, possibly by pulling from the `ActionCache`.
+    caching
+      .run(process.clone().into(), Context::default())
+      .await
+      .unwrap()
+  }
+
+  let lazy_result = run_process(false).await;
+  assert_eq!(lazy_result.exit_code, 0);
+  assert_eq!(lazy_result.stdout_digest, TestData::roland().digest());
+  assert_eq!(lazy_result.stderr_digest, TestData::roland().digest());
+
+  let eager_result = run_process(true).await;
+  assert_eq!(eager_result.exit_code, 1);
+  assert_ne!(eager_result.stdout_digest, TestData::roland().digest());
+  assert_ne!(eager_result.stderr_digest, TestData::roland().digest());
+}
+
 #[tokio::test]
 async fn make_tree_from_directory() {
   let store_dir = TempDir::new().unwrap();
@@ -334,6 +390,7 @@ async fn make_action_result_basic() {
     Platform::current().unwrap(),
     true,
     true,
+    false,
   )
   .expect("caching command runner");