Repository: pantsbuild/pants

Issue:
Digest does not exist when switching repos with pantsd off in one, on in the other
```
18:08:49.83 [WARN] <unknown>:882: DeprecationWarning: invalid escape sequence \d
18:08:52.21 [WARN] Completed: Find PEX Python - No bootstrap Python executable could be found from the option `interpreter_search_paths` in the `[python-setup]` scope. Will attempt to run PEXes directly.
18:08:53.11 [WARN] /data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/base/exception_sink.py:359: DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats
  process_title=setproctitle.getproctitle(),

18:08:53.11 [ERROR] 1 Exception encountered:

Engine traceback:
  in select
  in `binary` goal
  in pants.backend.python.rules.create_python_binary.create_python_binary
  in pants.backend.python.rules.pex.two_step_create_pex
  in pants.backend.python.rules.pex.create_pex
Traceback (no traceback):
  <pants native internals>
Exception: String("Digest Digest(Fingerprint<97adba2ad1bfef3ba1b37d6b119e15498ed2a60392241b5ea7c28c602826dd6c>, 97) did not exist in the Store.")
Traceback (most recent call last):
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 255, in run
    engine_result = self._run_v2()
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 166, in _run_v2
    return self._maybe_run_v2_body(goals, poll=False)
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 183, in _maybe_run_v2_body
    return self.graph_session.run_goal_rules(
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/init/engine_initializer.py", line 130, in run_goal_rules
    exit_code = self.scheduler_session.run_goal_rule(
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/engine/internals/scheduler.py", line 561, in run_goal_rule
    self._raise_on_error([t for _, t in throws])
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/engine/internals/scheduler.py", line 520, in _raise_on_error
    raise ExecutionError(
pants.engine.internals.scheduler.ExecutionError: 1 Exception encountered:

Engine traceback:
  in select
  in `binary` goal
  in pants.backend.python.rules.create_python_binary.create_python_binary
  in pants.backend.python.rules.pex.two_step_create_pex
  in pants.backend.python.rules.pex.create_pex
Traceback (no traceback):
  <pants native internals>
Exception: String("Digest Digest(Fingerprint<97adba2ad1bfef3ba1b37d6b119e15498ed2a60392241b5ea7c28c602826dd6c>, 97) did not exist in the Store.")
```


Discussion:
sometime, I see this extra error info:
```
  in pants.backend.python.rules.create_python_binary.create_python_binary
  in pants.backend.python.rules.pex.two_step_create_pex
  in pants.backend.python.rules.pex.create_pex
Traceback (no traceback):
  <pants native internals>
Exception: String("Digest Digest(Fingerprint<3dcdb1a3bd62e17cef0250301a85627cdd844605c1e1a514076b429cd39e6870>, 97) did not exist in the Store.")
Fatal Python error: This thread state must be current when releasing
Python runtime state: finalizing (tstate=0x556a5a059f10)

Thread 0x00007f9609246700 (most recent call first):
<no Python frame>

Thread 0x00007f9603fff700 (most recent call first):
<no Python frame>

Current thread 0x00007f9609045700 (most recent call first):
<no Python frame>

Thread 0x00007f9608e44700 (most recent call first):
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 4269 in postParse
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1408 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 3552 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 4005 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 3417 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 3400 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 3552 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 3417 in parseImpl
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1402 in _parseNoCache
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", line 1644 in parseString
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/requirements.py", line 98 in __init__
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/__init__.py", line 3119 in __init__
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pkg_resources/__init__.py", line 3109 in parse_requirements
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/backend/python/rules/pex_from_targets.py", line 209 in pex_from_targets
  File "/data/home/asher/.cache/pants/setup/bootstrap-Linux-x86_64/2.0.0a1_py38/lib/python3.8/site-packages/pants/engine/internals/native.py", line 67 in generator_send

Thread 0x00007f9611881080 (most recent call first):
<no Python frame>
Aborted (core dumped)
```

@stuhood @gshuflin 
. @asherf mentioned that this occurs on a machine where some repositories are using `pantsd` and some aren't. That should be accounted for in the lease extension code, but it might still have a gap. Determining whether the missing digest is an "inner" node (below a "root" digest that was kept alive) would likely be interesting.

Added to the `2.0.x` milestone.
I got this when running in the TC codebase (no pantsd) after just having had run in Pants's codebase (with pantsd):

```
Engine traceback:
  in select
  in `typecheck` goal
  in Typecheck using MyPy
  in pants.backend.python.util_rules.pex.create_pex
  in pants.backend.python.util_rules.pex_cli.setup_pex_cli_process
  in Find PEX Python
  in Find binary path
  in pants.engine.process.remove_platform_information
Traceback (no traceback):
  <pants native internals>
Exception: Bytes from stdout Digest Digest(Fingerprint<55fd2022c440089e0812a6a9dc5affeba6ba3d30715595dfde9092125d93909b>, 93) not found in store
```

Resolving patch:
diff --git a/src/python/pants/engine/internals/scheduler.py b/src/python/pants/engine/internals/scheduler.py
--- a/src/python/pants/engine/internals/scheduler.py
+++ b/src/python/pants/engine/internals/scheduler.py
@@ -346,8 +346,8 @@ def python_signal() -> bool:
     def lease_files_in_graph(self, session):
         self._native.lib.lease_files_in_graph(self._scheduler, session)
 
-    def garbage_collect_store(self):
-        self._native.lib.garbage_collect_store(self._scheduler)
+    def garbage_collect_store(self, target_size_bytes: int) -> None:
+        self._native.lib.garbage_collect_store(self._scheduler, target_size_bytes)
 
     def new_session(
         self,
@@ -659,5 +659,5 @@ def write_digest(self, digest: Digest, *, path_prefix: Optional[str] = None) ->
     def lease_files_in_graph(self):
         self._scheduler.lease_files_in_graph(self._session)
 
-    def garbage_collect_store(self):
-        self._scheduler.garbage_collect_store()
+    def garbage_collect_store(self, target_size_bytes: int) -> None:
+        self._scheduler.garbage_collect_store(target_size_bytes)
diff --git a/src/python/pants/pantsd/service/store_gc_service.py b/src/python/pants/pantsd/service/store_gc_service.py
--- a/src/python/pants/pantsd/service/store_gc_service.py
+++ b/src/python/pants/pantsd/service/store_gc_service.py
@@ -13,14 +13,19 @@ class StoreGCService(PantsService):
 
     This service both ensures that in-use files continue to be present in the engine's Store, and
     performs occasional garbage collection to bound the size of the engine's Store.
+
+    NB: The lease extension interval should be significantly less than the rust-side
+    sharded_lmdb::DEFAULT_LEASE_TIME to ensure that valid leases are extended well before they
+    might expire.
     """
 
     def __init__(
         self,
         scheduler: Scheduler,
         period_secs=10,
-        lease_extension_interval_secs=(30 * 60),
-        gc_interval_secs=(4 * 60 * 60),
+        lease_extension_interval_secs=(15 * 60),
+        gc_interval_secs=(1 * 60 * 60),
+        target_size_bytes=(4 * 1024 * 1024 * 1024),
     ):
         super().__init__()
         self._scheduler_session = scheduler.new_session(build_id="store_gc_service_session")
@@ -29,6 +34,7 @@ def __init__(
         self._period_secs = period_secs
         self._lease_extension_interval_secs = lease_extension_interval_secs
         self._gc_interval_secs = gc_interval_secs
+        self._target_size_bytes = target_size_bytes
 
         self._set_next_gc()
         self._set_next_lease_extension()
@@ -51,7 +57,7 @@ def _maybe_garbage_collect(self):
         if time.time() < self._next_gc:
             return
         self._logger.info("Garbage collecting store")
-        self._scheduler_session.garbage_collect_store()
+        self._scheduler_session.garbage_collect_store(self._target_size_bytes)
         self._logger.info("Done garbage collecting store")
         self._set_next_gc()
 


Test patch:
diff --git a/src/rust/engine/process_execution/src/cache_tests.rs b/src/rust/engine/process_execution/src/cache_tests.rs
--- a/src/rust/engine/process_execution/src/cache_tests.rs
+++ b/src/rust/engine/process_execution/src/cache_tests.rs
@@ -2,10 +2,12 @@ use crate::{
   CommandRunner as CommandRunnerTrait, Context, FallibleProcessResultWithPlatform, NamedCaches,
   Process, ProcessMetadata,
 };
+
+use std::convert::TryInto;
 use std::io::Write;
 use std::path::PathBuf;
-use std::sync::Arc;
 
+use futures::compat::Future01CompatExt;
 use sharded_lmdb::{ShardedLmdb, DEFAULT_LEASE_TIME};
 use store::Store;
 use tempfile::TempDir;
@@ -17,20 +19,55 @@ struct RoundtripResults {
   maybe_cached: Result<FallibleProcessResultWithPlatform, String>,
 }
 
-async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
+fn create_local_runner() -> (Box<dyn CommandRunnerTrait>, Store, TempDir) {
   let runtime = task_executor::Executor::new(Handle::current());
-  let work_dir = TempDir::new().unwrap();
-  let named_cache_dir = TempDir::new().unwrap();
-  let store_dir = TempDir::new().unwrap();
-  let store = Store::local_only(runtime.clone(), store_dir.path()).unwrap();
-  let local = crate::local::CommandRunner::new(
+  let base_dir = TempDir::new().unwrap();
+  let named_cache_dir = base_dir.path().join("named_cache_dir");
+  let store_dir = base_dir.path().join("store_dir");
+  let store = Store::local_only(runtime.clone(), store_dir).unwrap();
+  let runner = Box::new(crate::local::CommandRunner::new(
     store.clone(),
     runtime.clone(),
-    work_dir.path().to_owned(),
-    NamedCaches::new(named_cache_dir.path().to_owned()),
+    base_dir.path().to_owned(),
+    NamedCaches::new(named_cache_dir),
     true,
-  );
+  ));
+  (runner, store, base_dir)
+}
+
+fn create_cached_runner(
+  local: Box<dyn CommandRunnerTrait>,
+  store: Store,
+) -> (Box<dyn CommandRunnerTrait>, TempDir) {
+  let runtime = task_executor::Executor::new(Handle::current());
+  let cache_dir = TempDir::new().unwrap();
+  let max_lmdb_size = 50 * 1024 * 1024; //50 MB - I didn't pick that number but it seems reasonable.
+
+  let process_execution_store = ShardedLmdb::new(
+    cache_dir.path().to_owned(),
+    max_lmdb_size,
+    runtime.clone(),
+    DEFAULT_LEASE_TIME,
+  )
+  .unwrap();
+
+  let metadata = ProcessMetadata {
+    instance_name: None,
+    cache_key_gen_version: None,
+    platform_properties: vec![],
+  };
 
+  let runner = Box::new(crate::cache::CommandRunner::new(
+    local.into(),
+    process_execution_store,
+    store,
+    metadata,
+  ));
+
+  (runner, cache_dir)
+}
+
+fn create_script(script_exit_code: i8) -> (Process, PathBuf, TempDir) {
   let script_dir = TempDir::new().unwrap();
   let script_path = script_dir.path().join("script");
   std::fs::File::create(&script_path)
@@ -44,40 +81,25 @@ async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
     })
     .unwrap();
 
-  let request = Process::new(vec![
+  let process = Process::new(vec![
     testutil::path::find_bash(),
     format!("{}", script_path.display()),
   ])
   .output_files(vec![PathBuf::from("roland")].into_iter().collect());
 
-  let local_result = local.run(request.clone().into(), Context::default()).await;
-
-  let cache_dir = TempDir::new().unwrap();
-  let max_lmdb_size = 50 * 1024 * 1024; //50 MB - I didn't pick that number but it seems reasonable.
+  (process, script_path, script_dir)
+}
 
-  let process_execution_store = ShardedLmdb::new(
-    cache_dir.path().to_owned(),
-    max_lmdb_size,
-    runtime.clone(),
-    DEFAULT_LEASE_TIME,
-  )
-  .unwrap();
+async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
+  let (local, store, _local_runner_dir) = create_local_runner();
+  let (process, script_path, _script_dir) = create_script(script_exit_code);
 
-  let metadata = ProcessMetadata {
-    instance_name: None,
-    cache_key_gen_version: None,
-    platform_properties: vec![],
-  };
+  let local_result = local.run(process.clone().into(), Context::default()).await;
 
-  let caching = crate::cache::CommandRunner::new(
-    Arc::new(local),
-    process_execution_store,
-    store.clone(),
-    metadata,
-  );
+  let (caching, _cache_dir) = create_cached_runner(local, store.clone());
 
   let uncached_result = caching
-    .run(request.clone().into(), Context::default())
+    .run(process.clone().into(), Context::default())
     .await;
 
   assert_eq!(local_result, uncached_result);
@@ -86,7 +108,7 @@ async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
   // fail due to a FileNotFound error. So, If the second run succeeds, that implies that the
   // cache was successfully used.
   std::fs::remove_file(&script_path).unwrap();
-  let maybe_cached_result = caching.run(request.into(), Context::default()).await;
+  let maybe_cached_result = caching.run(process.into(), Context::default()).await;
 
   RoundtripResults {
     uncached: uncached_result,
@@ -107,3 +129,56 @@ async fn failures_not_cached() {
   assert_eq!(results.uncached.unwrap().exit_code, 1);
   assert_eq!(results.maybe_cached.unwrap().exit_code, 127); // aka the return code for file not found
 }
+
+#[tokio::test]
+async fn recover_from_missing_store_contents() {
+  let (local, store, _local_runner_dir) = create_local_runner();
+  let (caching, _cache_dir) = create_cached_runner(local, store.clone());
+  let (process, _script_path, _script_dir) = create_script(0);
+
+  // Run once to cache the process.
+  let first_result = caching
+    .run(process.clone().into(), Context::default())
+    .await
+    .unwrap();
+
+  // Delete the first child of the output directory parent to confirm that we ensure that more
+  // than just the root of the output is present when hitting the cache.
+  {
+    let output_dir_digest = first_result.output_directory;
+    let (output_dir, _) = store
+      .load_directory(output_dir_digest)
+      .await
+      .unwrap()
+      .unwrap();
+    let output_child_digest = output_dir
+      .get_files()
+      .first()
+      .unwrap()
+      .get_digest()
+      .try_into()
+      .unwrap();
+    let removed = store.remove_file(output_child_digest).await.unwrap();
+    assert!(removed);
+    assert!(store
+      .contents_for_directory(output_dir_digest)
+      .compat()
+      .await
+      .err()
+      .is_some())
+  }
+
+  // Ensure that we don't fail if we re-run.
+  let second_result = caching
+    .run(process.clone().into(), Context::default())
+    .await
+    .unwrap();
+
+  // And that the entire output directory can be loaded.
+  assert!(store
+    .contents_for_directory(second_result.output_directory)
+    .compat()
+    .await
+    .ok()
+    .is_some())
+}