Repository: pantsbuild/pants

Issue:
Garbage collection of the store is overly eager
I've seen a few cases of errors like:
```
File with digest Digest(Fingerprint<..>, 5175) not found
```
and
```
Couldn't find file contents for "contrib/go/src/python/pants/contrib/go/tasks/BUILD"
```
...which occur during 100% local runs with pantsd.

This is not confirmed, but I have a strong suspicion that when `pantsd` is running in multiple repositories (note: different repositories, although that doesn't feel strictly necessary), that garbage collection is overly eager. I suspect that multiple repositories is the relevant bit, because this garbage collection was deployed at scale in a single repository at Twitter, and we had no reports of this kind of issue.

----

Regardless, this probably boils down to leases not being extended on things that are being held in memory by `pantsd`, which then allows other `pantsd` to clear them.

To work around the issue, in some cases the `pantsd` process can be killed/restarted. In other cases, when content contained within a "directory" has been garbage collected, it's necessary to remove `~/.cache/pants/lmdb_store`.


Discussion:
This is a very high priority issue for 1.29.x.

Resolving patch:
diff --git a/src/python/pants/pantsd/service/store_gc_service.py b/src/python/pants/pantsd/service/store_gc_service.py
--- a/src/python/pants/pantsd/service/store_gc_service.py
+++ b/src/python/pants/pantsd/service/store_gc_service.py
@@ -44,17 +44,17 @@ def _set_next_lease_extension(self):
     def _maybe_extend_lease(self):
         if time.time() < self._next_lease_extension:
             return
-        self._logger.debug("Extending leases")
+        self._logger.info("Extending leases")
         self._scheduler_session.lease_files_in_graph()
-        self._logger.debug("Done extending leases")
+        self._logger.info("Done extending leases")
         self._set_next_lease_extension()
 
     def _maybe_garbage_collect(self):
         if time.time() < self._next_gc:
             return
-        self._logger.debug("Garbage collecting store")
+        self._logger.info("Garbage collecting store")
         self._scheduler_session.garbage_collect_store()
-        self._logger.debug("Done garbage collecting store")
+        self._logger.info("Done garbage collecting store")
         self._set_next_gc()
 
     def run(self):


Test patch:
diff --git a/src/rust/engine/fs/store/src/local_tests.rs b/src/rust/engine/fs/store/src/local_tests.rs
--- a/src/rust/engine/fs/store/src/local_tests.rs
+++ b/src/rust/engine/fs/store/src/local_tests.rs
@@ -1,11 +1,15 @@
 use crate::local::ByteStore;
 use crate::{EntryType, ShrinkBehavior};
+
+use std::path::Path;
+use std::time::Duration;
+
 use bytes::{BufMut, Bytes, BytesMut};
 use hashing::{Digest, Fingerprint};
-use std::path::Path;
 use tempfile::TempDir;
 use testutil::data::{TestData, TestDirectory};
 use tokio::runtime::Handle;
+use tokio::time::delay_for;
 use walkdir::WalkDir;
 
 #[tokio::test]
@@ -149,7 +153,8 @@ async fn garbage_collect_nothing_to_do_with_lease() {
   .unwrap();
   let file_digest = Digest(file_fingerprint, 10);
   store
-    .lease_all(vec![file_digest].iter())
+    .lease_all(vec![(file_digest, EntryType::File)].into_iter())
+    .await
     .expect("Error leasing");
   store
     .shrink(10, ShrinkBehavior::Fast)
@@ -160,6 +165,46 @@ async fn garbage_collect_nothing_to_do_with_lease() {
   );
 }
 
+#[tokio::test]
+async fn garbage_collect_expired() {
+  let lease_time = Duration::from_secs(1);
+  let dir = TempDir::new().unwrap();
+  let store = new_store_with_lease_time(dir.path(), lease_time);
+  let bytes = Bytes::from("0123456789");
+  let file_fingerprint = Fingerprint::from_hex_string(
+    "84d89877f0d4041efb6bf91a16f0248f2fd573e6af05c19f96bedb9f882f7882",
+  )
+  .unwrap();
+  let file_len = 10;
+  let file_digest = Digest(file_fingerprint, file_len);
+
+  // Store something (in a store with a shortened lease). Confirm that it hasn't immediately
+  // expired, and then wait for it to expire.
+  store
+    .store_bytes(EntryType::File, bytes.clone(), true)
+    .await
+    .expect("Error storing");
+  assert_eq!(
+    file_len,
+    store
+      .shrink(0, ShrinkBehavior::Fast)
+      .expect("Error shrinking"),
+  );
+  assert_eq!(
+    load_bytes(&store, EntryType::File, file_digest).await,
+    Ok(Some(bytes))
+  );
+
+  // Wait for it to expire.
+  delay_for(lease_time * 2).await;
+  assert_eq!(
+    0,
+    store
+      .shrink(0, ShrinkBehavior::Fast)
+      .expect("Should have cleared expired lease")
+  );
+}
+
 #[tokio::test]
 async fn garbage_collect_remove_one_of_two_files_no_leases() {
   let dir = TempDir::new().unwrap();
@@ -440,7 +485,7 @@ async fn entry_type_for_file() {
     .expect("Error storing");
   prime_store_with_file_bytes(&store, testdata.bytes()).await;
   assert_eq!(
-    store.entry_type(&testdata.fingerprint()),
+    store.entry_type(testdata.fingerprint()).await,
     Ok(Some(EntryType::File))
   )
 }
@@ -457,7 +502,7 @@ async fn entry_type_for_directory() {
     .expect("Error storing");
   prime_store_with_file_bytes(&store, testdata.bytes()).await;
   assert_eq!(
-    store.entry_type(&testdir.fingerprint()),
+    store.entry_type(testdir.fingerprint()).await,
     Ok(Some(EntryType::Directory))
   )
 }
@@ -474,7 +519,9 @@ async fn entry_type_for_missing() {
     .expect("Error storing");
   prime_store_with_file_bytes(&store, testdata.bytes()).await;
   assert_eq!(
-    store.entry_type(&TestDirectory::recursive().fingerprint()),
+    store
+      .entry_type(TestDirectory::recursive().fingerprint())
+      .await,
     Ok(None)
   )
 }
@@ -517,6 +564,15 @@ pub fn new_store<P: AsRef<Path>>(dir: P) -> ByteStore {
   ByteStore::new(task_executor::Executor::new(Handle::current()), dir).unwrap()
 }
 
+pub fn new_store_with_lease_time<P: AsRef<Path>>(dir: P, lease_time: Duration) -> ByteStore {
+  ByteStore::new_with_lease_time(
+    task_executor::Executor::new(Handle::current()),
+    dir,
+    lease_time,
+  )
+  .unwrap()
+}
+
 pub async fn load_file_bytes(store: &ByteStore, digest: Digest) -> Result<Option<Bytes>, String> {
   load_bytes(&store, EntryType::File, digest).await
 }
diff --git a/src/rust/engine/fs/store/src/tests.rs b/src/rust/engine/fs/store/src/tests.rs
--- a/src/rust/engine/fs/store/src/tests.rs
+++ b/src/rust/engine/fs/store/src/tests.rs
@@ -787,7 +787,7 @@ async fn upload_missing_files() {
     .expect_err("Want error");
   assert_eq!(
     error,
-    format!("Failed to upload digest {:?}: Not found", testdata.digest())
+    format!("Failed to expand digest {:?}: Not found", testdata.digest())
   );
 }
 
diff --git a/src/rust/engine/graph/src/tests.rs b/src/rust/engine/graph/src/tests.rs
--- a/src/rust/engine/graph/src/tests.rs
+++ b/src/rust/engine/graph/src/tests.rs
@@ -7,7 +7,6 @@ use std::thread;
 use std::time::{Duration, Instant};
 
 use async_trait::async_trait;
-use hashing::Digest;
 use parking_lot::Mutex;
 use rand::{self, Rng};
 use tokio::time::{timeout, Elapsed};
@@ -624,10 +623,6 @@ impl Node for TNode {
     }
   }
 
-  fn digest(_result: Self::Item) -> Option<Digest> {
-    None
-  }
-
   fn cacheable(&self) -> bool {
     self.1
   }
diff --git a/src/rust/engine/process_execution/src/cache_tests.rs b/src/rust/engine/process_execution/src/cache_tests.rs
--- a/src/rust/engine/process_execution/src/cache_tests.rs
+++ b/src/rust/engine/process_execution/src/cache_tests.rs
@@ -2,10 +2,11 @@ use crate::{
   CommandRunner as CommandRunnerTrait, Context, FallibleProcessResultWithPlatform, NamedCaches,
   Process, ProcessMetadata,
 };
-use sharded_lmdb::ShardedLmdb;
 use std::io::Write;
 use std::path::PathBuf;
 use std::sync::Arc;
+
+use sharded_lmdb::{ShardedLmdb, DEFAULT_LEASE_TIME};
 use store::Store;
 use tempfile::TempDir;
 use testutil::data::TestData;
@@ -54,8 +55,13 @@ async fn run_roundtrip(script_exit_code: i8) -> RoundtripResults {
   let cache_dir = TempDir::new().unwrap();
   let max_lmdb_size = 50 * 1024 * 1024; //50 MB - I didn't pick that number but it seems reasonable.
 
-  let process_execution_store =
-    ShardedLmdb::new(cache_dir.path().to_owned(), max_lmdb_size, runtime.clone()).unwrap();
+  let process_execution_store = ShardedLmdb::new(
+    cache_dir.path().to_owned(),
+    max_lmdb_size,
+    runtime.clone(),
+    DEFAULT_LEASE_TIME,
+  )
+  .unwrap();
 
   let metadata = ProcessMetadata {
     instance_name: None,