Repository: pantsbuild/pants

Issue:
DownloadFile intrinsic doesn't verify cached expected digest against URL
**Describe the bug**
Before downloading, the `load_or_download` Rust function checks if the expected digest exists in cache, and if it does, returns it without verifying that the requested URL actually contains that content. The result is that if you specify a URL+expected digest pair, where the expected digest is of some other content that is already cached (and this is an easy mistake to make), then you will get that content, even though it is *NOT* the content at the given URL!

**Pants version**
2.7

**OS**
Both

**Additional info**
Here is one way an end user may hit this problem: When setting `known_versions` in an `ExternalTool`, you add a new entry (say for a new version, or a new platform) but accidentally keep the expected sha256+length of an existing entry. 



Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/internals/scheduler.py b/src/python/pants/engine/internals/scheduler.py
--- a/src/python/pants/engine/internals/scheduler.py
+++ b/src/python/pants/engine/internals/scheduler.py
@@ -281,11 +281,14 @@ def invalidate_files(self, direct_filenames: Iterable[str]) -> int:
         filenames = set(direct_filenames)
         # TODO(#11707): Evaluate removing the invalidation of parent directories.
         filenames.update(os.path.dirname(f) for f in direct_filenames)
-        return native_engine.graph_invalidate(self.py_scheduler, tuple(filenames))
+        return native_engine.graph_invalidate_paths(self.py_scheduler, tuple(filenames))
 
     def invalidate_all_files(self) -> int:
         return native_engine.graph_invalidate_all_paths(self.py_scheduler)
 
+    def invalidate_all(self) -> None:
+        native_engine.graph_invalidate_all(self.py_scheduler)
+
     def check_invalidation_watcher_liveness(self) -> None:
         native_engine.check_invalidation_watcher_liveness(self.py_scheduler)
 


Test patch:
diff --git a/src/python/pants/engine/fs_test.py b/src/python/pants/engine/fs_test.py
--- a/src/python/pants/engine/fs_test.py
+++ b/src/python/pants/engine/fs_test.py
@@ -66,6 +66,9 @@ def rule_runner() -> RuleRunner:
     )
 
 
+ROLAND_FILE_DIGEST = FileDigest(
+    "693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16
+)
 ROLAND_DIGEST = Digest("63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16", 80)
 
 
@@ -828,6 +831,9 @@ def send_headers(self):
 DOWNLOADS_EXPECTED_DIRECTORY_DIGEST = Digest(
     "4c9cf91fcd7ba1abbf7f9a0a1c8175556a82bee6a398e34db3284525ac24a3ad", 84
 )
+ROLAND_DOWNLOAD_DIGEST = Digest(
+    "9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82
+)
 
 
 def test_download_valid(downloads_rule_runner: RuleRunner) -> None:
@@ -860,20 +866,34 @@ def test_download_wrong_digest(downloads_rule_runner: RuleRunner) -> None:
     assert "wrong digest" in str(exc.value).lower()
 
 
+def test_download_file(downloads_rule_runner: RuleRunner) -> None:
+    with temporary_dir() as temp_dir:
+        roland = Path(temp_dir, "roland")
+        roland.write_text("European Burmese")
+        snapshot = downloads_rule_runner.request(
+            Snapshot,
+            [DownloadFile(f"file://ignored.see.13054/{roland}", ROLAND_FILE_DIGEST)],
+        )
+
+    assert snapshot.files == ("roland",)
+    assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
+
+
 def test_download_caches(downloads_rule_runner: RuleRunner) -> None:
-    # We would error if we hit the HTTP server with 404, but we're not going to hit the HTTP
-    # server because it's cached, so we shouldn't see an error.
+    # We put the expected content in the store, but because we have never fetched it from this
+    # URL, we confirm the URL and attempt to refetch. Once it is cached, it does not need to be
+    # refetched.
     prime_store_with_roland_digest(downloads_rule_runner)
-    with http_server(StubHandler) as port:
-        download_file = DownloadFile(
-            f"http://localhost:{port}/roland",
-            FileDigest("693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16),
+    with temporary_dir() as temp_dir:
+        roland = Path(temp_dir, "roland")
+        roland.write_text("European Burmese")
+        snapshot = downloads_rule_runner.request(
+            Snapshot,
+            [DownloadFile(f"file://ignored.see.13054/{roland}", ROLAND_FILE_DIGEST)],
         )
-        snapshot = downloads_rule_runner.request(Snapshot, [download_file])
+
     assert snapshot.files == ("roland",)
-    assert snapshot.digest == Digest(
-        "9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82
-    )
+    assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
 
 
 def test_download_https() -> None:
diff --git a/src/rust/engine/fs/store/src/snapshot_ops_tests.rs b/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
--- a/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
+++ b/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
@@ -4,10 +4,10 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use bazel_protos::gen::build::bazel::remote::execution::v2 as remexec;
 use fs::{GlobExpansionConjunction, PosixFS, PreparedPathGlobs, StrictGlobMatching};
 use hashing::Digest;
 use parking_lot::Mutex;
+use protos::gen::build::bazel::remote::execution::v2 as remexec;
 use testutil::make_file;
 
 use crate::{
diff --git a/src/rust/engine/fs/store/src/tests.rs b/src/rust/engine/fs/store/src/tests.rs
--- a/src/rust/engine/fs/store/src/tests.rs
+++ b/src/rust/engine/fs/store/src/tests.rs
@@ -7,13 +7,13 @@ use std::time::Duration;
 use tempfile::TempDir;
 use testutil::data::{TestData, TestDirectory};
 
-use bazel_protos::gen::build::bazel::remote::execution::v2 as remexec;
 use bytes::{Bytes, BytesMut};
 use fs::{DigestEntry, FileEntry};
 use grpc_util::prost::MessageExt;
 use grpc_util::tls;
 use hashing::{Digest, Fingerprint, EMPTY_DIGEST};
 use mock::StubCAS;
+use protos::gen::build::bazel::remote::execution::v2 as remexec;
 
 use crate::{EntryType, FileContent, Store, UploadSummary, MEGABYTES};
 
diff --git a/src/rust/engine/process_execution/src/cache_tests.rs b/src/rust/engine/process_execution/src/cache_tests.rs
--- a/src/rust/engine/process_execution/src/cache_tests.rs
+++ b/src/rust/engine/process_execution/src/cache_tests.rs
@@ -2,7 +2,8 @@ use std::convert::TryInto;
 use std::io::Write;
 use std::path::PathBuf;
 
-use sharded_lmdb::{ShardedLmdb, DEFAULT_LEASE_TIME};
+use cache::PersistentCache;
+use sharded_lmdb::DEFAULT_LEASE_TIME;
 use store::Store;
 use tempfile::TempDir;
 use testutil::data::TestData;
@@ -43,8 +44,8 @@ fn create_cached_runner(
   let cache_dir = TempDir::new().unwrap();
   let max_lmdb_size = 50 * 1024 * 1024; //50 MB - I didn't pick that number but it seems reasonable.
 
-  let process_execution_store = ShardedLmdb::new(
-    cache_dir.path().to_owned(),
+  let cache = PersistentCache::new(
+    cache_dir.path(),
     max_lmdb_size,
     runtime.clone(),
     DEFAULT_LEASE_TIME,
@@ -54,7 +55,7 @@ fn create_cached_runner(
 
   let runner = Box::new(crate::cache::CommandRunner::new(
     local.into(),
-    process_execution_store,
+    cache,
     store,
     ProcessMetadata::default(),
   ));
diff --git a/src/rust/engine/process_execution/src/remote_cache_tests.rs b/src/rust/engine/process_execution/src/remote_cache_tests.rs
--- a/src/rust/engine/process_execution/src/remote_cache_tests.rs
+++ b/src/rust/engine/process_execution/src/remote_cache_tests.rs
@@ -5,12 +5,12 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
-use bazel_protos::gen::build::bazel::remote::execution::v2 as remexec;
 use fs::RelativePath;
 use grpc_util::tls;
 use hashing::{Digest, EMPTY_DIGEST};
 use maplit::hashset;
 use mock::{StubActionCache, StubCAS};
+use protos::gen::build::bazel::remote::execution::v2 as remexec;
 use remexec::ActionResult;
 use store::Store;
 use tempfile::TempDir;
diff --git a/src/rust/engine/process_execution/src/remote_tests.rs b/src/rust/engine/process_execution/src/remote_tests.rs
--- a/src/rust/engine/process_execution/src/remote_tests.rs
+++ b/src/rust/engine/process_execution/src/remote_tests.rs
@@ -3,8 +3,6 @@ use std::convert::TryInto;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
-use bazel_protos::gen::build::bazel::remote::execution::v2 as remexec;
-use bazel_protos::gen::google::longrunning::Operation;
 use bytes::Bytes;
 use grpc_util::prost::MessageExt;
 use grpc_util::tls;
@@ -12,6 +10,8 @@ use hashing::{Digest, Fingerprint, EMPTY_DIGEST};
 use maplit::{btreemap, hashset};
 use mock::execution_server::{ExpectedAPICall, MockOperation};
 use prost::Message;
+use protos::gen::build::bazel::remote::execution::v2 as remexec;
+use protos::gen::google::longrunning::Operation;
 use remexec::ExecutedActionMetadata;
 use spectral::prelude::*;
 use spectral::{assert_that, string::StrAssertions};
@@ -1147,13 +1147,11 @@ async fn bad_result_bytes() {
             name: op_name.clone(),
             done: true,
             result: Some(
-              bazel_protos::gen::google::longrunning::operation::Result::Response(
-                prost_types::Any {
-                  type_url: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteResponse"
-                    .into(),
-                  value: vec![0x00, 0x00, 0x00],
-                },
-              ),
+              protos::gen::google::longrunning::operation::Result::Response(prost_types::Any {
+                type_url: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteResponse"
+                  .into(),
+                value: vec![0x00, 0x00, 0x00],
+              }),
             ),
             ..Default::default()
           }),
@@ -1196,15 +1194,13 @@ async fn initial_response_error() {
             Operation {
               name: op_name.to_string(),
               done: true,
-              result: Some(
-                bazel_protos::gen::google::longrunning::operation::Result::Error(
-                  bazel_protos::gen::google::rpc::Status {
-                    code: Code::Internal as i32,
-                    message: "Something went wrong".to_string(),
-                    ..Default::default()
-                  },
-                ),
-              ),
+              result: Some(protos::gen::google::longrunning::operation::Result::Error(
+                protos::gen::google::rpc::Status {
+                  code: Code::Internal as i32,
+                  message: "Something went wrong".to_string(),
+                  ..Default::default()
+                },
+              )),
               ..Default::default()
             }
           })]),
@@ -1572,7 +1568,7 @@ async fn extract_execute_response_success() {
     name: "cat".to_owned(),
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
           result: Some(remexec::ActionResult {
             exit_code: wanted_exit_code,
@@ -1588,7 +1584,7 @@ async fn extract_execute_response_success() {
           }),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -1614,15 +1610,15 @@ async fn extract_execute_response_timeout() {
     name: "cat".to_owned(),
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
-          status: Some(bazel_protos::gen::google::rpc::Status {
+          status: Some(protos::gen::google::rpc::Status {
             code: Code::DeadlineExceeded as i32,
             ..Default::default()
           }),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -1661,7 +1657,7 @@ async fn extract_execute_response_missing_digests() {
 async fn extract_execute_response_missing_other_things() {
   let missing = vec![
     missing_preconditionfailure_violation(&TestData::roland().digest()),
-    bazel_protos::gen::google::rpc::precondition_failure::Violation {
+    protos::gen::google::rpc::precondition_failure::Violation {
       r#type: "MISSING".to_owned(),
       subject: "monkeys".to_owned(),
       ..Default::default()
@@ -1681,12 +1677,10 @@ async fn extract_execute_response_missing_other_things() {
 
 #[tokio::test]
 async fn extract_execute_response_other_failed_precondition() {
-  let missing = vec![
-    bazel_protos::gen::google::rpc::precondition_failure::Violation {
-      r#type: "OUT_OF_CAPACITY".to_owned(),
-      ..Default::default()
-    },
-  ];
+  let missing = vec![protos::gen::google::rpc::precondition_failure::Violation {
+    r#type: "OUT_OF_CAPACITY".to_owned(),
+    ..Default::default()
+  }];
 
   let operation = make_precondition_failure_operation(missing)
     .op
@@ -1720,15 +1714,15 @@ async fn extract_execute_response_other_status() {
     name: "cat".to_owned(),
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
-          status: Some(bazel_protos::gen::google::rpc::Status {
+          status: Some(protos::gen::google::rpc::Status {
             code: Code::PermissionDenied as i32,
             ..Default::default()
           }),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -1785,7 +1779,7 @@ async fn remote_workunits_are_stored() {
 
 #[tokio::test]
 async fn format_error_complete() {
-  let error = bazel_protos::gen::google::rpc::Status {
+  let error = protos::gen::google::rpc::Status {
     code: Code::Cancelled as i32,
     message: "Oops, oh well!".to_string(),
     ..Default::default()
@@ -1799,7 +1793,7 @@ async fn format_error_complete() {
 
 #[tokio::test]
 async fn extract_execute_response_unknown_code() {
-  let error = bazel_protos::gen::google::rpc::Status {
+  let error = protos::gen::google::rpc::Status {
     code: 555,
     message: "Oops, oh well!".to_string(),
     ..Default::default()
@@ -2019,7 +2013,7 @@ pub(crate) fn make_incomplete_operation(operation_name: &str) -> MockOperation {
 }
 
 pub(crate) fn make_retryable_operation_failure() -> MockOperation {
-  let status = bazel_protos::gen::google::rpc::Status {
+  let status = protos::gen::google::rpc::Status {
     code: Code::Aborted as i32,
     message: String::from("the bot running the task appears to be lost"),
     ..Default::default()
@@ -2028,12 +2022,12 @@ pub(crate) fn make_retryable_operation_failure() -> MockOperation {
   let operation = Operation {
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
           status: Some(status),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -2086,16 +2080,16 @@ fn make_successful_operation_with_maybe_metadata(
     name: operation_name.to_string(),
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
-          status: Some(bazel_protos::gen::google::rpc::Status {
+          status: Some(protos::gen::google::rpc::Status {
             code: Code::Ok as i32,
             ..Default::default()
           }),
           result: Some(make_action_result(stdout, stderr, exit_code, metadata)),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -2149,18 +2143,18 @@ fn timestamp_only_secs(v: i64) -> prost_types::Timestamp {
 }
 
 pub(crate) fn make_precondition_failure_operation(
-  violations: Vec<bazel_protos::gen::google::rpc::precondition_failure::Violation>,
+  violations: Vec<protos::gen::google::rpc::precondition_failure::Violation>,
 ) -> MockOperation {
   let operation = Operation {
     name: "cat".to_owned(),
     done: true,
     result: Some(
-      bazel_protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
+      protos::gen::google::longrunning::operation::Result::Response(make_any_proto(
         &remexec::ExecuteResponse {
           status: Some(make_precondition_failure_status(violations)),
           ..Default::default()
         },
-        "bazel_protos::gen::",
+        "protos::gen::",
       )),
     ),
     ..Default::default()
@@ -2169,13 +2163,13 @@ pub(crate) fn make_precondition_failure_operation(
 }
 
 fn make_precondition_failure_status(
-  violations: Vec<bazel_protos::gen::google::rpc::precondition_failure::Violation>,
-) -> bazel_protos::gen::google::rpc::Status {
-  bazel_protos::gen::google::rpc::Status {
+  violations: Vec<protos::gen::google::rpc::precondition_failure::Violation>,
+) -> protos::gen::google::rpc::Status {
+  protos::gen::google::rpc::Status {
     code: Code::FailedPrecondition as i32,
     details: vec![make_any_proto(
-      &bazel_protos::gen::google::rpc::PreconditionFailure { violations },
-      "bazel_protos::gen::",
+      &protos::gen::google::rpc::PreconditionFailure { violations },
+      "protos::gen::",
     )],
     ..Default::default()
   }
@@ -2353,9 +2347,9 @@ pub(crate) fn make_any_proto<T: Message>(message: &T, prefix: &str) -> prost_typ
 
 pub(crate) fn missing_preconditionfailure_violation(
   digest: &Digest,
-) -> bazel_protos::gen::google::rpc::precondition_failure::Violation {
+) -> protos::gen::google::rpc::precondition_failure::Violation {
   {
-    bazel_protos::gen::google::rpc::precondition_failure::Violation {
+    protos::gen::google::rpc::precondition_failure::Violation {
       r#type: "MISSING".to_owned(),
       subject: format!("blobs/{}/{}", digest.hash, digest.size_bytes),
       ..Default::default()
diff --git a/src/rust/engine/process_execution/src/tests.rs b/src/rust/engine/process_execution/src/tests.rs
--- a/src/rust/engine/process_execution/src/tests.rs
+++ b/src/rust/engine/process_execution/src/tests.rs
@@ -6,8 +6,8 @@ use std::hash::{Hash, Hasher};
 use std::time::Duration;
 
 use crate::{Process, ProcessResultMetadata, ProcessResultSource};
-use bazel_protos::gen::build::bazel::remote::execution::v2 as remexec;
 use prost_types::Timestamp;
+use protos::gen::build::bazel::remote::execution::v2 as remexec;
 use remexec::ExecutedActionMetadata;
 
 #[test]
diff --git a/src/rust/engine/bazel_protos/src/conversions_tests.rs b/src/rust/engine/protos/src/conversions_tests.rs
similarity index 100%
rename from src/rust/engine/bazel_protos/src/conversions_tests.rs
rename to src/rust/engine/protos/src/conversions_tests.rs
diff --git a/src/rust/engine/bazel_protos/src/verification_tests.rs b/src/rust/engine/protos/src/verification_tests.rs
similarity index 100%
rename from src/rust/engine/bazel_protos/src/verification_tests.rs
rename to src/rust/engine/protos/src/verification_tests.rs