Repository: pantsbuild/pants

Issue:
Add an intrinsic to create empty directories
Use case: creating an empty output directory called `_generate_files` when running Protoc.

An ideal API would allow you to both create files and directories in the same `await Get`. We'd use `FileContent` for files, and maybe something new like `Directory`. Something like:

```
await Get[Digest](DigestRequest([FileContent(..), Directory(..), FileContent(..)])
```

I don't think that we should be too wedded to `InputFilesContents`, which only works with `FileContent`.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/codegen/protobuf/python/rules.py b/src/python/pants/backend/codegen/protobuf/python/rules.py
--- a/src/python/pants/backend/codegen/protobuf/python/rules.py
+++ b/src/python/pants/backend/codegen/protobuf/python/rules.py
@@ -10,7 +10,15 @@
 from pants.core.util_rules.source_files import SourceFilesRequest
 from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
 from pants.engine.addresses import Addresses
-from pants.engine.fs import AddPrefix, Digest, MergeDigests, RemovePrefix, Snapshot
+from pants.engine.fs import (
+    AddPrefix,
+    CreateDigest,
+    Digest,
+    Directory,
+    MergeDigests,
+    RemovePrefix,
+    Snapshot,
+)
 from pants.engine.platform import Platform
 from pants.engine.process import Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
@@ -34,16 +42,7 @@ async def generate_python_from_protobuf(
     )
 
     output_dir = "_generated_files"
-    # TODO(#9650): replace this with a proper intrinsic to create empty directories.
-    create_output_dir_request = Get(
-        ProcessResult,
-        Process(
-            ("/bin/mkdir", output_dir),
-            description=f"Create the directory {output_dir}",
-            level=LogLevel.TRACE,
-            output_directories=(output_dir,),
-        ),
-    )
+    create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)]))
 
     # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
     # actually generate those dependencies; it only needs to look at their .proto files to work
@@ -64,7 +63,7 @@ async def generate_python_from_protobuf(
 
     (
         downloaded_protoc_binary,
-        create_output_dir_result,
+        empty_output_dir,
         all_sources_stripped,
         target_sources_stripped,
     ) = await MultiGet(
@@ -80,7 +79,7 @@ async def generate_python_from_protobuf(
             (
                 all_sources_stripped.snapshot.digest,
                 downloaded_protoc_binary.digest,
-                create_output_dir_result.output_digest,
+                empty_output_dir,
             )
         ),
     )
diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py
--- a/src/python/pants/backend/python/util_rules/pex_cli.py
+++ b/src/python/pants/backend/python/util_rules/pex_cli.py
@@ -14,7 +14,7 @@
     ExternalTool,
     ExternalToolRequest,
 )
-from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
+from pants.engine.fs import CreateDigest, Digest, Directory, MergeDigests
 from pants.engine.internals.selectors import MultiGet
 from pants.engine.platform import Platform
 from pants.engine.process import Process
@@ -101,9 +101,7 @@ async def setup_pex_cli_process(
     tmpdir = ".tmp"
     downloaded_pex_bin, tmp_dir_digest = await MultiGet(
         Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)),
-        # TODO(John Sirois): Use a Directory instead of this FileContent hack when a fix for
-        #  https://github.com/pantsbuild/pants/issues/9650 lands.
-        Get(Digest, CreateDigest([FileContent(f"{tmpdir}/.reserve", b"")])),
+        Get(Digest, CreateDigest([Directory(f"{tmpdir}/.reserve")])),
     )
 
     digests_to_merge = [downloaded_pex_bin.digest, tmp_dir_digest]
diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py
--- a/src/python/pants/engine/fs.py
+++ b/src/python/pants/engine/fs.py
@@ -3,7 +3,7 @@
 
 from dataclasses import dataclass
 from enum import Enum
-from typing import TYPE_CHECKING, Iterable, Optional, Tuple
+from typing import TYPE_CHECKING, Iterable, Optional, Tuple, Union
 
 from pants.engine.collection import Collection
 from pants.engine.rules import QueryRule, side_effecting
@@ -53,7 +53,11 @@ class Paths:
 
 @dataclass(frozen=True)
 class FileContent:
-    """The content of a file."""
+    """The content of a file.
+
+    This can be used to create a new Digest with `Get(Digest, CreateDigest)`. You can also get back
+    a list of `FileContent` objects by using `Get(DigestContents, Digest)`.
+    """
 
     path: str
     content: bytes
@@ -66,12 +70,29 @@ def __repr__(self) -> str:
         )
 
 
+@dataclass(frozen=True)
+class Directory:
+    """The path to a directory.
+
+    This can be used to create empty directories with `Get(Digest, CreateDigest)`.
+    """
+
+    path: str
+
+    def __repr__(self) -> str:
+        return f"Directory({repr(self.path)})"
+
+
 class DigestContents(Collection[FileContent]):
     """The file contents of a Digest."""
 
 
-class CreateDigest(Collection[FileContent]):
-    """A request to create a Digest with the input FileContent values.
+class CreateDigest(Collection[Union[FileContent, Directory]]):
+    """A request to create a Digest with the input FileContent and/or Directory values.
+
+    The engine will create any parent directories necessary, e.g. `FileContent('a/b/c.txt')` will
+    result in `a/`, `a/b`, and `a/b/c.txt` being created. You only need to use `Directory` to
+    create an empty directory.
 
     This does _not_ actually materialize the digest to the build root. You must use
     `engine.fs.Workspace` in a `@goal_rule` to save the resulting digest to disk.


Test patch:
diff --git a/src/python/pants/backend/python/goals/coverage_py_test.py b/src/python/pants/backend/python/goals/coverage_py_test.py
--- a/src/python/pants/backend/python/goals/coverage_py_test.py
+++ b/src/python/pants/backend/python/goals/coverage_py_test.py
@@ -19,6 +19,7 @@ def run_create_coverage_config_rule(coverage_config: Optional[str]) -> str:
     def mock_handle_config(request: CreateDigest) -> Digest:
         assert len(request) == 1
         assert request[0].path == ".coveragerc"
+        assert isinstance(request[0], FileContent)
         assert request[0].is_executable is False
         resolved_config.append(request[0].content.decode())
         return Digest("jerry", 30)
diff --git a/src/python/pants/engine/fs_test.py b/src/python/pants/engine/fs_test.py
--- a/src/python/pants/engine/fs_test.py
+++ b/src/python/pants/engine/fs_test.py
@@ -26,6 +26,7 @@
     Digest,
     DigestContents,
     DigestSubset,
+    Directory,
     DownloadFile,
     FileContent,
     GlobMatchErrorBehavior,
@@ -72,6 +73,7 @@ class FSTest(FSTestBase):
     def rules(cls):
         return (
             *super().rules(),
+            QueryRule(Snapshot, (CreateDigest,)),
             QueryRule(Snapshot, (DigestSubset,)),
         )
 
@@ -421,9 +423,7 @@ def test_add_prefix(self) -> None:
         assert digest == output_digest
 
         # Illegal.
-        with self.assertRaisesRegex(
-            Exception, r"Cannot add component .*ParentDir.* of path prefix `../something`."
-        ):
+        with self.assertRaisesRegex(Exception, r"The `prefix` must be relative."):
             self.request(Digest, [AddPrefix(digest, "../something")])
 
     def test_remove_prefix(self) -> None:
@@ -502,6 +502,19 @@ def test_remove_prefix(self) -> None:
                 "directory contained non-matching directory named: books and file named: index"
             ) in str(exc.value)
 
+    def test_create_empty_directory(self) -> None:
+        res = self.request(Snapshot, [CreateDigest([Directory("a/")])])
+        assert res.dirs == ("a",)
+        assert not res.files
+        assert res.digest != EMPTY_DIGEST
+
+        res = self.request(
+            Snapshot, [CreateDigest([Directory("x/y/z"), Directory("m"), Directory("m/n")])]
+        )
+        assert res.dirs == ("m", "m/n", "x", "x/y", "x/y/z")
+        assert not res.files
+        assert res.digest != EMPTY_DIGEST
+
     def test_lift_digest_to_snapshot(self) -> None:
         digest = self.prime_store_with_roland_digest()
         snapshot = self.request(Snapshot, [digest])
diff --git a/src/rust/engine/fs/store/src/snapshot_ops_tests.rs b/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
--- a/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
+++ b/src/rust/engine/fs/store/src/snapshot_ops_tests.rs
@@ -5,7 +5,7 @@ use testutil::make_file;
 use crate::{
   snapshot_ops::StoreWrapper,
   snapshot_tests::{expand_all_sorted, setup, STR, STR2},
-  OneOffStoreFileByDigest, Snapshot, SnapshotOps, Store, SubsetParams,
+  OneOffStoreFileByDigest, RelativePath, Snapshot, SnapshotOps, Store, SubsetParams,
 };
 use bazel_protos::remote_execution as remexec;
 use fs::{GlobExpansionConjunction, PosixFS, PreparedPathGlobs, StrictGlobMatching};
@@ -187,8 +187,9 @@ async fn subset_tracking_load_counts() {
   )
   .await;
 
+  let prefix = RelativePath::new(PathBuf::from("subdir")).unwrap();
   let subdir_digest = load_tracking_store
-    .strip_prefix(merged_digest, PathBuf::from("subdir"))
+    .strip_prefix(merged_digest, prefix)
     .await
     .unwrap();
 
diff --git a/src/rust/engine/fs/store/src/snapshot_tests.rs b/src/rust/engine/fs/store/src/snapshot_tests.rs
--- a/src/rust/engine/fs/store/src/snapshot_tests.rs
+++ b/src/rust/engine/fs/store/src/snapshot_tests.rs
@@ -9,7 +9,7 @@ use testutil::data::TestDirectory;
 use testutil::make_file;
 use tokio::runtime::Handle;
 
-use crate::{OneOffStoreFileByDigest, Snapshot, SnapshotOps, Store};
+use crate::{OneOffStoreFileByDigest, RelativePath, Snapshot, SnapshotOps, Store};
 use fs::{
   Dir, File, GitignoreStyleExcludes, GlobExpansionConjunction, GlobMatching, PathGlobs, PathStat,
   PosixFS, StrictGlobMatching,
@@ -369,7 +369,8 @@ async fn strip_empty_prefix() {
     .await
     .expect("Error storing directory");
 
-  let result = store.strip_prefix(dir.digest(), PathBuf::from("")).await;
+  let prefix = RelativePath::new(PathBuf::from("")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
   assert_eq!(result, Ok(dir.digest()));
 }
 
@@ -387,9 +388,8 @@ async fn strip_non_empty_prefix() {
     .await
     .expect("Error storing directory");
 
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("cats"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("cats")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
   assert_eq!(result, Ok(TestDirectory::containing_roland().digest()));
 }
 
@@ -403,9 +403,8 @@ async fn strip_prefix_empty_subdir() {
     .await
     .expect("Error storing directory");
 
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("falcons/peregrine"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("falcons/peregrine")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
   assert_eq!(result, Ok(TestDirectory::empty().digest()));
 }
 
@@ -413,7 +412,8 @@ async fn strip_prefix_empty_subdir() {
 async fn strip_dir_not_in_store() {
   let (store, _, _, _) = setup();
   let digest = TestDirectory::nested().digest();
-  let result = store.strip_prefix(digest, PathBuf::from("cats")).await;
+  let prefix = RelativePath::new(PathBuf::from("cats")).unwrap();
+  let result = store.strip_prefix(digest, prefix).await;
   assert_eq!(result, Err(format!("{:?} was not known", digest).into()));
 }
 
@@ -425,9 +425,8 @@ async fn strip_subdir_not_in_store() {
     .record_directory(&dir.directory(), false)
     .await
     .expect("Error storing directory");
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("cats"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("cats")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
   assert_eq!(
     result,
     Err(
@@ -453,9 +452,8 @@ async fn strip_prefix_non_matching_file() {
     .record_directory(&child_dir.directory(), false)
     .await
     .expect("Error storing directory");
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("cats"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("cats")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
 
   assert_eq!(result, Err(format!("Cannot strip prefix cats from root directory {:?} - root directory contained non-matching file named: treats", dir.digest()).into()));
 }
@@ -473,9 +471,8 @@ async fn strip_prefix_non_matching_dir() {
     .record_directory(&child_dir.directory(), false)
     .await
     .expect("Error storing directory");
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("animals/cats"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("animals/cats")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
 
   assert_eq!(result, Err(format!("Cannot strip prefix animals/cats from root directory {:?} - subdirectory animals contained non-matching directory named: birds", dir.digest()).into()));
 }
@@ -492,9 +489,8 @@ async fn strip_subdir_not_in_dir() {
     .record_directory(&TestDirectory::containing_roland().directory(), false)
     .await
     .expect("Error storing directory");
-  let result = store
-    .strip_prefix(dir.digest(), PathBuf::from("cats/ugly"))
-    .await;
+  let prefix = RelativePath::new(PathBuf::from("cats/ugly")).unwrap();
+  let result = store.strip_prefix(dir.digest(), prefix).await;
   assert_eq!(result, Err(format!("Cannot strip prefix cats/ugly from root directory {:?} - subdirectory cats didn't contain a directory named ugly but did contain file named: roland", dir.digest()).into()));
 }