Repository: pantsbuild/pants

Issue:
Mixed use of Digest type can be confusing
Post #10698, the Python `workunit.artifacts` list will contain a mix of `Snapshots` and `Digests`. But this is confusing, because part of the goal of #10698 (I thought) was to ensure that a `Digest` in Python _always_ represented a `Directory`.

On the rust side, a `Digest` has multiple meanings: it might be a `Directory`, a `Tree`, file content, anything. But on the Python side (so-far) it has always meant "directory" or `Snapshot` (a collection of files). When that is the case "directory", `Snapshot`, and `Digest` are synonyms on the Python side... there would not be a reason to try to differentiate `Snapshots` and `Digests`. The type "directory" is not actually exposed to Python: it always comes across as a `Snapshot` or `Digest`.

My feeling is that we _must_ be consistent, and not try to give `Digest` multiple meanings on the Python side. If the `Digest` that is being returned in the `workunit.artifacts` is truly supposed to represent a single file, then it should be given a new type, period.


Discussion:

Resolving patch:
diff --git a/src/python/pants/core/util_rules/external_tool.py b/src/python/pants/core/util_rules/external_tool.py
--- a/src/python/pants/core/util_rules/external_tool.py
+++ b/src/python/pants/core/util_rules/external_tool.py
@@ -8,7 +8,7 @@
 
 from pants.core.util_rules import archive
 from pants.core.util_rules.archive import ExtractedArchive
-from pants.engine.fs import Digest, DownloadFile
+from pants.engine.fs import Digest, DownloadFile, FileDigest
 from pants.engine.platform import Platform
 from pants.engine.rules import Get, collect_rules, rule
 from pants.option.subsystem import Subsystem
@@ -154,7 +154,7 @@ def get_request(self, plat: Platform) -> ExternalToolRequest:
                     f"help-advanced {self.options_scope}): {known_version}"
                 )
             if plat_val == plat.value and ver == self.version:
-                digest = Digest(fingerprint=sha256, serialized_bytes_length=int(length))
+                digest = FileDigest(fingerprint=sha256, serialized_bytes_length=int(length))
                 try:
                     url = self.generate_url(plat)
                     exe = self.generate_exe(plat)
diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py
--- a/src/python/pants/engine/fs.py
+++ b/src/python/pants/engine/fs.py
@@ -51,6 +51,14 @@ class Paths:
     dirs: Tuple[str, ...]
 
 
+@dataclass(frozen=True)
+class FileDigest:
+    """A FileDigest is a digest that refers to a file's content, without its name."""
+
+    fingerprint: str
+    serialized_bytes_length: int
+
+
 @dataclass(frozen=True)
 class FileContent:
     """The content of a file.
@@ -251,7 +259,7 @@ class DownloadFile:
     """
 
     url: str
-    expected_digest: Digest
+    expected_digest: FileDigest
 
 
 @side_effecting
@@ -272,6 +280,7 @@ def write_digest(self, digest: Digest, *, path_prefix: Optional[str] = None) ->
 
 _EMPTY_FINGERPRINT = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 EMPTY_DIGEST = Digest(fingerprint=_EMPTY_FINGERPRINT, serialized_bytes_length=0)
+EMPTY_FILE_DIGEST = FileDigest(fingerprint=_EMPTY_FINGERPRINT, serialized_bytes_length=0)
 EMPTY_SNAPSHOT = Snapshot(EMPTY_DIGEST, files=(), dirs=())
 
 
diff --git a/src/python/pants/engine/internals/scheduler.py b/src/python/pants/engine/internals/scheduler.py
--- a/src/python/pants/engine/internals/scheduler.py
+++ b/src/python/pants/engine/internals/scheduler.py
@@ -22,6 +22,7 @@
     DigestSubset,
     DownloadFile,
     FileContent,
+    FileDigest,
     MergeDigests,
     PathGlobs,
     PathGlobsAndRoot,
@@ -130,6 +131,7 @@ def __init__(
 
         types = PyTypes(
             directory_digest=Digest,
+            file_digest=FileDigest,
             snapshot=Snapshot,
             paths=Paths,
             file_content=FileContent,


Test patch:
diff --git a/src/python/pants/core/util_rules/external_tool_test.py b/src/python/pants/core/util_rules/external_tool_test.py
--- a/src/python/pants/core/util_rules/external_tool_test.py
+++ b/src/python/pants/core/util_rules/external_tool_test.py
@@ -9,7 +9,7 @@
     ExternalToolRequest,
     UnknownVersion,
 )
-from pants.engine.fs import Digest, DownloadFile
+from pants.engine.fs import DownloadFile, FileDigest
 from pants.engine.platform import Platform
 from pants.testutil.option_util import create_subsystem
 
@@ -49,7 +49,7 @@ def do_test(
         assert (
             ExternalToolRequest(
                 DownloadFile(
-                    url=expected_url, expected_digest=Digest(expected_sha256, expected_length)
+                    url=expected_url, expected_digest=FileDigest(expected_sha256, expected_length)
                 ),
                 f"foobar-{version}/bin/foobar",
             )
diff --git a/src/python/pants/engine/fs_test.py b/src/python/pants/engine/fs_test.py
--- a/src/python/pants/engine/fs_test.py
+++ b/src/python/pants/engine/fs_test.py
@@ -31,6 +31,7 @@
     Directory,
     DownloadFile,
     FileContent,
+    FileDigest,
     GlobMatchErrorBehavior,
     MergeDigests,
     PathGlobs,
@@ -797,7 +798,7 @@ def mutation_function(project_tree, dir_path):
 
 
 class DownloadsTest(FSTestBase):
-    file_digest = Digest("8fcbc50cda241aee7238c71e87c27804e7abc60675974eaf6567aa16366bc105", 14)
+    file_digest = FileDigest("8fcbc50cda241aee7238c71e87c27804e7abc60675974eaf6567aa16366bc105", 14)
 
     expected_snapshot_digest = Digest(
         "4c9cf91fcd7ba1abbf7f9a0a1c8175556a82bee6a398e34db3284525ac24a3ad", 84
@@ -842,7 +843,7 @@ def test_download_wrong_digest(self) -> None:
                         [
                             DownloadFile(
                                 f"http://localhost:{port}/file.txt",
-                                Digest(
+                                FileDigest(
                                     self.file_digest.fingerprint,
                                     self.file_digest.serialized_bytes_length + 1,
                                 ),
@@ -861,7 +862,9 @@ def test_caches_downloads(self) -> None:
                 # so we shouldn't see an error...
                 url = DownloadFile(
                     f"http://localhost:{port}/roland",
-                    Digest("693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16),
+                    FileDigest(
+                        "693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16
+                    ),
                 )
                 snapshot = self.request(Snapshot, [url])
                 self.assert_snapshot_equals(
diff --git a/src/python/pants/engine/internals/engine_test.py b/src/python/pants/engine/internals/engine_test.py
--- a/src/python/pants/engine/internals/engine_test.py
+++ b/src/python/pants/engine/internals/engine_test.py
@@ -10,7 +10,7 @@
 
 from pants.engine.engine_aware import EngineAwareReturnType
 from pants.engine.fs import (
-    EMPTY_DIGEST,
+    EMPTY_FILE_DIGEST,
     EMPTY_SNAPSHOT,
     CreateDigest,
     Digest,
@@ -692,7 +692,7 @@ def test_process_digests_on_workunits(self):
         stderr_digest = process_workunit["artifacts"]["stderr_digest"]
 
         assert result.stdout == b"stdout output\n"
-        assert stderr_digest == EMPTY_DIGEST
+        assert stderr_digest == EMPTY_FILE_DIGEST
         assert stdout_digest.serialized_bytes_length == len(result.stdout)
 
         tracker = WorkunitTracker()
@@ -721,7 +721,7 @@ def test_process_digests_on_workunits(self):
         stderr_digest = process_workunit["artifacts"]["stderr_digest"]
 
         assert result.stderr == b"stderr output\n"
-        assert stdout_digest == EMPTY_DIGEST
+        assert stdout_digest == EMPTY_FILE_DIGEST
         assert stderr_digest.serialized_bytes_length == len(result.stderr)
 
         try: