Repository: pantsbuild/pants

Issue:
Add a V2 builtin rule to go from FilesContent -> Snapshot
We would like a nice abstraction to create a `Snapshot` given a set of file paths with their corresponding content (i.e. `FilesContent`).

This would be far better than the current approach of using `touch` to do the same thing, as done in https://github.com/pantsbuild/pants/pull/7696. Notably, the current implementation only works for empty files too.

However, going from `FilesContent` -> `Snapshot` would likely lead to a cycle so we might need a different name than this. Refer to https://github.com/pantsbuild/pants/pull/7696#discussion_r283733641.


Discussion:
@stuhood Any thoughts on what this API should look like? Ideally in my mind it would be:
```python
files_content = FilesContent((
  FileContent(path="roland", content=b"European Burmese"),
  FileContent(path="susannah", content=b"Not sure actually"),
))
snapshot = yield Get(Snapshot, FilesContent, files_content)
```

but that would lead to graph ambiguity through `Snapshot -> FilesContent -> Snapshot` cycles. I guess we could do something like:
```python
files_content = FilesToSnapshot((
  FileContent(path="roland", content=b"European Burmese"),
  FileContent(path="susannah", content=b"Not sure actually"),
))
snapshot = yield Get(Snapshot, FilesToSnapshot, files_content)
```

but it feels kind of sketchy to introduce a whole new type just to differentiate between "we expect this to be an output" and "we expect this to be an input" when really the two _should_ be interchangeable...
> but that would lead to graph ambiguity through Snapshot -> FilesContent -> Snapshot cycles. I guess we could do something like:

It shouldn't, I don't think? Although I have been surprised before (cf #7710).
If I add a rule to go from `FilesContent` -> `Digest`, I get a whole host of `No rule was available to compute` errors.

If I add a rule to go from `FilesContent` -> `Snapshot`, I get no errors. But I expect to because there _is_ a cycle in the rule graph.

The fact that these give different results is surprising and concerning...
The Digest issue is expected due to #7710 (the bug is that there is ambiguity between a Digest already in a rules' params and requesting a Digest via a Get).

> If I add a rule to go from FilesContent -> Snapshot, I get no errors. But I expect to because there is a cycle in the rule graph.

Mind attaching a dot file or PDF?
Oh, and I don't expect `FilesContent -> Snapshot` to be ambiguous in practice precisely due to the constraint that is described as "mostly working" in #7710: the parameter to a Get must be consumed in a subgraph, so it will be non-ambiguous to Get in that case.

EDIT: except the cycle would still theoretically be usable. Hm. Will think about it.
> Oh, and I don't expect `FilesContent -> Snapshot` to be ambiguous in practice precisely due to the constraint that is described as "mostly working" in #7710: the parameter to a Get must be consumed in a subgraph, so it will be non-ambiguous to Get in that case.
> 
> EDIT: except the cycle would still theoretically be usable. Hm. Will think about it.

Having said that, if it currently works, I'd rather that you take advantage of that, as it will force me to apply constraints in #7710 that continue to allow for the clearly useful case of a bidirectional conversion.

Resolving patch:
diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py
--- a/src/python/pants/engine/fs.py
+++ b/src/python/pants/engine/fs.py
@@ -21,6 +21,13 @@ def __str__(self):
     return repr(self)
 
 
+class InputFileContent(FileContent):
+  """A newtype wrapper for FileContent.
+  TODO(7710): This class is currently necessary because the engine
+  otherwise finds a cycle between FileContent <=> DirectoryDigest.
+  """
+
+
 class PathGlobs(datatype([
     'include',
     'exclude',
@@ -178,6 +185,7 @@ class UrlToFetch(datatype([('url', str), ('digest', Digest)])):
 def create_fs_rules():
   """Creates rules that consume the intrinsic filesystem types."""
   return [
+    RootRule(InputFileContent),
     RootRule(Digest),
     RootRule(DirectoriesToMerge),
     RootRule(PathGlobs),
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -333,12 +333,23 @@ def extern_type_to_str(self, context_handle, type_id):
     c = self._ffi.from_handle(context_handle)
     return c.utf8_buf(str(c.from_id(type_id.tup_0).__name__))
 
+  # If we try to pass a None to the CFFI layer, it will silently fail 
+  # in a weird way. So instead we use the empty string/bytestring as
+  # a de-facto null value, in both `extern_val_to_str` and
+  # `extern_val_to_bytes`.
+  @_extern_decl('Buffer', ['ExternContext*', 'Handle*'])
+  def extern_val_to_bytes(self, context_handle, val):
+    """Given a Handle for `obj`, write bytes(obj) and return it."""
+    c = self._ffi.from_handle(context_handle)
+    v = c.from_value(val[0])
+    v_bytes = b'' if v is None else bytes(v)
+    return c.buf(v_bytes)
+
   @_extern_decl('Buffer', ['ExternContext*', 'Handle*'])
   def extern_val_to_str(self, context_handle, val):
     """Given a Handle for `obj`, write str(obj) and return it."""
     c = self._ffi.from_handle(context_handle)
     v = c.from_value(val[0])
-    # Consistently use the empty string to indicate None.
     v_str = '' if v is None else str(v)
     return c.utf8_buf(v_str)
 
@@ -683,6 +694,7 @@ def init_externs():
                            self.ffi_lib.extern_clone_val,
                            self.ffi_lib.extern_drop_handles,
                            self.ffi_lib.extern_type_to_str,
+                           self.ffi_lib.extern_val_to_bytes,
                            self.ffi_lib.extern_val_to_str,
                            self.ffi_lib.extern_store_tuple,
                            self.ffi_lib.extern_store_set,
@@ -785,6 +797,7 @@ def new_scheduler(self,
                     type_merge_snapshots_request,
                     type_directory_with_prefix_to_strip,
                     type_files_content,
+                    type_input_file_content,
                     type_dir,
                     type_file,
                     type_link,
@@ -815,6 +828,7 @@ def ti(type_obj):
         ti(type_merge_snapshots_request),
         ti(type_directory_with_prefix_to_strip),
         ti(type_files_content),
+        ti(type_input_file_content),
         ti(type_dir),
         ti(type_file),
         ti(type_link),
diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py
--- a/src/python/pants/engine/scheduler.py
+++ b/src/python/pants/engine/scheduler.py
@@ -14,8 +14,8 @@
 from pants.base.project_tree import Dir, File, Link
 from pants.build_graph.address import Address
 from pants.engine.fs import (Digest, DirectoriesToMerge, DirectoryToMaterialize,
-                             DirectoryWithPrefixToStrip, FileContent, FilesContent, PathGlobs,
-                             PathGlobsAndRoot, Snapshot, UrlToFetch)
+                             DirectoryWithPrefixToStrip, FileContent, FilesContent,
+                             InputFileContent, PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch)
 from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
 from pants.engine.native import Function, TypeId
 from pants.engine.nodes import Return, Throw
@@ -105,6 +105,7 @@ def __init__(
       type_merge_snapshots_request=DirectoriesToMerge,
       type_directory_with_prefix_to_strip=DirectoryWithPrefixToStrip,
       type_files_content=FilesContent,
+      type_input_file_content=InputFileContent,
       type_dir=Dir,
       type_file=File,
       type_link=Link,


Test patch:
diff --git a/tests/python/pants_test/engine/test_isolated_process.py b/tests/python/pants_test/engine/test_isolated_process.py
--- a/tests/python/pants_test/engine/test_isolated_process.py
+++ b/tests/python/pants_test/engine/test_isolated_process.py
@@ -4,8 +4,8 @@
 import os
 import unittest
 
-from pants.engine.fs import (EMPTY_DIRECTORY_DIGEST, Digest, FileContent, FilesContent, PathGlobs,
-                             Snapshot)
+from pants.engine.fs import (EMPTY_DIRECTORY_DIGEST, Digest, FileContent, FilesContent,
+                             InputFileContent, PathGlobs, Snapshot)
 from pants.engine.isolated_process import (ExecuteProcessRequest, ExecuteProcessResult,
                                            FallibleExecuteProcessResult, ProcessExecutionFailure)
 from pants.engine.rules import RootRule, rule
@@ -253,6 +253,40 @@ def test_create_from_snapshot_with_env(self):
     self.assertEqual(req.env, ('VAR', 'VAL'))
 
 
+class TestInputFileCreation(TestBase):
+  def test_input_file_creation(self):
+    file_name = 'some.filename'
+    file_contents = b'some file contents'
+
+    input_file = InputFileContent(path=file_name, content=file_contents)
+    digest, = self.scheduler.product_request(Digest, [input_file])
+
+    req = ExecuteProcessRequest(
+      argv=('/bin/cat', file_name),
+      input_files=digest,
+      description='cat the contents of this file',
+    )
+
+    result, = self.scheduler.product_request(ExecuteProcessResult, [req])
+    self.assertEqual(result.stdout, file_contents)
+
+  def test_file_in_directory_creation(self):
+    path = 'somedir/filename'
+    content = b'file contents'
+
+    input_file = InputFileContent(path=path, content=content)
+    digest, = self.scheduler.product_request(Digest, [input_file])
+
+    req = ExecuteProcessRequest(
+      argv=('/bin/cat', 'somedir/filename'),
+      input_files=digest,
+      description="Cat a file in a directory to make sure that doesn't break",
+    )
+
+    result, = self.scheduler.product_request(ExecuteProcessResult, [req])
+    self.assertEqual(result.stdout, content)
+
+
 class IsolatedProcessTest(TestBase, unittest.TestCase):
 
   @classmethod