Repository: pantsbuild/pants

Issue:
Support output directory saving for local process execution
When a process is executed, we can currently specify a list of files which we're interested in saving and accessing after the process executes in the `ExecuteProcessRequest`: https://github.com/pantsbuild/pants/blob/c4c1c29ce5f83b4dcefec6150ca41c52fe601962/src/rust/engine/process_execution/src/lib.rs#L60

For local execution, this saving happens here: https://github.com/pantsbuild/pants/blob/c4c1c29ce5f83b4dcefec6150ca41c52fe601962/src/rust/engine/process_execution/src/local.rs#L59-L91

and those files are merged together into a `Directory` protocol buffer, whose digest we return in the `ExecuteProcessResult`: https://github.com/pantsbuild/pants/blob/c4c1c29ce5f83b4dcefec6150ca41c52fe601962/src/rust/engine/process_execution/src/lib.rs#L74

We should add a new field to `ExecuteProcessRequest` called `output_directories` which is a `BTreeSet<PathBuf>` which is a list of directories whose contents we should save and merge into that `Directory` protocol buffer.

This should probably be implemented by, where we get the `PathStat`s for the output files (https://github.com/pantsbuild/pants/blob/c4c1c29ce5f83b4dcefec6150ca41c52fe601962/src/rust/engine/process_execution/src/local.rs#L77) also constructing a `PathGlobs` for the directories, and calling `expand` on the `PosixFS` (https://github.com/pantsbuild/pants/blob/c4c1c29ce5f83b4dcefec6150ca41c52fe601962/src/rust/engine/fs/src/lib.rs#L712), and then merging the two lists of `PathStat`s.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/graph_info/tasks/cloc.py b/src/python/pants/backend/graph_info/tasks/cloc.py
--- a/src/python/pants/backend/graph_info/tasks/cloc.py
+++ b/src/python/pants/backend/graph_info/tasks/cloc.py
@@ -77,7 +77,15 @@ def console_output(self, targets):
     )
 
     # The cloc script reaches into $PATH to look up perl. Let's assume it's in /usr/bin.
-    req = ExecuteProcessRequest(cmd, (), directory_digest, ('ignored', 'report'), 15 * 60, 'cloc')
+    req = ExecuteProcessRequest(
+      cmd,
+      (),
+      directory_digest,
+      ('ignored', 'report'),
+      (),
+      15 * 60,
+      'cloc'
+    )
     exec_result = self.context.execute_process_synchronously(req, 'cloc', (WorkUnitLabel.TOOL,))
 
     # TODO: Remove this check when https://github.com/pantsbuild/pants/issues/5719 is resolved.
diff --git a/src/python/pants/engine/isolated_process.py b/src/python/pants/engine/isolated_process.py
--- a/src/python/pants/engine/isolated_process.py
+++ b/src/python/pants/engine/isolated_process.py
@@ -24,6 +24,7 @@ class ExecuteProcessRequest(datatype([
   ('env', tuple),
   ('input_files', DirectoryDigest),
   ('output_files', tuple),
+  ('output_directories', tuple),
   # NB: timeout_seconds covers the whole remote operation including queuing and setup.
   ('timeout_seconds', Exactly(float, int)),
   ('description', SubclassesOf(*six.string_types)),
@@ -31,20 +32,46 @@ class ExecuteProcessRequest(datatype([
   """Request for execution with args and snapshots to extract."""
 
   @classmethod
-  def create_from_snapshot(cls, argv, env, snapshot, output_files, timeout_seconds=_default_timeout_seconds, description='process'):
+  def create_from_snapshot(
+    cls,
+    argv,
+    env,
+    snapshot,
+    output_files=(),
+    output_directories=(),
+    timeout_seconds=_default_timeout_seconds,
+    description='process'
+  ):
     cls._verify_env_is_dict(env)
     return ExecuteProcessRequest(
       argv=argv,
       env=tuple(env.items()),
       input_files=snapshot.directory_digest,
       output_files=output_files,
+      output_directories=output_directories,
       timeout_seconds=timeout_seconds,
       description=description,
     )
 
   @classmethod
-  def create_with_empty_snapshot(cls, argv, env, output_files, timeout_seconds=_default_timeout_seconds, description='process'):
-    return cls.create_from_snapshot(argv, env, EMPTY_SNAPSHOT, output_files, timeout_seconds, description)
+  def create_with_empty_snapshot(
+    cls,
+    argv,
+    env,
+    output_files=(),
+    output_directories=(),
+    timeout_seconds=_default_timeout_seconds,
+    description='process'
+  ):
+    return cls.create_from_snapshot(
+      argv,
+      env,
+      EMPTY_SNAPSHOT,
+      output_files,
+      output_directories,
+      timeout_seconds,
+      description
+    )
 
   @classmethod
   def _verify_env_is_dict(cls, env):


Test patch:
diff --git a/tests/python/pants_test/engine/test_isolated_process.py b/tests/python/pants_test/engine/test_isolated_process.py
--- a/tests/python/pants_test/engine/test_isolated_process.py
+++ b/tests/python/pants_test/engine/test_isolated_process.py
@@ -195,11 +195,12 @@ def javac_compile_process_result(javac_compile_req):
     if not java_file.endswith(".java"):
       raise ValueError("Can only compile .java files but got {}".format(java_file))
   sources_snapshot = yield Get(Snapshot, PathGlobs, PathGlobs(java_files, ()))
+  output_dirs = tuple({os.path.dirname(java_file) for java_file in java_files})
   process_request = ExecuteProcessRequest.create_from_snapshot(
     argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
     env=dict(),
     snapshot=sources_snapshot,
-    output_files=tuple(java_file[:-5] + ".class" for java_file in java_files),
+    output_directories=output_dirs,
   )
   javac_proc_result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, process_request)
 
@@ -252,31 +253,39 @@ def test_blows_up_on_invalid_args(self):
         env=dict(),
         input_files='',
         output_files=(),
+        output_directories=(),
         timeout_seconds=0.1,
-        description='')
+        description=''
+      )
     with self.assertRaisesRegexp(TypeCheckError, "input_files"):
       ExecuteProcessRequest(argv=('1',),
         env=dict(),
         input_files=3,
         output_files=(),
+        output_directories=(),
         timeout_seconds=0.1,
-        description='')
+        description=''
+      )
     with self.assertRaisesRegexp(TypeCheckError, "output_files"):
       ExecuteProcessRequest(
         argv=('1',),
         env=tuple(),
         input_files=EMPTY_DIRECTORY_DIGEST,
-        output_files=["blah"],
+        output_files=("blah"),
+        output_directories=(),
         timeout_seconds=0.1,
-        description='')
+        description=''
+      )
     with self.assertRaisesRegexp(TypeCheckError, "timeout"):
       ExecuteProcessRequest(
         argv=('1',),
         env=tuple(),
         input_files=EMPTY_DIRECTORY_DIGEST,
-        output_files=["blah"],
+        output_files=("blah"),
+        output_directories=(),
         timeout_seconds=None,
-        description='')
+        description=''
+      )
 
 
 class IsolatedProcessTest(SchedulerTestBase, unittest.TestCase):
@@ -373,8 +382,11 @@ def test_javac_compilation_example_success(self):
     files_content = self.execute_expecting_one_result(scheduler, FilesContent, result.directory_digest).value.dependencies
 
     self.assertEquals(
-      ("scheduler_inputs/src/java/simple/Simple.class",),
-      tuple(file.path for file in files_content)
+      tuple(sorted((
+        "scheduler_inputs/src/java/simple/Simple.java",
+        "scheduler_inputs/src/java/simple/Simple.class",
+      ))),
+      tuple(sorted(file.path for file in files_content))
     )
 
     self.assertGreater(len(files_content[0].content), 0)