Repository: pantsbuild/pants

Issue:
Move local process execution tempdirs into the workdir, add option to not delete them
When executing processes locally, we currently make a temporary directory: https://github.com/pantsbuild/pants/blob/d0aeac5cb50de06db253f566ad26eb5cc19a8da3/src/rust/engine/process_execution/src/local.rs#L79-L86

We should pass the workdir where we make the `local::CommandRunner` (https://github.com/pantsbuild/pants/blob/d0aeac5cb50de06db253f566ad26eb5cc19a8da3/src/rust/engine/src/context.rs#L94-L97) and add it as a new field.

Then we should switch from calling `tempdir()` to calling `tempdir_in` (https://docs.rs/tempfile/3.0.2/tempfile/struct.Builder.html#method.tempdir_in) in a directory under the workdir

Then we should add an boolean flag along the lines of `--delete-local-process-execution-directories` which defaults to True (see https://github.com/pantsbuild/pants/pull/5932 for how we pass flags), and if it's set to False, not delete the temporary directory. This can be achieved by calling `into_path()` on the `TempDir`: https://docs.rs/tempfile/3.0.2/tempfile/struct.TempDir.html#method.into_path


Discussion:
Oh and if we're keeping the directories, adding some logging showing the directory, so people can find it :)

Resolving patch:
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -209,7 +209,8 @@
                             uint64_t,
                             uint64_t,
                             uint64_t,
-                            uint64_t);
+                            uint64_t,
+                            _Bool);
 void scheduler_pre_fork(Scheduler*);
 Value scheduler_metrics(Scheduler*, Session*);
 RawNodes* scheduler_execute(Scheduler*, Session*, ExecutionRequest*);
@@ -793,6 +794,7 @@ def tc(constraint):
         execution_options.remote_store_chunk_bytes,
         execution_options.remote_store_chunk_upload_timeout_seconds,
         execution_options.process_execution_parallelism,
+        execution_options.process_execution_cleanup_local_dirs
       )
     return self.gc(scheduler, self.lib.scheduler_destroy)
 
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -67,6 +67,7 @@ class ExecutionOptions(datatype([
   'remote_store_chunk_bytes',
   'remote_store_chunk_upload_timeout_seconds',
   'process_execution_parallelism',
+  'process_execution_cleanup_local_dirs',
 ])):
   """A collection of all options related to (remote) execution of processes.
 
@@ -83,6 +84,7 @@ def from_bootstrap_options(cls, bootstrap_options):
       remote_store_chunk_bytes=bootstrap_options.remote_store_chunk_bytes,
       remote_store_chunk_upload_timeout_seconds=bootstrap_options.remote_store_chunk_upload_timeout_seconds,
       process_execution_parallelism=bootstrap_options.process_execution_parallelism,
+      process_execution_cleanup_local_dirs=bootstrap_options.process_execution_cleanup_local_dirs,
     )
 
 
@@ -93,6 +95,7 @@ def from_bootstrap_options(cls, bootstrap_options):
     remote_store_chunk_bytes=1024*1024,
     remote_store_chunk_upload_timeout_seconds=60,
     process_execution_parallelism=multiprocessing.cpu_count()*2,
+    process_execution_cleanup_local_dirs=True,
   )
 
 
@@ -315,6 +318,9 @@ def register_bootstrap_options(cls, register):
     register('--process-execution-parallelism', type=int, default=multiprocessing.cpu_count(),
              advanced=True,
              help='Number of concurrent processes that may be executed either locally and remotely.')
+    register('--process-execution-cleanup-local-dirs', type=bool, default=True,
+             help='Whether or not to cleanup directories used for local process execution '
+                  '(primarily useful for e.g. debugging).')
 
   @classmethod
   def register_options(cls, register):


Test patch: