Repository: pantsbuild/pants

Issue:
protoc fails in remote execution
The remote-apis-testing test for pants is failing in remote execution. https://gitlab.com/remote-apis-testing/remote-apis-testing/-/jobs/1275989846

```
client_1  | 11:36:36.64 [ERROR] Exception caught: (pants.engine.internals.scheduler.ExecutionError)
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 234, in _run_inner
client_1  |     return self._perform_run(goals)
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 173, in _perform_run
client_1  |     return self._perform_run_body(goals, poll=False)
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/bin/local_pants_runner.py", line 190, in _perform_run_body
client_1  |     return self.graph_session.run_goal_rules(
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/init/engine_initializer.py", line 135, in run_goal_rules
client_1  |     exit_code = self.scheduler_session.run_goal_rule(
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/engine/internals/scheduler.py", line 530, in run_goal_rule
client_1  |     self._raise_on_error([t for _, t in throws])
client_1  |   File "/root/.cache/pants/setup/bootstrap-Linux-x86_64/2.5.0rc2_py38/lib/python3.8/site-packages/pants/engine/internals/scheduler.py", line 498, in _raise_on_error
client_1  |     raise ExecutionError(
client_1  | 
client_1  | Exception message: 1 Exception encountered:
client_1  | 
client_1  |   Exception: Too many failures from server. The last error was: Failed to run command: Failed to start process: fork/exec ./bin/protoc: no such file or directory
```



Discussion:
> The remote-apis-testing test for pants is failing in remote execution. https://gitlab.com/remote-apis-testing/remote-apis-testing/-/jobs/1275989846
> ```

Note that there is now an issue and a Merge request for this on the remote-apis-testing repo.

Issue: https://gitlab.com/remote-apis-testing/remote-apis-testing/-/issues/121
Merge Request: https://gitlab.com/remote-apis-testing/remote-apis-testing/-/merge_requests/244

The CI pipeline on this new Merge Request confirms the results of the original CI pipeline.
This can be reproduced using the branch from [remote-apis-testing project](https://gitlab.com/remote-apis-testing/remote-apis-testing/):

```
$ git clone https://gitlab.com/remote-apis-testing/remote-apis-testing.git
$ cd remote-apis-testing
$ git fetch origin merge-requests/244/head:update_pants_sha
$ git checkout update_pants_sha
$ cd docker-compose
$ python -m venv venv
$ source venv/bin/activate
$ pip install cryptography==3.3.2 docker-compose jinja2 ruamel.yaml
$ ./run.sh -g -p -s docker-compose-buildbarn.yml -c docker-compose-pants.yml
```
It looks like https://github.com/pantsbuild/pants/pull/11867 stumbled upon an existing issue triggering this error. That commit does work with Pants local execution sandboxing; but our local execution sandboxing is non-conforming to the REAPI:
https://github.com/pantsbuild/pants/blob/35c3a0fdacf9e35b5db378fda233f833100b5d55/src/rust/engine/process_execution/bazel_protos/protos/bazelbuild_remote-apis/build/bazel/remote/execution/v2/remote_execution.proto#L574-L595

We do not take any working directory into account when capturing outputs:
https://github.com/pantsbuild/pants/blob/35c3a0fdacf9e35b5db378fda233f833100b5d55/src/rust/engine/process_execution/src/local.rs#L547-L566

Instead of rooting the output capture filesystem at `workdir_path` (e.g.: `/tmp/process-executionXYZ`), we need to be rooting it at the working directory inside that `workdir_path` when the working directory for the Process is set:
https://github.com/pantsbuild/pants/blob/35c3a0fdacf9e35b5db378fda233f833100b5d55/src/rust/engine/process_execution/src/local.rs#L547-L549

Semi-relevant is the discussion to change the REAPI spec to make the paths be relative the input root. Although that change was ultimately rejected. https://github.com/bazelbuild/remote-apis/pull/191#issuecomment-821536302
Aha. Yeah, that's super-relevant: https://github.com/bazelbuild/remote-apis/pull/191#issuecomment-821536302
> From Pants perspective, Pants uses the (non-compliant) input root-relative behavior for output paths. No objection from us to changing the spec.

So you knew about this issue but didn't put two and two together / didn't realize we used `working_directory` when unzipping the archive containing `./bin/protoc`. That's in fact the only place we currently use `working_directory` in any Process.

Resolving patch:
diff --git a/src/python/pants/core/util_rules/archive.py b/src/python/pants/core/util_rules/archive.py
--- a/src/python/pants/core/util_rules/archive.py
+++ b/src/python/pants/core/util_rules/archive.py
@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from enum import Enum
 
-from pants.engine.fs import CreateDigest, Digest, Directory, MergeDigests, RemovePrefix, Snapshot
+from pants.engine.fs import CreateDigest, Digest, Directory, MergeDigests, Snapshot
 from pants.engine.process import (
     BinaryNotFoundError,
     BinaryPath,
@@ -170,10 +170,10 @@ async def maybe_extract_archive(
     digest: Digest, tar_binary: TarBinary, unzip_binary: UnzipBinary
 ) -> ExtractedArchive:
     """If digest contains a single archive file, extract it, otherwise return the input digest."""
-    output_dir = "__output"
+    extract_archive_dir = "__extract_archive_dir"
     snapshot, output_dir_digest = await MultiGet(
         Get(Snapshot, Digest, digest),
-        Get(Digest, CreateDigest([Directory(output_dir)])),
+        Get(Digest, CreateDigest([Directory(extract_archive_dir)])),
     )
     if len(snapshot.files) != 1:
         return ExtractedArchive(digest)
@@ -199,12 +199,11 @@ async def maybe_extract_archive(
             input_digest=input_digest,
             description=f"Extract {fp}",
             level=LogLevel.DEBUG,
-            output_directories=(output_dir,),
-            working_directory=output_dir,
+            output_directories=(".",),
+            working_directory=extract_archive_dir,
         ),
     )
-    strip_output_dir = await Get(Digest, RemovePrefix(result.output_digest, output_dir))
-    return ExtractedArchive(strip_output_dir)
+    return ExtractedArchive(result.output_digest)
 
 
 def rules():
diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py
--- a/src/python/pants/engine/process.py
+++ b/src/python/pants/engine/process.py
@@ -89,21 +89,24 @@ def __init__(
         that are not explicitly populated. For example, $PATH will not be defined by default, unless
         populated through the `env` parameter.
 
-        Usually, you will want to provide input files/directories via the parameter `input_digest`. The
-        process will then be able to access these paths through relative paths. If you want to give
-        multiple input digests, first merge them with `await Get(Digest, MergeDigests)`.
+        Usually, you will want to provide input files/directories via the parameter `input_digest`.
+        The process will then be able to access these paths through relative paths. If you want to
+        give multiple input digests, first merge them with `await Get(Digest, MergeDigests)`.
 
-        Often, you will want to capture the files/directories created in the process. To do this, you
-        can either set `output_files` or `output_directories`. The specified paths will then be used to
-        populate `output_digest` on the `ProcessResult`. If you want to split up this output digest
-        into multiple digests, use `await Get(Digest, DigestSubset)` on the `output_digest`.
+        Often, you will want to capture the files/directories created in the process. To do this,
+        you can either set `output_files` or `output_directories`. The specified paths should be
+        specified relative to the `working_directory`, if any, and will then be used to populate
+        `output_digest` on the `ProcessResult`. If you want to split up this output digest into
+        multiple digests, use `await Get(Digest, DigestSubset)` on the `output_digest`.
 
         To actually run the process, use `await Get(ProcessResult, Process)` or
         `await Get(FallibleProcessResult, Process)`.
 
         Example:
 
-            result = await Get(ProcessResult, Process(["/bin/echo", "hello world"], description="demo"))
+            result = await Get(
+                ProcessResult, Process(["/bin/echo", "hello world"], description="demo")
+            )
             assert result.stdout == b"hello world"
         """
         if isinstance(argv, str):


Test patch:
diff --git a/src/rust/engine/process_execution/src/local_tests.rs b/src/rust/engine/process_execution/src/local_tests.rs
--- a/src/rust/engine/process_execution/src/local_tests.rs
+++ b/src/rust/engine/process_execution/src/local_tests.rs
@@ -9,7 +9,7 @@ use hashing::EMPTY_DIGEST;
 use shell_quote::bash;
 use spectral::{assert_that, string::StrAssertions};
 use std;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::PathBuf;
 use std::str;
 use std::time::Duration;
@@ -567,8 +567,8 @@ async fn working_directory() {
   let executor = task_executor::Executor::new();
   let store = Store::local_only(executor.clone(), store_dir.path()).unwrap();
 
-  // Prepare the store to contain /cats/roland.ext, because the EPR needs to materialize it and then run
-  // from the ./cats directory.
+  // Prepare the store to contain /cats/roland.ext, because the EPR needs to materialize it and
+  // then run from the ./cats directory.
   store
     .store_file_bytes(TestData::roland().bytes(), false)
     .await
@@ -586,6 +586,7 @@ async fn working_directory() {
 
   let mut process = Process::new(vec![find_bash(), "-c".to_owned(), "/bin/ls".to_string()]);
   process.working_directory = Some(RelativePath::new("cats").unwrap());
+  process.output_directories = relative_paths(&["roland.ext"]).collect::<BTreeSet<_>>();
   process.input_files = TestDirectory::nested().digest();
   process.timeout = one_second();
   process.description = "confused-cat".to_string();
@@ -603,7 +604,10 @@ async fn working_directory() {
   assert_eq!(result.stdout_bytes, "roland.ext\n".as_bytes());
   assert_eq!(result.stderr_bytes, "".as_bytes());
   assert_eq!(result.original.exit_code, 0);
-  assert_eq!(result.original.output_directory, EMPTY_DIGEST);
+  assert_eq!(
+    result.original.output_directory,
+    TestDirectory::containing_roland().digest()
+  );
   assert_eq!(result.original.platform, Platform::current().unwrap());
 }