Repository: pantsbuild/pants

Issue:
Docker environment can break permissions in the pants cache
**Describe the bug**
When using a docker environment under some circumstances (If a new docker container is spun up?)  the permissions in the cache seem to become broken, causing subsequent builds to break. You then have to delete the caches in order to build again.

**Pants version**
Which version of Pants are you using?
2.16.dev7
**OS**
linux


Docker environment and pex_binary layout loose causes cryptic error message
**Describe the bug**
It seems that if you have a pex_binary that uses the loose layout and you use the new docker environment you get a very cryptic error  ` Error expanding output globs: Permission denied (os error 13)`

**Pants version**
2.16.dev7

**OS**
linux





Discussion:
Which caches, specifically?
I've had to clear out my entire pants cache about 4 times this morning due to this but of course you ask for more information and I can't reproduce it on demand. If it happens again I'll be sure to update.
Just from a gedanken, this makes sense. I think we volume mount in the named caches, and if we do this without special (racy!) post-processing of `chmod`ing, the common `root` container user is going to wreak havoc. The quick check for bug in the code will be - do we run as the current uid:gid when we execute in the container? If not - clearly buggy.
Here is how we connect:
https://github.com/pantsbuild/pants/blob/3e62816f6e2bbe180962d855438ad6fcddb669fa/src/rust/engine/process_execution/src/docker.rs#L76-L77

No user / group in sight here:
https://github.com/pantsbuild/pants/blob/3e62816f6e2bbe180962d855438ad6fcddb669fa/src/rust/engine/process_execution/src/docker.rs#L471-L498

And this is where we would have plumbed through the current user / group up on line 477: https://docs.rs/bollard/latest/bollard/exec/struct.CreateExecOptions.html
What Pants command are you running that exhibits this error? And can you post the stack trace (run with `--print-stacktrace`)?
Just a package of a pex_binary. 

```
Engine traceback:
  in select
    ..
  in pants.core.goals.package.package_asset
    `package` goal

Traceback (most recent call last):
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 623, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/core/goals/package.py", line 157, in package_asset
    packages = await MultiGet(
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 358, in MultiGet
    return await _MultiGet(tuple(__arg0))
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 165, in __await__
    result = yield self.gets
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 623, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/core/goals/package.py", line 108, in environment_aware_package
    package = await Get(
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 118, in __await__
    result = yield self
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 623, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/backend/python/goals/package_pex_binary.py", line 154, in package_pex_binary
    pex = await Get(
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 118, in __await__
    result = yield self
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 623, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/backend/python/util_rules/pex.py", line 657, in create_pex
    result = await Get(BuildPexResult, PexRequest, request)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 118, in __await__
    result = yield self
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 623, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/backend/python/util_rules/pex.py", line 579, in build_pex
    result = await Get(
  File "/home/john/.cache/nce/f936ae2506b2c5cda52905c74c48fbc91fc2cc3e1ec3a1b57e08a277fda3c5b6/bindings/venvs/2.16.0.dev7/lib/python3.9/site-packages/pants/engine/internals/selectors.py", line 118, in __await__
    result = yield self
native_engine.IntrinsicError: Failed to execute: Process {
    argv: [
        "/usr/local/bin/python",
        "./pex",
        "--tmpdir",
        ".tmp",
        "--jobs",
        "9",
        "--pip-version",
        "22.3",
        "--python-path",
        "/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
        "--output-file",
        "authentication/authentication_alembic.pex",
        "--no-emit-warnings",
        "--requirements-pex",
        "local_dists.pex",
        "--interpreter-constraint",
        "CPython==3.10.*",
        "--console-script",
        "alembic",
        "--sources-directory=source_files",
        "PyMySQL<2.0.0,>=1.0.2",
        "SQLAlchemy<2.0.0,>=1.4.35",
        "alembic<2.0.0,>=1.8.0",
        "ddtrace==1.4.3",
        "psycopg2-binary<3.0.0,>=2.9.3",
        "pydantic<2.0.0,>=1.9.0",
        "python-json-logger<3.0.0,>=2.0.2",
        "sentry-sdk==1.14.0",
        "sqlmodel<0.0.9,>=0.0.8",
        "--lock",
        "authentication-lock.txt",
        "--no-pypi",
        "--index=https://dl.cloudsmith.io/xBP9rdTpontfzstK/test-sHf/python-test/python/simple/",
        "--index=https://pypi.python.org/simple",
        "--manylinux",
        "manylinux2014",
        "--layout",
        "loose",
    ],
    env: {
        "CPPFLAGS": "",
        "LANG": "C.UTF-8",
        "LDFLAGS": "",
        "PATH": "/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin",
        "PEX_IGNORE_RCFILES": "true",
        "PEX_PYTHON_PATH": "/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
        "PEX_ROOT": ".cache/pex_root",
    },
    working_directory: None,
    input_digests: InputDigests {
        complete: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<3e368718f48ef654c7be62537a17f90756a027d52cf9724851f96b6a0727a6dc>,
                size_bytes: 437,
            },
            tree: "Some(..)",
        },
        nailgun: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855>,
                size_bytes: 0,
            },
            tree: "Some(..)",
        },
        input_files: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<3e368718f48ef654c7be62537a17f90756a027d52cf9724851f96b6a0727a6dc>,
                size_bytes: 437,
            },
            tree: "Some(..)",
        },
        immutable_inputs: {},
        use_nailgun: {},
    },
    output_files: {},
    output_directories: {
        RelativePath(
            "authentication/authentication_alembic.pex",
        ),
    },
    timeout: None,
    execution_slot_variable: None,
    concurrency_available: 9,
    description: "Building 9 requirements for authentication/authentication_alembic.pex from the authentication-lock.txt resolve: PyMySQL<2.0.0,>=1.0.2, SQLAlchemy<2.0.0,>=1.4.35, alembic<2.0.0,>=1.8.0, ddtrace==1.4.3, psycopg2-binary<3.0.0,>=2.9.3, pydantic<2.0.0,>=1.9.0, python-json-logger<3.0.0,>=2.0.2, sentry-sdk==1.14.0, sqlmodel<0.0.9,>=0.0.8",
    level: Info,
    append_only_caches: {
        CacheName(
            "pex_root",
        ): RelativePath(
            ".cache/pex_root",
        ),
    },
    jdk_home: None,
    platform: Linux_x86_64,
    cache_scope: Successful,
    execution_strategy: Docker(
        "sha256:8683c3c52bf6fc2da81aad973f3f98f105d9fc25a8ab1c0617d1eca1355659ec",
    ),
    remote_cache_speculation_delay: 0ns,
}

Error expanding output globs: Permission denied (os error 13)
```
This is almost certainly due to the same underlying issue as https://github.com/pantsbuild/pants/issues/18308#issuecomment-1437180721. The engine runs on the host, it execs into a container where the inputs and outputs are fed-in and extracted, respectively, via volume mounts. The outputs are commonly created by the `root` user since most container images will default to that user and we make no attempt to bridge from a different host uid:gid to the container. As such, when we go to glob up an output directory, it seems very likely the root directory is only traversable by `root`, say `0o700` perms, and we die.

Resolving patch:
diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py
--- a/build-support/bin/generate_github_workflows.py
+++ b/build-support/bin/generate_github_workflows.py
@@ -316,6 +316,14 @@ def platform_env(self) -> dict[str, str]:
             ret["PANTS_CONFIG_FILES"] = "+['pants.ci.toml','pants.ci.aarch64.toml']"
         return ret
 
+    def maybe_append_cargo_test_parallelism(self, cmd: str) -> str:
+        if self.platform == Platform.LINUX_ARM64:
+            # TODO: The ARM64 runner has enough cores to reliably trigger #18191 using
+            # our default settings. We lower parallelism here as a bandaid to work around
+            # #18191 until it can be resolved.
+            return f"{cmd} --test-threads=8"
+        return cmd
+
     def wrap_cmd(self, cmd: str) -> str:
         if self.platform == Platform.MACOS11_ARM64:
             # The self-hosted M1 runner is an X86_64 binary that runs under Rosetta,
@@ -533,19 +541,23 @@ def bootstrap_jobs(
         # We pass --tests to skip doc tests because our generated protos contain
         # invalid doc tests in their comments. We do not pass --all as BRFS tests don't
         # pass on GHA MacOS containers.
-        step_cmd = helper.wrap_cmd("./cargo test --tests -- --nocapture")
+        step_cmd = helper.wrap_cmd(
+            helper.maybe_append_cargo_test_parallelism("./cargo test --tests -- --nocapture")
+        )
     elif rust_testing == RustTesting.ALL:
         human_readable_job_name += ", test and lint Rust"
         human_readable_step_name = "Test and lint Rust"
         # We pass --tests to skip doc tests because our generated protos contain
         # invalid doc tests in their comments.
-        step_cmd = dedent(
-            """\
-            ./build-support/bin/check_rust_pre_commit.sh
-            ./cargo test --all --tests -- --nocapture
-            ./cargo check --benches
-            ./cargo doc
-            """
+        step_cmd = "\n".join(
+            [
+                "./build-support/bin/check_rust_pre_commit.sh",
+                helper.maybe_append_cargo_test_parallelism(
+                    "./cargo test --all --tests -- --nocapture"
+                ),
+                "./cargo check --benches",
+                "./cargo doc",
+            ]
         )
     else:
         raise ValueError(f"Unrecognized RustTesting value: {rust_testing}")


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -102,7 +102,7 @@ jobs:
         TMPDIR: ${{ runner.temp }}
       if: needs.classify_changes.outputs.rust == 'true'
       name: Test Rust
-      run: ./cargo test --tests -- --nocapture
+      run: ./cargo test --tests -- --nocapture --test-threads=8
     strategy:
       matrix:
         python-version:
@@ -217,9 +217,7 @@ jobs:
 
         ./cargo check --benches
 
-        ./cargo doc
-
-        '
+        ./cargo doc'
     strategy:
       matrix:
         python-version:
diff --git a/src/rust/engine/process_execution/src/cache_tests.rs b/src/rust/engine/process_execution/src/cache_tests.rs
--- a/src/rust/engine/process_execution/src/cache_tests.rs
+++ b/src/rust/engine/process_execution/src/cache_tests.rs
@@ -32,7 +32,7 @@ fn create_local_runner() -> (Box<dyn CommandRunnerTrait>, Store, TempDir) {
     store.clone(),
     runtime,
     base_dir.path().to_owned(),
-    NamedCaches::new(named_cache_dir),
+    NamedCaches::new_local(named_cache_dir),
     ImmutableInputs::new(store.clone(), base_dir.path()).unwrap(),
     KeepSandboxes::Never,
   ));
diff --git a/src/rust/engine/process_execution/src/docker_tests.rs b/src/rust/engine/process_execution/src/docker_tests.rs
--- a/src/rust/engine/process_execution/src/docker_tests.rs
+++ b/src/rust/engine/process_execution/src/docker_tests.rs
@@ -541,6 +541,38 @@ async fn all_containing_directories_for_outputs_are_created() {
   assert_eq!(result.original.platform, platform_for_tests().unwrap());
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+async fn outputs_readable_only_by_container_user_are_captured() {
+  skip_if_no_docker_available_in_macos_ci!();
+
+  let result = run_command_via_docker(
+    Process::new(vec![
+      SH_PATH.to_string(),
+      "-c".to_owned(),
+      format!(
+        // Ensure that files are only readable by the container user (which on Linux would usually
+        // mean that a non-root user outside the container would not have access).
+        "/bin/mkdir birds/falcons && echo -n {} > cats/roland.ext && chmod o-r -R birds cats",
+        TestData::roland().string()
+      ),
+    ])
+    .output_files(relative_paths(&["cats/roland.ext"]).collect())
+    .output_directories(relative_paths(&["birds/falcons"]).collect())
+    .docker(IMAGE.to_owned()),
+  )
+  .await
+  .unwrap();
+
+  assert_eq!(result.stdout_bytes, "".as_bytes());
+  assert_eq!(result.stderr_bytes, "".as_bytes());
+  assert_eq!(result.original.exit_code, 0);
+  assert_eq!(
+    result.original.output_directory,
+    TestDirectory::nested_dir_and_file().directory_digest()
+  );
+  assert_eq!(result.original.platform, platform_for_tests().unwrap());
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
 async fn output_empty_dir() {
   skip_if_no_docker_available_in_macos_ci!();
@@ -733,7 +765,7 @@ async fn run_command_via_docker_in_dir(
   let executor = executor.unwrap_or_else(task_executor::Executor::new);
   let store =
     store.unwrap_or_else(|| Store::local_only(executor.clone(), store_dir.path()).unwrap());
-  let (_caches_dir, named_caches, immutable_inputs) =
+  let (_caches_dir, _named_caches, immutable_inputs) =
     named_caches_and_immutable_inputs(store.clone());
   let docker = Box::new(DockerOnceCell::new());
   let image_pull_cache = Box::new(ImagePullCache::new());
@@ -743,7 +775,6 @@ async fn run_command_via_docker_in_dir(
     &docker,
     &image_pull_cache,
     dir.clone(),
-    named_caches,
     immutable_inputs,
     cleanup,
   )?;
diff --git a/src/rust/engine/process_execution/src/local_tests.rs b/src/rust/engine/process_execution/src/local_tests.rs
--- a/src/rust/engine/process_execution/src/local_tests.rs
+++ b/src/rust/engine/process_execution/src/local_tests.rs
@@ -769,7 +769,7 @@ pub(crate) fn named_caches_and_immutable_inputs(
 
   (
     root,
-    NamedCaches::new(named_cache_dir),
+    NamedCaches::new_local(named_cache_dir),
     ImmutableInputs::new(store, &root_path).unwrap(),
   )
 }
diff --git a/src/rust/engine/process_execution/src/nailgun/tests.rs b/src/rust/engine/process_execution/src/nailgun/tests.rs
--- a/src/rust/engine/process_execution/src/nailgun/tests.rs
+++ b/src/rust/engine/process_execution/src/nailgun/tests.rs
@@ -22,7 +22,7 @@ fn pool(size: usize) -> (NailgunPool, NamedCaches, ImmutableInputs) {
   let pool = NailgunPool::new(base_dir.clone(), size, store.clone(), executor);
   (
     pool,
-    NamedCaches::new(named_caches_dir.path().to_owned()),
+    NamedCaches::new_local(named_caches_dir.path().to_owned()),
     ImmutableInputs::new(store, &base_dir).unwrap(),
   )
 }