Repository: pantsbuild/pants

Issue:
'archive' with many files can hit 'Argument list too long'
**Describe the bug**

Creating an archive seems to specify the files as individual command line arguments to the relevant compressor, which results in too-long argument lists if there's a lot of them (or their names are long):

```toml
# pants.toml
[GLOBAL]
pants_version = "2.12.0"

backend_packages = [
  "pants.backend.shell",
]

[anonymous-telemetry]
enabled = false
```

```python
# BUILD
experimental_shell_command(
    name="files-command",
    command="mkdir subdir && for i in $(seq -w 1 20000); do touch subdir/make-the-arg-list-really-long-and-even-longer-1234567890-$i & done; wait",
    tools=["mkdir", "touch", "seq"],
    outputs=["subdir/"]
)
archive(name="files-archive", files=[":files-command"], format="zip")
```

Running `./pants package //:files-archive` gives 20k lines of output:

```
09:32:54.90 ERROR Exception encountered:

  Exception: Failed to execute: Process {
    argv: [
        "/usr/bin/zip",
        "files-archive.zip",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-00001",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-00002",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-00003",
...
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-19996",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-19997",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-19998",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-19999",
        "subdir/make-the-arg-list-really-long-and-even-longer-1234567890-20000",
    ],
    env: {},
    working_directory: None,
    input_digests: InputDigests {
        complete: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<8899bf044ea265fe2dc7499db85d01b5a145879218a22130ebed5fb503fe4c44>,
                size_bytes: 83,
            },
            tree: "Some(..)",
        },
        nailgun: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855>,
                size_bytes: 0,
            },
            tree: "Some(..)",
        },
        input_files: DirectoryDigest {
            digest: Digest {
                hash: Fingerprint<8899bf044ea265fe2dc7499db85d01b5a145879218a22130ebed5fb503fe4c44>,
                size_bytes: 83,
            },
            tree: "Some(..)",
        },
        immutable_inputs: {},
        use_nailgun: [],
    },
    output_files: {
        RelativePath(
            "files-archive.zip",
        ),
    },
    output_directories: {},
    timeout: None,
    execution_slot_variable: None,
    concurrency_available: 0,
    description: "Create files-archive.zip",
    level: Debug,
    append_only_caches: {},
    jdk_home: None,
    platform_constraint: None,
    cache_scope: Successful,
}

Error launching process: Os { code: 7, kind: ArgumentListTooLong, message: "Argument list too long" }
```


**Pants version**
2.12.0

**OS**
macOS

**Additional info**
N/A


Discussion:

Resolving patch:
diff --git a/src/python/pants/core/util_rules/archive.py b/src/python/pants/core/util_rules/archive.py
--- a/src/python/pants/core/util_rules/archive.py
+++ b/src/python/pants/core/util_rules/archive.py
@@ -3,7 +3,9 @@
 
 from __future__ import annotations
 
+import logging
 import os
+import textwrap
 from dataclasses import dataclass
 from pathlib import PurePath
 
@@ -11,6 +13,8 @@
 from pants.core.util_rules.system_binaries import SEARCH_PATHS
 from pants.core.util_rules.system_binaries import ArchiveFormat as ArchiveFormat
 from pants.core.util_rules.system_binaries import (
+    BashBinary,
+    BashBinaryRequest,
     GunzipBinary,
     GunzipBinaryRequest,
     TarBinary,
@@ -20,11 +24,21 @@
     ZipBinary,
     ZipBinaryRequest,
 )
-from pants.engine.fs import CreateDigest, Digest, Directory, MergeDigests, RemovePrefix, Snapshot
+from pants.engine.fs import (
+    CreateDigest,
+    Digest,
+    Directory,
+    FileContent,
+    MergeDigests,
+    RemovePrefix,
+    Snapshot,
+)
 from pants.engine.process import Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.util.logging import LogLevel
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass(frozen=True)
 class CreateArchive:
@@ -40,15 +54,43 @@ class CreateArchive:
 
 @rule(desc="Creating an archive file", level=LogLevel.DEBUG)
 async def create_archive(request: CreateArchive) -> Digest:
+
+    # #16091 -- if an arg list is really long, archive utilities tend to get upset.
+    # passing a list of filenames into the utilities fixes this.
+    FILE_LIST_FILENAME = "__pants_archive_filelist__"
+    file_list_file = FileContent(
+        FILE_LIST_FILENAME, "\n".join(request.snapshot.files).encode("utf-8")
+    )
+    file_list_file_digest = await Get(Digest, CreateDigest([file_list_file]))
+    files_digests = [file_list_file_digest, request.snapshot.digest]
+
     if request.format == ArchiveFormat.ZIP:
-        zip_binary = await Get(ZipBinary, ZipBinaryRequest())
-        argv = zip_binary.create_archive_argv(request.output_filename, request.snapshot.files)
+        zip_binary, bash_binary = await MultiGet(
+            Get(ZipBinary, ZipBinaryRequest()),
+            Get(BashBinary, BashBinaryRequest()),
+        )
+        ZIP_SCRIPT_FILENAME = "__pants_zip_wrapper_script__.sh"
+        zip_script = FileContent(
+            ZIP_SCRIPT_FILENAME,
+            # Using POSIX location/arg format for `cat`. If this gets more complicated, refactor.
+            textwrap.dedent(
+                f"""\
+                set -e
+                /bin/cat {FILE_LIST_FILENAME} | {zip_binary.path} --names-stdin {request.output_filename}
+                """
+            ).encode("utf-8"),
+        )
+        zip_script_digest = await Get(Digest, CreateDigest([zip_script]))
+
         env = {}
-        input_digest = request.snapshot.digest
+        input_digests = [zip_script_digest]
+        argv: tuple[str, ...] = (bash_binary.path, ZIP_SCRIPT_FILENAME)
     else:
         tar_binary = await Get(TarBinary, TarBinaryRequest())
         argv = tar_binary.create_archive_argv(
-            request.output_filename, request.snapshot.files, request.format
+            request.output_filename,
+            request.format,
+            input_file_list_filename=FILE_LIST_FILENAME,
         )
         # `tar` expects to find a couple binaries like `gzip` and `xz` by looking on the PATH.
         env = {"PATH": os.pathsep.join(SEARCH_PATHS)}
@@ -56,7 +98,9 @@ async def create_archive(request: CreateArchive) -> Digest:
         output_dir_digest = await Get(
             Digest, CreateDigest([Directory(os.path.dirname(request.output_filename))])
         )
-        input_digest = await Get(Digest, MergeDigests([output_dir_digest, request.snapshot.digest]))
+        input_digests = [output_dir_digest]
+
+    input_digest = await Get(Digest, MergeDigests([*files_digests, *input_digests]))
 
     result = await Get(
         ProcessResult,
diff --git a/src/python/pants/core/util_rules/system_binaries.py b/src/python/pants/core/util_rules/system_binaries.py
--- a/src/python/pants/core/util_rules/system_binaries.py
+++ b/src/python/pants/core/util_rules/system_binaries.py
@@ -294,7 +294,12 @@ def extract_archive_argv(self, archive_path: str, extract_path: str) -> tuple[st
 
 class TarBinary(BinaryPath):
     def create_archive_argv(
-        self, output_filename: str, input_files: Sequence[str], tar_format: ArchiveFormat
+        self,
+        output_filename: str,
+        tar_format: ArchiveFormat,
+        *,
+        input_files: Sequence[str] = (),
+        input_file_list_filename: str | None = None,
     ) -> tuple[str, ...]:
         # Note that the parent directory for the output_filename must already exist.
         #
@@ -304,7 +309,9 @@ def create_archive_argv(
         compression = {ArchiveFormat.TGZ: "z", ArchiveFormat.TBZ2: "j", ArchiveFormat.TXZ: "J"}.get(
             tar_format, ""
         )
-        return (self.path, f"c{compression}f", output_filename, *input_files)
+
+        files_from = ("--files-from", input_file_list_filename) if input_file_list_filename else ()
+        return (self.path, f"c{compression}f", output_filename, *input_files) + files_from
 
     def extract_archive_argv(
         self, archive_path: str, extract_path: str, *, archive_suffix: str


Test patch: