Repository: frewsxcv/cargo-all-features (language: rust)

Issue:
Support chunking
This tool tends to fill up target directories really quickly. For large projects like ICU4X, this puts us in danger of hitting CI size limits. Furthermore, it just takes pretty long and it would be nice to be able to make use of parallelism.

A typical technique in CI systems is to support "chunking", where you have something like `--chunks 3 --chunk 1`, which splits the list of jobs into three chunks, and directs it to run just the jobs for the first one, so you can have the chunks get run on separate runners.

Thoughts?


Discussion / hints:
@Manishearth Do you know which aspect of compiling with different feature flags causes `target/` to fill up? I'm open to your suggestion, but also wonder if there's a way we can clean up some files between runs.
I'm not sure! I upgraded rustc a couple versions in https://github.com/unicode-org/icu4x/pull/1783 and suddenly disk space usage for the feature task has more than doubled (I don't have the exact numbers since i'm still waiting for the job to complete)

I suspect it's codegen unit stuff or whatever. But I'm still investigating. Probably I'll try to measure this for a normal build first instead of all-features which takes forever.

For now I've also switched over to using `check-all-features`, which should be at least a _bit_ better.
You could use `cargo build -v --message-format=json` and delete selected binaries but it's fiddly and you also kinda want stuff to be cached.
What about something like:

```
cargo build

cp -R target target-cached

for features in feature_flag_combinations:
    cargo build --features features
    rm -rf target
    cp -R target-cached target
```
Actually that's probably not good since features could have a ton of dependencies. Maybe the first line should be `cargo build --all-features`. Hmm
yeah the thing is that all of that is doable, but chunking kinda lets you do much the same without as many nuances
_okay so_ there's actually a bug in `cargo-all-features` here wrt the new dependency stuff, `skip_optional_dependencies` isn't working
Yeah, `cargo metadata` unconditionally generates a `"foo": ["dep:foo"]` entry for each optional dependency. Gonna fixup the code here.
(is there a reason this crate isn't using serde-json and/or clap?)
https://github.com/frewsxcv/cargo-all-features/issues/29
Another option here would be to just give a way to filter the return value of `determine_packages_to_test()` in workspaces.

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 5e54526..7f251c5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,21 +2,174 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
 [[package]]
 name = "cargo-all-features"
-version = "1.8.0"
+version = "1.9.0"
 dependencies = [
+ "clap",
  "itertools",
  "json",
  "termcolor",
 ]
 
+[[package]]
+name = "cc"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+
+[[package]]
+name = "clap"
+version = "4.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.3.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
 [[package]]
 name = "either"
 version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "errno"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.1"
@@ -32,6 +185,72 @@ version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
 
+[[package]]
+name = "libc"
+version = "0.2.147"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+
+[[package]]
+name = "once_cell"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rustix"
+version = "0.38.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "2.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.2"
@@ -41,6 +260,18 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "unicode-ident"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -71,3 +302,69 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
diff --git a/Cargo.toml b/Cargo.toml
index 0b4d34a..155c10b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,3 +20,4 @@ path = "src/bin/cargo-test-all-features.rs"
 json = "0.12"
 itertools = "0.10"
 termcolor = "1"
+clap = { version = "4.3.19", features = ["derive"] }
diff --git a/README.md b/README.md
index d2b4222..febb193 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,8 @@ max_combination_size = 4
 allowlist = ["foo", "bar"]
 ```
 
+The project also supports chunking: `--n-chunks 3 --chunks 1` will split the crates being tested into three sets (alphabetically, currently), and run the requested command for the first set of crates only. This is useful for splitting up CI jobs or performing disk cleanups since for large workspaces `check-all-features` and friends can take a very long time and produce a ton of artifacts.
+
 ## License
 
 Licensed under either of
diff --git a/src/lib.rs b/src/lib.rs
index 82c909f..1c237cd 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,4 @@
+use clap::{error::ErrorKind, Command, Parser};
 use std::{env, error, ffi, process};
 
 pub mod cargo_metadata;
@@ -5,18 +6,90 @@ pub mod features_finder;
 pub mod test_runner;
 mod types;
 
+#[derive(Parser, Clone)]
+#[command(author, version, about = "See https://crates.io/crates/cargo-all-features", long_about = None)]
+#[command(bin_name = "cargo")]
+/// The cargo wrapper so that `cargo check-all-features ...` will work, since it internally invokes `check-all-features` with itself
+/// as the first argument
+enum CargoCli {
+    #[command(name = "check-all-features")]
+    #[command(alias = "build-all-features")]
+    #[command(alias = "test-all-features")]
+    Subcommand(Cli),
+}
+
+#[derive(Parser, Clone)]
+#[command(author, version, about = "See https://crates.io/crates/cargo-all-features", long_about = None)]
+struct Cli {
+    #[arg(
+        long,
+        default_value_t = 1,
+        requires = "chunk",
+        help = "Split the workspace into n chunks, each chunk containing a roughly equal number of crates"
+    )]
+    n_chunks: usize,
+    #[arg(
+        long,
+        default_value_t = 1,
+        requires = "n_chunks",
+        help = "Which chunk to test, indexed at 1"
+    )]
+    chunk: usize,
+
+    #[arg(
+        help = "arguments to pass down to cargo",
+        allow_hyphen_values = true,
+        trailing_var_arg = true
+    )]
+    cargo_args: Vec<String>,
+}
+
 pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box<dyn error::Error>> {
-    if let Some(arg) = env::args().nth(1) {
-        if arg == "--help" {
-            println!("See https://crates.io/crates/cargo-all-features");
-            return Ok(());
-        }
+    let CargoCli::Subcommand(cli) = CargoCli::parse();
+    let mut cmd = Command::new("cargo-all-features");
+    if cli.chunk > cli.n_chunks || cli.chunk < 1 {
+        cmd.error(
+            ErrorKind::InvalidValue,
+            "Must not ask for chunks out of bounds",
+        )
+        .print()?;
+        process::exit(1);
+    }
+
+    if cli.n_chunks == 0 {
+        cmd.error(ErrorKind::InvalidValue, "--n-chunks must be at least 1")
+            .print()?;
+        process::exit(1)
     }
 
     let packages = determine_packages_to_test()?;
 
-    for package in packages {
-        let outcome = test_all_features_for_package(&package, cargo_command)?;
+    // chunks() takes a chunk size, not a number of chunks
+    // we must adjust to deal with the fact that if things are not a perfect multiple,
+    // len / n_chunks will end up with an uncounted remainder chunk
+    let mut chunk_size = packages.len() / cli.n_chunks;
+    if packages.len() % cli.n_chunks != 0 {
+        chunk_size += 1;
+    }
+
+    // - 1 since we are 1-indexing
+    let chunk = if let Some(chunk) = packages.chunks(chunk_size).nth(cli.chunk - 1) {
+        chunk
+    } else {
+        println!("Chunk is empty (did you ask for more chunks than there are packages?");
+        return Ok(());
+    };
+    if cli.n_chunks != 1 {
+        let packages: String = chunk.iter().flat_map(|p| [&p.name, ","]).collect();
+        let packages = packages.trim_end_matches(',');
+        println!(
+            "Running on chunk {} out of {} ({chunk_size} packages: {packages})",
+            cli.chunk, cli.n_chunks
+        );
+    }
+
+    for package in chunk {
+        let outcome = test_all_features_for_package(package, cargo_command, &cli.cargo_args)?;
 
         if let TestOutcome::Fail(exit_status) = outcome {
             process::exit(exit_status.code().unwrap());
@@ -29,6 +102,7 @@ pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box<dyn error
 fn test_all_features_for_package(
     package: &cargo_metadata::Package,
     command: crate::test_runner::CargoCommand,
+    cargo_args: &[String],
 ) -> Result<TestOutcome, Box<dyn error::Error>> {
     let feature_sets = crate::features_finder::fetch_feature_sets(package);
 
@@ -37,6 +111,7 @@ fn test_all_features_for_package(
             command,
             package.name.clone(),
             feature_set.clone(),
+            cargo_args,
             package
                 .manifest_path
                 .parent()
diff --git a/src/types.rs b/src/types.rs
index 6d7a2b8..a5c3926 100644
--- a/src/types.rs
+++ b/src/types.rs
@@ -22,13 +22,13 @@ impl FromIterator<Feature> for FeatureList {
 
 impl AsMut<<FeatureList as Deref>::Target> for &mut FeatureList {
     fn as_mut(&mut self) -> &mut <FeatureList as Deref>::Target {
-        self.deref_mut()
+        self
     }
 }
 
 impl AsRef<<FeatureList as Deref>::Target> for &FeatureList {
     fn as_ref(&self) -> &<FeatureList as Deref>::Target {
-        self.deref()
+        self
     }
 }
 
@@ -48,7 +48,7 @@ impl Deref for FeatureList {
 
 impl AsRef<str> for &Feature {
     fn as_ref(&self) -> &str {
-        self.deref()
+        self
     }
 }
 


Test patch:
diff --git a/src/test_runner.rs b/src/test_runner.rs
index 1fa253e..646a4ae 100644
--- a/src/test_runner.rs
+++ b/src/test_runner.rs
@@ -1,5 +1,5 @@
 use crate::types::FeatureList;
-use std::{env, error, path, process};
+use std::{error, path, process};
 use termcolor::WriteColor;
 
 pub struct TestRunner {
@@ -16,15 +16,12 @@ impl TestRunner {
         cargo_command: CargoCommand,
         crate_name: String,
         feature_set: FeatureList,
+        cargo_args: &[String],
         working_dir: path::PathBuf,
     ) -> Self {
         let mut command = process::Command::new(&crate::cargo_cmd());
 
-        command.arg(match cargo_command {
-            CargoCommand::Build => "build",
-            CargoCommand::Check => "check",
-            CargoCommand::Test => "test",
-        });
+        command.arg(cargo_command.get_name());
         command.arg("--no-default-features");
 
         let mut features = feature_set
@@ -39,8 +36,8 @@ impl TestRunner {
         }
 
         // Pass through cargo args
-        for arg in env::args().skip(2) {
-            command.arg(&arg);
+        for arg in cargo_args {
+            command.arg(arg);
         }
 
         TestRunner {
@@ -90,3 +87,20 @@ pub enum CargoCommand {
     Check,
     Test,
 }
+
+impl CargoCommand {
+    pub fn get_name(self) -> &'static str {
+        match self {
+            CargoCommand::Build => "build",
+            CargoCommand::Check => "check",
+            CargoCommand::Test => "test",
+        }
+    }
+    pub fn get_cli_name(self) -> &'static str {
+        match self {
+            CargoCommand::Build => "build-all-features",
+            CargoCommand::Check => "check-all-features",
+            CargoCommand::Test => "test-all-features",
+        }
+    }
+}