Repository: taiki-e/cargo-hack (language: rust)

Issue:
Idea: add flag to test all feature flag permutation w/ a max # of simultaneous feature flags
maybe `--feature-powerset --depth 2`.

Tokio has too many features to test the power set. However, **most** issues can be reproduced by testing at most 2 features simultaneously. It would be nice to have a feature that tested all permutations of **at most** N simultaneous feature flags. This would catch most problems.


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 6a3802fd..0149c391 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,12 @@ The following flags can be used with `--each-feature` and `--feature-powerset`.
 
   Skip run of just `--no-default-features` flag.
 
+* **`--depth`**
+
+  Specify a max number of simultaneous feature flags of `--feature-powerset`.
+
+  If the number is set to 1, `--feature-powerset` is equivalent to `--each-feature`.
+
 `cargo-hack` changes the behavior of the following existing flags.
 
 * **`--features`**, **`--no-default-features`**
@@ -124,7 +130,7 @@ The following flags can be used with `--each-feature` and `--feature-powerset`.
   # If you use cargo-hack, you don't need to maintain this list manually.
   members=("foo" "bar")
 
-  for member in ${members[@]}; do
+  for member in "${members[@]}"; do
       cargo check --manifest-path "${member}/Cargo.toml"
   done
   ```
diff --git a/src/cli.rs b/src/cli.rs
index 99f0a84e..0a9a82d5 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -39,6 +39,15 @@ const HELP: &[(&str, &str, &str, &[&str])] = &[
     ("", "--skip-no-default-features", "Skip run of just --no-default-features flag", &[
         "This flag can only be used with either --each-feature flag or --feature-powerset flag.",
     ]),
+    (
+        "",
+        "--depth <NUM>",
+        "Specify a max number of simultaneous feature flags of --feature-powerset",
+        &[
+            "If NUM is set to 1, --feature-powerset is equivalent to --each-feature.",
+            "This flag can only be used with --feature-powerset flag.",
+        ],
+    ),
     ("", "--no-dev-deps", "Perform without dev-dependencies", &[
         "This flag removes dev-dependencies from real `Cargo.toml` while cargo-hack is running and restores it when finished.",
     ]),
@@ -190,6 +199,8 @@ pub(crate) struct Args {
     pub(crate) clean_per_run: bool,
     /// -v, --verbose, -vv
     pub(crate) verbose: bool,
+    /// --depth <NUM>
+    pub(crate) depth: Option<usize>,
 
     // flags that will be propagated to cargo
     /// --features <FEATURES>...
@@ -270,6 +281,7 @@ pub(crate) fn args(coloring: &mut Option<Coloring>) -> Result<Option<Args>> {
     let mut skip_no_default_features = false;
     let mut clean_per_run = false;
     let mut verbose = false;
+    let mut depth = None;
 
     let res = (|| -> Result<()> {
         while let Some(arg) = args.next() {
@@ -373,6 +385,7 @@ pub(crate) fn args(coloring: &mut Option<Coloring>) -> Result<Option<Args>> {
             }
 
             parse_opt!(manifest_path, false, "--manifest-path", "--manifest-path <PATH>");
+            parse_opt!(depth, false, "--depth", "--depth <NUM>");
             parse_opt!(color, true, "--color", "--color <WHEN>");
 
             parse_multi_opt!(package, false, true, "--package", "--package <SPEC>...");
@@ -463,6 +476,10 @@ pub(crate) fn args(coloring: &mut Option<Coloring>) -> Result<Option<Args>> {
             );
         }
     }
+    if depth.is_some() && !feature_powerset {
+        bail!("--depth can only be used with --feature-powerset");
+    }
+    let depth = depth.map(|s| s.parse::<usize>()).transpose()?;
 
     if let Some(subcommand) = &subcommand {
         if subcommand == "test" || subcommand == "bench" {
@@ -546,6 +563,7 @@ For more information try --help
         optional_deps,
         skip_no_default_features,
         clean_per_run,
+        depth,
 
         features,
         color,
diff --git a/src/package.rs b/src/package.rs
index 86f07dd6..3f9f5ee8 100644
--- a/src/package.rs
+++ b/src/package.rs
@@ -92,9 +92,9 @@ impl<'a> Kind<'a> {
             }
         } else if args.feature_powerset {
             let features = if let Some(opt_deps) = opt_deps {
-                powerset(features.chain(opt_deps))
+                powerset(features.chain(opt_deps), args.depth)
             } else {
-                powerset(features)
+                powerset(features, args.depth)
             };
 
             if package.features.is_empty() && features.is_empty() {
@@ -219,13 +219,91 @@ fn cargo_clean(cargo: &OsStr, args: &Args, package: &Package<'_>) -> Result<()>
     line.exec()
 }
 
-fn powerset<T: Clone>(iter: impl IntoIterator<Item = T>) -> Vec<Vec<T>> {
+fn powerset<T: Clone>(iter: impl IntoIterator<Item = T>, depth: Option<usize>) -> Vec<Vec<T>> {
     iter.into_iter().fold(vec![vec![]], |mut acc, elem| {
         let ext = acc.clone().into_iter().map(|mut curr| {
             curr.push(elem.clone());
             curr
         });
-        acc.extend(ext);
+        if let Some(depth) = depth {
+            acc.extend(ext.filter(|f| f.len() <= depth));
+        } else {
+            acc.extend(ext);
+        }
         acc
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::powerset;
+
+    #[test]
+    fn powerset_full() {
+        let v = powerset(vec![1, 2, 3, 4], None);
+        assert_eq!(v, vec![
+            vec![],
+            vec![1],
+            vec![2],
+            vec![1, 2],
+            vec![3],
+            vec![1, 3],
+            vec![2, 3],
+            vec![1, 2, 3],
+            vec![4],
+            vec![1, 4],
+            vec![2, 4],
+            vec![1, 2, 4],
+            vec![3, 4],
+            vec![1, 3, 4],
+            vec![2, 3, 4],
+            vec![1, 2, 3, 4],
+        ]);
+    }
+
+    #[test]
+    fn powerset_depth1() {
+        let v = powerset(vec![1, 2, 3, 4], Some(1));
+        assert_eq!(v, vec![vec![], vec![1], vec![2], vec![3], vec![4],]);
+    }
+
+    #[test]
+    fn powerset_depth2() {
+        let v = powerset(vec![1, 2, 3, 4], Some(2));
+        assert_eq!(v, vec![
+            vec![],
+            vec![1],
+            vec![2],
+            vec![1, 2],
+            vec![3],
+            vec![1, 3],
+            vec![2, 3],
+            vec![4],
+            vec![1, 4],
+            vec![2, 4],
+            vec![3, 4],
+        ]);
+    }
+
+    #[test]
+    fn powerset_depth3() {
+        let v = powerset(vec![1, 2, 3, 4], Some(3));
+        assert_eq!(v, vec![
+            vec![],
+            vec![1],
+            vec![2],
+            vec![1, 2],
+            vec![3],
+            vec![1, 3],
+            vec![2, 3],
+            vec![1, 2, 3],
+            vec![4],
+            vec![1, 4],
+            vec![2, 4],
+            vec![1, 2, 4],
+            vec![3, 4],
+            vec![1, 3, 4],
+            vec![2, 3, 4],
+        ]);
+    }
+}
diff --git a/src/remove_dev_deps.rs b/src/remove_dev_deps.rs
index fb88153d..26e96004 100644
--- a/src/remove_dev_deps.rs
+++ b/src/remove_dev_deps.rs
@@ -1,5 +1,6 @@
 use std::cmp;
 
+// Note: The input must be a valid TOML.
 pub(crate) fn remove_dev_deps(bytes: &str) -> String {
     const DEV_DEPS: &str = "dev-dependencies";
     const TARGET: &str = "target.";


Test patch:
diff --git a/tests/long-help.txt b/tests/long-help.txt
index 7a8678ea..ec679d80 100644
--- a/tests/long-help.txt
+++ b/tests/long-help.txt
@@ -51,6 +51,13 @@ OPTIONS:
 
             This flag can only be used with either --each-feature flag or --feature-powerset flag.
 
+        --depth <NUM>
+            Specify a max number of simultaneous feature flags of --feature-powerset.
+
+            If NUM is set to 1, --feature-powerset is equivalent to --each-feature.
+
+            This flag can only be used with --feature-powerset flag.
+
         --no-dev-deps
             Perform without dev-dependencies.
 
diff --git a/tests/short-help.txt b/tests/short-help.txt
index efcd5582..fa08df51 100644
--- a/tests/short-help.txt
+++ b/tests/short-help.txt
@@ -19,6 +19,7 @@ OPTIONS:
         --optional-deps [DEPS]...  Use optional dependencies as features
         --skip <FEATURES>...       Space-separated list of features to skip
         --skip-no-default-features Skip run of just --no-default-features flag
+        --depth <NUM>              Specify a max number of simultaneous feature flags of --feature-powerset
         --no-dev-deps              Perform without dev-dependencies
         --remove-dev-deps          Equivalent to --no-dev-deps flag except for does not restore the original `Cargo.toml` after performed
         --ignore-private           Skip to perform on `publish = false` packages
diff --git a/tests/test.rs b/tests/test.rs
index 4f6b0d96..1deffff4 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -573,6 +573,37 @@ fn feature_powerset() {
         );
 }
 
+#[test]
+fn feature_powerset_depth() {
+    cargo_hack()
+        .args(&["check", "--feature-powerset", "--depth", "2"])
+        .current_dir(test_dir("tests/fixtures/real"))
+        .output()
+        .unwrap()
+        .assert_success()
+        .assert_stderr_contains("running `cargo check` on real (1/8)")
+        .assert_stderr_contains("running `cargo check --no-default-features` on real (2/8)")
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features a` on real (3/8)",
+        )
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features b` on real (4/8)",
+        )
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features c` on real (6/8)",
+        )
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features a,b` on real (5/8)",
+        )
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features a,c` on real (7/8)",
+        )
+        .assert_stderr_contains(
+            "running `cargo check --no-default-features --features b,c` on real (8/8)",
+        )
+        .assert_stderr_not_contains("--features a,b,c");
+}
+
 #[test]
 fn skip_failure() {
     cargo_hack()