Repository: taiki-e/cargo-llvm-cov (rust)

Issue:
Add more options to fail on insufficient coverage
Enforcing code coverage is good and the current option `--fail-under-lines` is a way to do it but unfortunately rather crude one. Percentage is relative and may change due to amount of code changing vis-a-vis covered code. The `llvm-cov report` does provide absolute values of _covered_ and **uncovered** for each of the 4 categories:

* function
* line
* region
* branch

The better way would therefore be to fail coverage on **uncovered** absolute values rather than percentages which is volatile. That way if new code is added that is fully covered the coverage will pass. However when new code is not fully covered it will also require changing of the threshold(s) which is good thing - it might be justified of course but nevertheless should be visible.

I propose the following options:
```
--fail-uncovered-functions
--fail-uncovered-lines
--fail-uncovered-branches
--fail-uncovered-regions
```


Pull request description:
Add --fail-uncovered-{lines,regions,functions} options
```
        --fail-uncovered-lines <MAX>
            Exit with a status of 1 if the uncovered lines are greater than MAX

        --fail-uncovered-regions <MAX>
            Exit with a status of 1 if the uncovered regions are greater than MAX

        --fail-uncovered-functions <MAX>
            Exit with a status of 1 if the uncovered functions are greater than MAX
```

There is no `--fail-uncovered-branches` option because branch coverage is not supported yet by rustc (#8).

Closes #170 

Patch:
diff --git a/README.md b/README.md
index cbef997..ac01974 100644
--- a/README.md
+++ b/README.md
@@ -130,6 +130,15 @@ OPTIONS:
         --fail-under-lines <MIN>
             Exit with a status of 1 if the total line coverage is less than MIN percent
 
+        --fail-uncovered-lines <MAX>
+            Exit with a status of 1 if the uncovered lines are greater than MAX
+
+        --fail-uncovered-regions <MAX>
+            Exit with a status of 1 if the uncovered regions are greater than MAX
+
+        --fail-uncovered-functions <MAX>
+            Exit with a status of 1 if the uncovered functions are greater than MAX
+
         --show-missing-lines
             Show lines with no coverage
 
diff --git a/src/cli.rs b/src/cli.rs
index e5498eb..bd67f6c 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -319,6 +319,15 @@ pub(crate) struct LlvmCovOptions {
     /// Exit with a status of 1 if the total line coverage is less than MIN percent.
     #[clap(long, value_name = "MIN")]
     pub(crate) fail_under_lines: Option<f64>,
+    /// Exit with a status of 1 if the uncovered lines are greater than MAX.
+    #[clap(long, value_name = "MAX")]
+    pub(crate) fail_uncovered_lines: Option<u64>,
+    /// Exit with a status of 1 if the uncovered regions are greater than MAX.
+    #[clap(long, value_name = "MAX")]
+    pub(crate) fail_uncovered_regions: Option<u64>,
+    /// Exit with a status of 1 if the uncovered functions are greater than MAX.
+    #[clap(long, value_name = "MAX")]
+    pub(crate) fail_uncovered_functions: Option<u64>,
     /// Show lines with no coverage.
     #[clap(long)]
     pub(crate) show_missing_lines: bool,
diff --git a/src/json.rs b/src/json.rs
index 3dab9b1..78a93af 100644
--- a/src/json.rs
+++ b/src/json.rs
@@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize};
 // https://github.com/llvm/llvm-project/blob/llvmorg-14.0.0/llvm/tools/llvm-cov/CoverageExporterJson.cpp#L13-L47
 #[derive(Debug, Serialize, Deserialize)]
 #[cfg_attr(test, serde(deny_unknown_fields))]
-#[allow(unreachable_pub)]
 pub struct LlvmCovJsonExport {
     /// List of one or more export objects
     pub(crate) data: Vec<Export>,
@@ -20,7 +19,6 @@ pub struct LlvmCovJsonExport {
 pub(crate) type UncoveredLines = BTreeMap<String, Vec<u64>>;
 
 impl LlvmCovJsonExport {
-    #[allow(unreachable_pub, dead_code)]
     pub fn demangle(&mut self) {
         for data in &mut self.data {
             if let Some(functions) = &mut data.functions {
@@ -32,7 +30,6 @@ impl LlvmCovJsonExport {
     }
 
     /// Gets the minimal lines coverage of all files.
-    #[allow(unreachable_pub, dead_code)]
     pub fn get_lines_percent(&self) -> Result<f64> {
         let mut count = 0_f64;
         let mut covered = 0_f64;
@@ -51,7 +48,6 @@ impl LlvmCovJsonExport {
     }
 
     /// Gets the list of uncovered lines of all files.
-    #[allow(unreachable_pub, dead_code)]
     #[must_use]
     pub fn get_uncovered_lines(&self, ignore_filename_regex: &Option<String>) -> UncoveredLines {
         let mut uncovered_files: UncoveredLines = BTreeMap::new();
@@ -98,6 +94,42 @@ impl LlvmCovJsonExport {
         }
         uncovered_files
     }
+
+    pub fn count_uncovered_functions(&self) -> Result<u64> {
+        let mut count = 0_u64;
+        let mut covered = 0_u64;
+        for data in &self.data {
+            let totals = &data.totals.as_object().context("totals is not an object")?;
+            let functions = &totals["functions"].as_object().context("no functions")?;
+            count += functions["count"].as_u64().context("no count")?;
+            covered += functions["covered"].as_u64().context("no covered")?;
+        }
+        Ok(count.saturating_sub(covered))
+    }
+
+    pub fn count_uncovered_lines(&self) -> Result<u64> {
+        let mut count = 0_u64;
+        let mut covered = 0_u64;
+        for data in &self.data {
+            let totals = &data.totals.as_object().context("totals is not an object")?;
+            let lines = &totals["lines"].as_object().context("no lines")?;
+            count += lines["count"].as_u64().context("no count")?;
+            covered += lines["covered"].as_u64().context("no covered")?;
+        }
+        Ok(count.saturating_sub(covered))
+    }
+
+    pub fn count_uncovered_regions(&self) -> Result<u64> {
+        let mut count = 0_u64;
+        let mut covered = 0_u64;
+        for data in &self.data {
+            let totals = &data.totals.as_object().context("totals is not an object")?;
+            let regions = &totals["regions"].as_object().context("no regions")?;
+            count += regions["count"].as_u64().context("no count")?;
+            covered += regions["covered"].as_u64().context("no covered")?;
+        }
+        Ok(count.saturating_sub(covered))
+    }
 }
 
 /// Json representation of one `CoverageMapping`
@@ -206,6 +238,8 @@ pub(crate) struct CoverageCounts {
 
 #[cfg(test)]
 mod tests {
+    use std::path::Path;
+
     use fs_err as fs;
 
     use super::*;
@@ -246,6 +280,26 @@ mod tests {
         assert!((percent - 69.565_217_391_304_34).abs() < error_margin);
     }
 
+    #[test]
+    fn test_count_uncovered() {
+        let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+
+        let cases = &[
+            // (path, uncovered_functions, uncovered_lines, uncovered_regions)
+            ("tests/fixtures/coverage-reports/no_coverage/no_coverage.json", 0, 7, 6),
+            ("tests/fixtures/coverage-reports/no_test/no_test.json", 1, 7, 6),
+        ];
+
+        for &(file, uncovered_functions, uncovered_lines, uncovered_regions) in cases {
+            let file = manifest_dir.join(file);
+            let s = fs::read_to_string(file).unwrap();
+            let json = serde_json::from_str::<LlvmCovJsonExport>(&s).unwrap();
+            assert_eq!(json.count_uncovered_functions().unwrap(), uncovered_functions);
+            assert_eq!(json.count_uncovered_lines().unwrap(), uncovered_lines);
+            assert_eq!(json.count_uncovered_regions().unwrap(), uncovered_regions);
+        }
+    }
+
     #[test]
     fn test_get_uncovered_lines() {
         // Given a coverage report which includes function regions:
diff --git a/src/main.rs b/src/main.rs
index 136735b..5708120 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -22,7 +22,6 @@ mod context;
 mod demangler;
 mod env;
 mod fs;
-mod json;
 
 use std::{
     collections::HashMap,
@@ -34,6 +33,7 @@ use std::{
 
 use anyhow::{Context as _, Result};
 use camino::{Utf8Path, Utf8PathBuf};
+use cargo_llvm_cov::json;
 use clap::Parser;
 use cli::{RunOptions, ShowEnvOptions};
 use regex::Regex;
@@ -366,7 +366,12 @@ fn generate_report(cx: &Context) -> Result<()> {
             .context("failed to generate report")?;
     }
 
-    if cx.cov.fail_under_lines.is_some() || cx.cov.show_missing_lines {
+    if cx.cov.fail_under_lines.is_some()
+        || cx.cov.fail_uncovered_functions.is_some()
+        || cx.cov.fail_uncovered_lines.is_some()
+        || cx.cov.fail_uncovered_regions.is_some()
+        || cx.cov.show_missing_lines
+    {
         let format = Format::Json;
         let json = format
             .get_json(cx, &object_files, ignore_filename_regex.as_ref())
@@ -380,6 +385,31 @@ fn generate_report(cx: &Context) -> Result<()> {
             }
         }
 
+        if let Some(fail_uncovered_functions) = cx.cov.fail_uncovered_functions {
+            // Handle --fail-uncovered-functions.
+            let uncovered =
+                json.count_uncovered_functions().context("failed to count uncovered functions")?;
+            if uncovered > fail_uncovered_functions {
+                term::error::set(true);
+            }
+        }
+        if let Some(fail_uncovered_lines) = cx.cov.fail_uncovered_lines {
+            // Handle --fail-uncovered-lines.
+            let uncovered =
+                json.count_uncovered_lines().context("failed to count uncovered lines")?;
+            if uncovered > fail_uncovered_lines {
+                term::error::set(true);
+            }
+        }
+        if let Some(fail_uncovered_regions) = cx.cov.fail_uncovered_regions {
+            // Handle --fail-uncovered-regions.
+            let uncovered =
+                json.count_uncovered_regions().context("failed to count uncovered regions")?;
+            if uncovered > fail_uncovered_regions {
+                term::error::set(true);
+            }
+        }
+
         if cx.cov.show_missing_lines {
             // Handle --show-missing-lines.
             let uncovered_files = json.get_uncovered_lines(&ignore_filename_regex);


Test patch:
diff --git a/tests/long-help.txt b/tests/long-help.txt
index 19d1e9e..d30e379 100644
--- a/tests/long-help.txt
+++ b/tests/long-help.txt
@@ -89,6 +89,15 @@ OPTIONS:
         --fail-under-lines <MIN>
             Exit with a status of 1 if the total line coverage is less than MIN percent
 
+        --fail-uncovered-lines <MAX>
+            Exit with a status of 1 if the uncovered lines are greater than MAX
+
+        --fail-uncovered-regions <MAX>
+            Exit with a status of 1 if the uncovered regions are greater than MAX
+
+        --fail-uncovered-functions <MAX>
+            Exit with a status of 1 if the uncovered functions are greater than MAX
+
         --show-missing-lines
             Show lines with no coverage
 
diff --git a/tests/short-help.txt b/tests/short-help.txt
index 91f9460..d2ca015 100644
--- a/tests/short-help.txt
+++ b/tests/short-help.txt
@@ -57,6 +57,15 @@ OPTIONS:
         --fail-under-lines <MIN>
             Exit with a status of 1 if the total line coverage is less than MIN percent
 
+        --fail-uncovered-lines <MAX>
+            Exit with a status of 1 if the uncovered lines are greater than MAX
+
+        --fail-uncovered-regions <MAX>
+            Exit with a status of 1 if the uncovered regions are greater than MAX
+
+        --fail-uncovered-functions <MAX>
+            Exit with a status of 1 if the uncovered functions are greater than MAX
+
         --show-missing-lines
             Show lines with no coverage