Repository: mozilla/cargo-vet
Pull request #210 (2022-06-20 20:39:22)

Improve criteria selection in certify
This patch makes 2 major changes to certify:

First, certify will no longer attempt to guess the version you are trying to certify unless there is a previous diff or inspect command in history for the crate being certified. This is to avoid potential mistakes, as it would be easy to miss the fact that the wrong version
was selected.

Second, when suggesting criteria to certify, the command will try harder to find the relevant set of criteria. If the diff or version
being certified isn't found in the `suggest` output history, it will now also check the unaudited table, and fall back to running the resolver to determine what criteria would heal the graph when specified for this revision.

This pass with the resolver is intended to be more relaxed that `suggest`, as the precise delta being checked is already known, so it
should be able to give suggested criteria even for audits which would not be suggested by the `suggest` algorithm.

Fixes #192 


Patch:
diff --git a/src/format.rs b/src/format.rs
index b630a29..86fec02 100644
--- a/src/format.rs
+++ b/src/format.rs
@@ -126,7 +126,7 @@ impl MetaConfig {
 pub type AuditedDependencies = SortedMap<PackageName, Vec<AuditEntry>>;
 
 /// audits.toml
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct AuditsFile {
     /// A map of criteria_name to details on that criteria.
     #[serde(skip_serializing_if = "SortedMap::is_empty")]
@@ -271,7 +271,7 @@ impl Serialize for Delta {
 ////////////////////////////////////////////////////////////////////////////////////
 
 /// config.toml
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct ConfigFile {
     /// This top-level key specifies the default criteria that cargo vet certify will use
     /// when recording audits. If unspecified, this defaults to "safe-to-deploy".
@@ -323,7 +323,7 @@ fn is_default_criteria(val: &CriteriaName) -> bool {
 /// If this sounds overwhelming, don't worry, everything defaults to "nothing special"
 /// and an empty PolicyTable basically just means "everything should satisfy the
 /// default criteria in audits.toml".
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct PolicyEntry {
     /// Whether this nominally-first-party crate should actually be subject to audits
     /// as-if it was third-party, based on matches to crates.io packages with the same
@@ -388,7 +388,7 @@ pub static DEFAULT_POLICY_CRITERIA: CriteriaStr = SAFE_TO_DEPLOY;
 pub static DEFAULT_POLICY_DEV_CRITERIA: CriteriaStr = SAFE_TO_RUN;
 
 /// A remote audits.toml that we trust the contents of (by virtue of trusting the maintainer).
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct RemoteImport {
     /// URL of the foreign audits.toml
     pub url: String,
@@ -398,7 +398,7 @@ pub struct RemoteImport {
 }
 
 /// Translations of foreign criteria to local criteria.
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct CriteriaMapping {
     /// This local criteria is implied...
     pub ours: CriteriaName,
@@ -454,7 +454,7 @@ fn is_default_unaudited_suggest(val: &bool) -> bool {
 ////////////////////////////////////////////////////////////////////////////////////
 
 /// imports.lock, not sure what I want to put in here yet.
-#[derive(serde::Serialize, serde::Deserialize)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct ImportsFile {
     pub audits: SortedMap<ImportName, AuditsFile>,
 }
@@ -509,16 +509,8 @@ impl FetchCommand {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct SuggestedAudit {
-    #[serde(flatten)]
-    pub command: FetchCommand,
-    pub criteria: Vec<CriteriaName>,
-}
-
 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
 pub struct CommandHistory {
     #[serde(flatten)]
     pub last_fetch: Option<FetchCommand>,
-    pub last_suggest: Vec<SuggestedAudit>,
 }
diff --git a/src/main.rs b/src/main.rs
index 82de316..6a9f172 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -504,18 +504,18 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
     // Certify that you have reviewed a crate's source for some version / delta
     let mut store = Store::acquire(cfg)?;
     let network = Network::acquire(cfg);
-    // Grab the command history and immediately drop the cache
-    let command_history = Cache::acquire(cfg)?.get_command_history();
+
+    // Grab the last fetch and immediately drop the cache
+    let last_fetch = Cache::acquire(cfg)?.get_last_fetch();
 
     let term = Term::stdout();
 
     // Before setting up magic, we need to agree on a package
-    let last_fetch_name = command_history.last_fetch.as_ref().map(|f| f.package());
     let package = if let Some(package) = &sub_args.package {
         package.clone()
-    } else if let Some(package) = last_fetch_name {
+    } else if let Some(last_fetch) = &last_fetch {
         // If we just fetched a package, assume we want to certify it
-        package.to_owned()
+        last_fetch.package().to_owned()
     } else {
         // ERRORS: immediate fatal diagnostic
         writeln!(
@@ -539,13 +539,6 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
         panic_any(ExitPanic(-1));
     }
 
-    // If the package name now matches last_fetch, make it available for further magic
-    let last_fetch = if last_fetch_name == Some(&package) {
-        command_history.last_fetch.as_ref()
-    } else {
-        None
-    };
-
     let dependency_criteria = if sub_args.dependency_criteria.is_empty() {
         // TODO: look at the current audits to infer this? prompt?
         DependencyCriteria::new()
@@ -560,7 +553,6 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
         dep_criteria
     };
 
-    let mut criteria_guess = None;
     let kind = if let Some(v1) = &sub_args.version1 {
         // If explicit versions were provided, use those
         if let Some(v2) = &sub_args.version2 {
@@ -579,90 +571,19 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
                 dependency_criteria,
             }
         }
-    } else if let Some(fetch) = last_fetch {
+    } else if let Some(fetch) = last_fetch.filter(|f| f.package() == package) {
         // Otherwise, is we just fetched this package, use the version(s) we fetched
         match fetch {
             FetchCommand::Inspect { version, .. } => AuditKind::Full {
-                version: version.clone(),
-                dependency_criteria,
-            },
-            FetchCommand::Diff {
-                version1, version2, ..
-            } => AuditKind::Delta {
-                delta: Delta {
-                    from: version1.clone(),
-                    to: version2.clone(),
-                },
-                dependency_criteria,
-            },
-        }
-    } else if let Some(unaudited_list) = store.config.unaudited.get(&package) {
-        // Otherwise, if we have an unaudited entry for this package, use that version
-        if unaudited_list.len() > 1 {
-            // ERRORS: immediate fatal diagnostic
-            writeln!(
-                out,
-                "error: couldn't guess what version to certify, you have multiple 'unaudited' entries for {}:",
-                package
-            )?;
-            for entry in unaudited_list {
-                writeln!(out, "  {}", entry.version)?;
-            }
-            panic_any(ExitPanic(-1));
-        }
-        let entry = &unaudited_list[0];
-        criteria_guess = Some(entry.criteria.clone());
-        // FIXME: this should arguably use entry.dependency_criteria unless the cli specified,
-        // should probably have a more coherent "strategy picking" right at the start instead
-        // of individually sourcing each piece of information
-        AuditKind::Full {
-            version: entry.version.clone(),
-            dependency_criteria,
-        }
-    } else if !command_history.last_suggest.is_empty() {
-        // Otherwise, if we suggested a fetch for this package, use that version
-        let relevant_suggestions = command_history
-            .last_suggest
-            .iter()
-            .filter(|s| s.command.package() == package)
-            .collect::<Vec<_>>();
-        if relevant_suggestions.is_empty() {
-            // ERRORS: immediate fatal diagnostic
-            writeln!(
-                out,
-                "error: couldn't guess what version to certify, please specify"
-            )?;
-            panic_any(ExitPanic(-1));
-        }
-        if relevant_suggestions.len() > 1 {
-            // ERRORS: immediate fatal diagnostic
-            writeln!(
-                out,
-                "error: couldn't guess what version to certify, you have multiple suggestions for {}:",
-                package
-            )?;
-            for entry in relevant_suggestions {
-                match &entry.command {
-                    FetchCommand::Inspect { version, .. } => writeln!(out, "inspect {}", version)?,
-                    FetchCommand::Diff {
-                        version1, version2, ..
-                    } => writeln!(out, "diff {} {}", version1, version2)?,
-                }
-            }
-            panic_any(ExitPanic(-1));
-        }
-        criteria_guess = Some(relevant_suggestions[0].criteria.clone());
-        match &relevant_suggestions[0].command {
-            FetchCommand::Inspect { version, .. } => AuditKind::Full {
-                version: version.clone(),
+                version,
                 dependency_criteria,
             },
             FetchCommand::Diff {
                 version1, version2, ..
             } => AuditKind::Delta {
                 delta: Delta {
-                    from: version1.clone(),
-                    to: version2.clone(),
+                    from: version1,
+                    to: version2,
                 },
                 dependency_criteria,
             },
@@ -671,7 +592,8 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
         // ERRORS: immediate fatal diagnostic
         writeln!(
             out,
-            "error: couldn't guess what version to certify, please specify"
+            "error: couldn't guess what version of '{}' to certify, please specify",
+            package
         )?;
         panic_any(ExitPanic(-1));
     };
@@ -689,33 +611,12 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
     let criteria_names = if sub_args.criteria.is_empty() {
         // If we don't have explicit cli criteria, guess the criteria
         //
-        // * If any previous operation resulted in a guess, use that
-        // * Otherwise check for a suggest on this exact audit
+        // * Check what would cause `cargo vet` to encounter fewer errors
+        // * Otherwise check what would cause `cargo vet suggest` to suggest fewer audits
         // * Otherwise guess nothing
         //
         // Regardless of the guess, prompt the user to confirm (just needs to mash enter)
-        let mut chosen_criteria = criteria_guess
-            .or_else(|| {
-                command_history
-                    .last_suggest
-                    .into_iter()
-                    .filter(|s| s.command.package() == package)
-                    .find(|v| match (&kind, &v.command) {
-                        (
-                            AuditKind::Full { version: lhs, .. },
-                            FetchCommand::Inspect { version: rhs, .. },
-                        ) => lhs == rhs,
-                        (
-                            AuditKind::Delta { delta, .. },
-                            FetchCommand::Diff {
-                                version1, version2, ..
-                            },
-                        ) => &delta.from == version1 && &delta.to == version2,
-                        _ => false,
-                    })
-                    .map(|s| s.criteria)
-            })
-            .unwrap_or_default();
+        let mut chosen_criteria = guess_audit_criteria(cfg, &store, &package, &kind);
 
         // Prompt for criteria
         loop {
@@ -923,6 +824,57 @@ fn cmd_certify(out: &mut dyn Write, cfg: &Config, sub_args: &CertifyArgs) -> Res
     Ok(())
 }
 
+/// Attempt to guess which criteria are being certified for a given package and
+/// audit kind.
+///
+/// The logic which this method uses to guess the criteria to use is as follows:
+///
+/// * Check what would cause `cargo vet` to encounter fewer errors
+/// * Otherwise check what would cause `cargo vet suggest` to suggest fewer audits
+/// * Otherwise guess nothing
+fn guess_audit_criteria(
+    cfg: &Config,
+    store: &Store,
+    package: PackageStr<'_>,
+    kind: &AuditKind,
+) -> Vec<String> {
+    // Convert the audit to a normalized delta to run against the resolver.
+    let delta = match kind {
+        AuditKind::Full { version, .. } => Delta {
+            from: resolver::ROOT_VERSION.clone(),
+            to: version.clone(),
+        },
+        AuditKind::Delta { delta, .. } => delta.clone(),
+        _ => return Vec::new(),
+    };
+
+    // Attempt to resolve a normal `cargo vet`, and try to find criteria which
+    // would heal some errors in that result if it fails.
+    let criteria = resolver::resolve(
+        &cfg.metadata,
+        cfg.cli.filter_graph.as_ref(),
+        store,
+        ResolveDepth::Deep,
+    )
+    .compute_suggested_criteria(package, &delta);
+    if !criteria.is_empty() {
+        return criteria;
+    }
+
+    // If a normal `cargo vet` failed to turn up any criteria, try a more
+    // aggressive `cargo vet suggest`.
+    //
+    // This is as much as we can do, so just return the result whether or not we
+    // find anything.
+    resolver::resolve(
+        &cfg.metadata,
+        cfg.cli.filter_graph.as_ref(),
+        &store.clone_for_suggest(),
+        ResolveDepth::Deep,
+    )
+    .compute_suggested_criteria(package, &delta)
+}
+
 fn cmd_record_violation(
     out: &mut dyn Write,
     cfg: &Config,
@@ -1055,19 +1007,14 @@ fn cmd_add_unaudited(
 fn cmd_suggest(out: &mut dyn Write, cfg: &Config, _sub_args: &SuggestArgs) -> Result<(), VetError> {
     // Run the checker to validate that the current set of deps is covered by the current cargo vet store
     trace!("suggesting...");
-    let mut store = Store::acquire(cfg)?;
+    let suggest_store = Store::acquire(cfg)?.clone_for_suggest();
     let network = Network::acquire(cfg);
 
-    // Delete all unaudited entries except those that are suggest=false
-    for versions in &mut store.config.unaudited.values_mut() {
-        versions.retain(|e| !e.suggest);
-    }
-
     // DO THE THING!!!!
     let report = resolver::resolve(
         &cfg.metadata,
         cfg.cli.filter_graph.as_ref(),
-        &store,
+        &suggest_store,
         if cfg.cli.shallow {
             ResolveDepth::Shallow
         } else {
@@ -1080,8 +1027,6 @@ fn cmd_suggest(out: &mut dyn Write, cfg: &Config, _sub_args: &SuggestArgs) -> Re
         OutputFormat::Json => report.print_json(out, cfg, suggest.as_ref())?,
     }
 
-    // Don't commit the store, because we purged the unaudited table above.
-
     Ok(())
 }
 
diff --git a/src/resolver.rs b/src/resolver.rs
index e978ec6..2b35c09 100644
--- a/src/resolver.rs
+++ b/src/resolver.rs
@@ -68,8 +68,8 @@ use std::io::Write;
 use tracing::{error, trace, trace_span, warn};
 
 use crate::format::{
-    self, AuditKind, CriteriaName, CriteriaStr, Delta, DiffStat, FetchCommand, ImportName,
-    PackageName, PackageStr, PolicyEntry, SuggestedAudit, UnauditedDependency,
+    self, AuditKind, CriteriaName, CriteriaStr, Delta, DiffStat, ImportName, PackageName,
+    PackageStr, PolicyEntry, UnauditedDependency,
 };
 use crate::format::{FastMap, FastSet, SortedMap, SortedSet};
 use crate::network::Network;
@@ -2506,30 +2506,6 @@ impl<'a> ResolveReport<'a> {
                 .push(s);
         }
 
-        let mut last_suggest = vec![];
-        for suggestion in &suggestions {
-            let package = self.graph.nodes[suggestion.package].name.to_string();
-            let command = if suggestion.suggested_diff.from == ROOT_VERSION {
-                FetchCommand::Inspect {
-                    package,
-                    version: suggestion.suggested_diff.to.clone(),
-                }
-            } else {
-                FetchCommand::Diff {
-                    package,
-                    version1: suggestion.suggested_diff.from.clone(),
-                    version2: suggestion.suggested_diff.to.clone(),
-                }
-            };
-            let criteria = self
-                .criteria_mapper
-                .all_criteria_names(&suggestion.suggested_criteria)
-                .map(CriteriaName::from)
-                .collect::<Vec<_>>();
-            last_suggest.push(SuggestedAudit { command, criteria });
-        }
-        cache.set_last_suggest(last_suggest);
-
         Ok(Some(Suggest {
             suggestions,
             suggestions_by_criteria,
@@ -2537,6 +2513,56 @@ impl<'a> ResolveReport<'a> {
         }))
     }
 
+    /// Given a package name and a delta to be certified, determine the set of
+    /// additional criteria for that delta/version pair which would have a
+    /// healing impact on the audit graph.
+    ///
+    /// This is more reliable than running a suggest and looking for a matching
+    /// output, as it will also select criteria for non-suggested audits.
+    pub fn compute_suggested_criteria(
+        &self,
+        package_name: PackageStr<'_>,
+        delta: &Delta,
+    ) -> Vec<CriteriaName> {
+        let fail = if let Conclusion::FailForVet(fail) = &self.conclusion {
+            fail
+        } else {
+            return Vec::new();
+        };
+
+        let mut criteria = self.criteria_mapper.no_criteria();
+
+        // Enumerate over the recorded failures, adding any criteria for this
+        // delta which would connect that package version into the audit graph.
+        for (&failure_idx, audit_failure) in &fail.failures {
+            let package = &self.graph.nodes[failure_idx];
+            if package.name != package_name {
+                continue;
+            }
+
+            let result = &self.results[failure_idx];
+            for criteria_idx in audit_failure.criteria_failures.all().indices() {
+                let search_result = &result.search_results[criteria_idx];
+                if let SearchResult::Disconnected {
+                    reachable_from_root,
+                    reachable_from_target,
+                } = search_result
+                {
+                    if reachable_from_target.contains(&delta.to)
+                        && reachable_from_root.contains(&delta.from)
+                    {
+                        criteria.set_criteria(criteria_idx);
+                    }
+                }
+            }
+        }
+
+        self.criteria_mapper
+            .criteria_names(&criteria)
+            .map(str::to_owned)
+            .collect()
+    }
+
     /// Print a full human-readable report
     pub fn print_human(
         &self,
diff --git a/src/storage.rs b/src/storage.rs
index b397a22..4692d95 100644
--- a/src/storage.rs
+++ b/src/storage.rs
@@ -22,7 +22,7 @@ use crate::{
     flock::{FileLock, Filesystem},
     format::{
         AuditsFile, CommandHistory, ConfigFile, Delta, DiffCache, DiffStat, FastMap, FetchCommand,
-        ImportsFile, MetaConfig, PackageStr, SortedMap, SuggestedAudit,
+        ImportsFile, MetaConfig, PackageStr, SortedMap,
     },
     network::Network,
     resolver,
@@ -186,6 +186,29 @@ impl Store {
         }
     }
 
+    /// Create a clone of the store for use to resolve `suggest`.
+    ///
+    /// This cloned store will not contain `unaudited` entries from the config,
+    /// unless they're marked as `suggest = false`, such that the resolver will
+    /// identify these missing audits when generating a report.
+    ///
+    /// Unlike the primary store created with `Store::acquire` or
+    /// `Store::create`, this store will not hold the store lock, and cannot be
+    /// committed to disk by calling `commit()`.
+    pub fn clone_for_suggest(&self) -> Self {
+        let mut clone = Self {
+            lock: None,
+            config: self.config.clone(),
+            imports: self.imports.clone(),
+            audits: self.audits.clone(),
+        };
+        // Delete all unaudited entries except those that are suggest=false
+        for versions in &mut clone.config.unaudited.values_mut() {
+            versions.retain(|e| !e.suggest);
+        }
+        clone
+    }
+
     /// Commit the store's contents back to disk
     pub fn commit(self) -> Result<(), VetError> {
         // TODO: make this truly transactional?
@@ -842,20 +865,15 @@ impl Cache {
         tokio::runtime::Handle::current().block_on(self.clean())
     }
 
-    pub fn get_command_history(&self) -> CommandHistory {
+    pub fn get_last_fetch(&self) -> Option<FetchCommand> {
         let guard = self.state.lock().unwrap();
-        guard.command_history.clone()
+        guard.command_history.last_fetch.clone()
     }
 
     pub fn set_last_fetch(&self, last_fetch: FetchCommand) {
         let mut guard = self.state.lock().unwrap();
         guard.command_history.last_fetch = Some(last_fetch);
     }
-
-    pub fn set_last_suggest(&self, last_suggest: Vec<SuggestedAudit>) {
-        let mut guard = self.state.lock().unwrap();
-        guard.command_history.last_suggest = last_suggest;
-    }
 }
 
 /// Queries a package in the crates.io registry for a specific published version


Test patch:
diff --git a/src/tests/certify.rs b/src/tests/certify.rs
new file mode 100644
index 0000000..5d05b2b
--- /dev/null
+++ b/src/tests/certify.rs
@@ -0,0 +1,85 @@
+use super::*;
+use std::fmt::Write;
+
+#[test]
+fn mock_simple_suggested_criteria() {
+    let mock = MockMetadata::simple();
+
+    let metadata = mock.metadata();
+
+    let (mut config, mut audits, imports) = files_no_unaudited(&metadata);
+
+    config.policy.insert(
+        "first-party".to_string(),
+        dep_policy([("third-party1", ["strong-reviewed"])]),
+    );
+
+    audits.audits.insert(
+        "third-party1".to_owned(),
+        vec![
+            full_audit(ver(2), "weak-reviewed"),
+            full_audit(ver(3), "reviewed"),
+            full_audit(ver(4), "strong-reviewed"),
+            delta_audit(ver(6), ver(DEFAULT_VER), "strong-reviewed"),
+            delta_audit(ver(7), ver(DEFAULT_VER), "reviewed"),
+            delta_audit(ver(8), ver(DEFAULT_VER), "weak-reviewed"),
+        ],
+    );
+    audits.audits.insert(
+        "third-party2".to_owned(),
+        vec![
+            full_audit(ver(2), "weak-reviewed"),
+            full_audit(ver(3), "reviewed"),
+            full_audit(ver(4), "strong-reviewed"),
+            delta_audit(ver(6), ver(DEFAULT_VER), "strong-reviewed"),
+            delta_audit(ver(7), ver(DEFAULT_VER), "reviewed"),
+            delta_audit(ver(8), ver(DEFAULT_VER), "weak-reviewed"),
+        ],
+    );
+
+    let store = Store::mock(config, audits, imports);
+    let report = crate::resolver::resolve(&metadata, None, &store, ResolveDepth::Deep);
+
+    let mut output = String::new();
+    for (from, to, descr) in [
+        (0, DEFAULT_VER, "full audit"),
+        // from existing audit
+        (2, DEFAULT_VER, "from weak-reviewed"),
+        (3, DEFAULT_VER, "from reviewed"),
+        (4, DEFAULT_VER, "from strong-reviewed"),
+        // to existing audit
+        (0, 6, "to strong-reviewed"),
+        (0, 7, "to reviewed"),
+        (0, 8, "to weak-reviewed"),
+        // bridge existing audits
+        (2, 6, "from weak-reviewed to strong-reviewed"),
+        (2, 7, "from weak-reviewed to reviewed"),
+        (2, 8, "from weak-reviewed to weak-reviewed"),
+        (3, 6, "from reviewed to strong-reviewed"),
+        (3, 7, "from reviewed to reviewed"),
+        (3, 8, "from reviewed to weak-reviewed"),
+        (4, 6, "from strong-reviewed to strong-reviewed"),
+        (4, 7, "from strong-reviewed to reviewed"),
+        (4, 8, "from strong-reviewed to weak-reviewed"),
+    ] {
+        let delta = Delta {
+            from: ver(from),
+            to: ver(to),
+        };
+        writeln!(output, "{} ({} -> {})", descr, delta.from, delta.to).unwrap();
+        writeln!(
+            output,
+            "  third-party1: {:?}",
+            report.compute_suggested_criteria("third-party1", &delta)
+        )
+        .unwrap();
+        writeln!(
+            output,
+            "  third-party2: {:?}",
+            report.compute_suggested_criteria("third-party2", &delta)
+        )
+        .unwrap();
+    }
+
+    insta::assert_snapshot!("mock-simple-suggested-criteria", output);
+}
diff --git a/src/tests/mod.rs b/src/tests/mod.rs
index 1194c0d..7f1809d 100644
--- a/src/tests/mod.rs
+++ b/src/tests/mod.rs
@@ -15,6 +15,7 @@ use crate::{
 };
 
 mod audit_as_crates_io;
+mod certify;
 mod regenerate_unaudited;
 mod vet;
 mod violations;
diff --git a/src/tests/snapshots/cargo_vet__tests__certify__mock-simple-suggested-criteria.snap b/src/tests/snapshots/cargo_vet__tests__certify__mock-simple-suggested-criteria.snap
new file mode 100644
index 0000000..adc161d
--- /dev/null
+++ b/src/tests/snapshots/cargo_vet__tests__certify__mock-simple-suggested-criteria.snap
@@ -0,0 +1,53 @@
+---
+source: src/tests/certify.rs
+expression: output
+---
+full audit (0.0.0 -> 10.0.0)
+  third-party1: ["strong-reviewed"]
+  third-party2: ["reviewed"]
+from weak-reviewed (2.0.0 -> 10.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from reviewed (3.0.0 -> 10.0.0)
+  third-party1: ["reviewed"]
+  third-party2: ["reviewed"]
+from strong-reviewed (4.0.0 -> 10.0.0)
+  third-party1: ["strong-reviewed"]
+  third-party2: ["reviewed"]
+to strong-reviewed (0.0.0 -> 6.0.0)
+  third-party1: ["strong-reviewed"]
+  third-party2: ["reviewed"]
+to reviewed (0.0.0 -> 7.0.0)
+  third-party1: ["reviewed"]
+  third-party2: ["reviewed"]
+to weak-reviewed (0.0.0 -> 8.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from weak-reviewed to strong-reviewed (2.0.0 -> 6.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from weak-reviewed to reviewed (2.0.0 -> 7.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from weak-reviewed to weak-reviewed (2.0.0 -> 8.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from reviewed to strong-reviewed (3.0.0 -> 6.0.0)
+  third-party1: ["reviewed"]
+  third-party2: ["reviewed"]
+from reviewed to reviewed (3.0.0 -> 7.0.0)
+  third-party1: ["reviewed"]
+  third-party2: ["reviewed"]
+from reviewed to weak-reviewed (3.0.0 -> 8.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+from strong-reviewed to strong-reviewed (4.0.0 -> 6.0.0)
+  third-party1: ["strong-reviewed"]
+  third-party2: ["reviewed"]
+from strong-reviewed to reviewed (4.0.0 -> 7.0.0)
+  third-party1: ["reviewed"]
+  third-party2: ["reviewed"]
+from strong-reviewed to weak-reviewed (4.0.0 -> 8.0.0)
+  third-party1: ["weak-reviewed"]
+  third-party2: ["weak-reviewed"]
+