Repository: sharkdp/hyperfine (language: rust)

Issue:
Exporting can fail with "Relative speed comparison is not available" for fast-running commands
This happens when the mean is zero:

```
▶ hyperfine --runs=1 'true' --export-markdown -
Benchmark 1: true
  Time (abs ≡):          0.0 ms               [User: 0.2 ms, System: 0.0 ms]
 
  Warning: Command took less than 5 ms to complete. Note that the results might be inaccurate because hyperfine can not calibrate the shell startup time much more precise than this limit. You can try to use the `-N`/`--shell=none` option to disable the shell completely.
 
Error: Relative speed comparison is not available for markup exporter.
```
Exporting can fail with "Relative speed comparison is not available" for fast-running commands
This happens when the mean is zero:

```
▶ hyperfine --runs=1 'true' --export-markdown -
Benchmark 1: true
  Time (abs ≡):          0.0 ms               [User: 0.2 ms, System: 0.0 ms]
 
  Warning: Command took less than 5 ms to complete. Note that the results might be inaccurate because hyperfine can not calibrate the shell startup time much more precise than this limit. You can try to use the `-N`/`--shell=none` option to disable the shell completely.
 
Error: Relative speed comparison is not available for markup exporter.
```


Discussion / hints:

Fix patch:
diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs
index 02de11b6d..ad9ad5a3a 100644
--- a/src/benchmark/relative_speed.rs
+++ b/src/benchmark/relative_speed.rs
@@ -15,43 +15,72 @@ pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
     l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
 }
 
-pub fn compute(results: &[BenchmarkResult]) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
-    let fastest: &BenchmarkResult = results
+fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
+    results
         .iter()
         .min_by(|&l, &r| compare_mean_time(l, r))
-        .expect("at least one benchmark result");
+        .expect("at least one benchmark result")
+}
+
+fn compute_relative_speeds<'a>(
+    results: &'a [BenchmarkResult],
+    fastest: &'a BenchmarkResult,
+) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
+    results
+        .iter()
+        .map(|result| {
+            let is_fastest = result == fastest;
+
+            if result.mean == 0.0 {
+                return BenchmarkResultWithRelativeSpeed {
+                    result,
+                    relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
+                    relative_speed_stddev: None,
+                    is_fastest,
+                };
+            }
+
+            let ratio = result.mean / fastest.mean;
+
+            // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
+            // Covariance asssumed to be 0, i.e. variables are assumed to be independent
+            let ratio_stddev = match (result.stddev, fastest.stddev) {
+                (Some(result_stddev), Some(fastest_stddev)) => Some(
+                    ratio
+                        * ((result_stddev / result.mean).powi(2)
+                            + (fastest_stddev / fastest.mean).powi(2))
+                        .sqrt(),
+                ),
+                _ => None,
+            };
+
+            BenchmarkResultWithRelativeSpeed {
+                result,
+                relative_speed: ratio,
+                relative_speed_stddev: ratio_stddev,
+                is_fastest,
+            }
+        })
+        .collect()
+}
+
+pub fn compute_with_check(
+    results: &[BenchmarkResult],
+) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
+    let fastest = fastest_of(results);
 
     if fastest.mean == 0.0 {
         return None;
     }
 
-    Some(
-        results
-            .iter()
-            .map(|result| {
-                let ratio = result.mean / fastest.mean;
-
-                // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
-                // Covariance asssumed to be 0, i.e. variables are assumed to be independent
-                let ratio_stddev = match (result.stddev, fastest.stddev) {
-                    (Some(result_stddev), Some(fastest_stddev)) => Some(
-                        ratio
-                            * ((result_stddev / result.mean).powi(2)
-                                + (fastest_stddev / fastest.mean).powi(2))
-                            .sqrt(),
-                    ),
-                    _ => None,
-                };
+    Some(compute_relative_speeds(results, fastest))
+}
 
-                BenchmarkResultWithRelativeSpeed {
-                    result,
-                    relative_speed: ratio,
-                    relative_speed_stddev: ratio_stddev,
-                    is_fastest: result == fastest,
-                }
-            })
-            .collect(),
-    )
+/// Same as compute_with_check, potentially resulting in relative speeds of infinity
+pub fn compute(results: &[BenchmarkResult]) -> Vec<BenchmarkResultWithRelativeSpeed> {
+    let fastest = fastest_of(results);
+
+    compute_relative_speeds(results, fastest)
 }
 
 #[cfg(test)]
@@ -83,7 +112,7 @@ fn test_compute_relative_speed() {
         create_result("cmd3", 5.0),
     ];
 
-    let annotated_results = compute(&results).unwrap();
+    let annotated_results = compute_with_check(&results).unwrap();
 
     assert_relative_eq!(1.5, annotated_results[0].relative_speed);
     assert_relative_eq!(1.0, annotated_results[1].relative_speed);
@@ -94,7 +123,7 @@ fn test_compute_relative_speed() {
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
 
-    let annotated_results = compute(&results);
+    let annotated_results = compute_with_check(&results);
 
     assert!(annotated_results.is_none());
 }
diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs
index 734efbb60..1c86f4a23 100644
--- a/src/benchmark/scheduler.rs
+++ b/src/benchmark/scheduler.rs
@@ -44,10 +44,9 @@ impl<'a> Scheduler<'a> {
             self.results
                 .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
 
-            // We export (all results so far) after each individual benchmark, because
-            // we would risk losing all results if a later benchmark fails.
-            self.export_manager
-                .write_results(&self.results, self.options.time_unit)?;
+            // We export results after each individual benchmark, because
+            // we would risk losing them if a later benchmark fails.
+            self.export_manager.write_results(&self.results, true)?;
         }
 
         Ok(())
@@ -62,7 +61,7 @@ impl<'a> Scheduler<'a> {
             return;
         }
 
-        if let Some(mut annotated_results) = relative_speed::compute(&self.results) {
+        if let Some(mut annotated_results) = relative_speed::compute_with_check(&self.results) {
             annotated_results.sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result));
 
             let fastest = &annotated_results[0];
@@ -88,10 +87,15 @@ impl<'a> Scheduler<'a> {
                 "{}: The benchmark comparison could not be computed as some benchmark times are zero. \
                  This could be caused by background interference during the initial calibration phase \
                  of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
-                 Try to re-run the benchmark on a quiet system. If it does not help, you command is \
-                 most likely too fast to be accurately benchmarked by hyperfine.",
+                 Try to re-run the benchmark on a quiet system. If you did not do so already, try the \
+                 --shell=none/-N option. If it does not help either, you command is most likely too fast \
+                 to be accurately benchmarked by hyperfine.",
                  "Note".bold().red()
             );
         }
     }
+
+    pub fn final_export(&self) -> Result<()> {
+        self.export_manager.write_results(&self.results, false)
+    }
 }
diff --git a/src/export/markup.rs b/src/export/markup.rs
index 2b7c153d8..6984225d6 100644
--- a/src/export/markup.rs
+++ b/src/export/markup.rs
@@ -4,12 +4,13 @@ use crate::output::format::format_duration_value;
 use crate::util::units::Unit;
 
 use super::Exporter;
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 
 pub enum Alignment {
     Left,
     Right,
 }
+
 pub trait MarkupExporter {
     fn table_results(&self, entries: &[BenchmarkResultWithRelativeSpeed], unit: Unit) -> String {
         // prepare table header strings
@@ -105,13 +106,8 @@ impl<T: MarkupExporter> Exporter for T {
     fn serialize(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<Vec<u8>> {
         let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
         let entries = relative_speed::compute(results);
-        if entries.is_none() {
-            return Err(anyhow!(
-                "Relative speed comparison is not available for markup exporter."
-            ));
-        }
 
-        let table = self.table_results(&entries.unwrap(), unit);
+        let table = self.table_results(&entries, unit);
         Ok(table.as_bytes().to_vec())
     }
 }
diff --git a/src/export/mod.rs b/src/export/mod.rs
index fec101609..556409c69 100644
--- a/src/export/mod.rs
+++ b/src/export/mod.rs
@@ -45,22 +45,31 @@ trait Exporter {
     fn serialize(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<Vec<u8>>;
 }
 
-struct ExporterWithFilename {
+pub enum ExportTarget {
+    File(String),
+    Stdout,
+}
+
+struct ExporterWithTarget {
     exporter: Box<dyn Exporter>,
-    filename: String,
+    target: ExportTarget,
 }
 
 /// Handles the management of multiple file exporters.
 #[derive(Default)]
 pub struct ExportManager {
-    exporters: Vec<ExporterWithFilename>,
+    exporters: Vec<ExporterWithTarget>,
+    time_unit: Option<Unit>,
 }
 
 impl ExportManager {
     /// Build the ExportManager that will export the results specified
     /// in the given ArgMatches
-    pub fn from_cli_arguments(matches: &ArgMatches) -> Result<Self> {
-        let mut export_manager = Self::default();
+    pub fn from_cli_arguments(matches: &ArgMatches, time_unit: Option<Unit>) -> Result<Self> {
+        let mut export_manager = Self {
+            exporters: vec![],
+            time_unit,
+        };
         {
             let mut add_exporter = |flag, exporttype| -> Result<()> {
                 if let Some(filename) = matches.get_one::<String>(flag) {
@@ -87,29 +96,43 @@ impl ExportManager {
             ExportType::Orgmode => Box::<OrgmodeExporter>::default(),
         };
 
-        self.exporters.push(ExporterWithFilename {
+        self.exporters.push(ExporterWithTarget {
             exporter,
-            filename: if filename == "-" {
-                if cfg!(windows) {
-                    "con:".to_string()
-                } else {
-                    "/dev/stdout".to_string()
-                }
+            target: if filename == "-" {
+                ExportTarget::Stdout
             } else {
                 let _ = File::create(filename)
                     .with_context(|| format!("Could not create export file '{}'", filename))?;
-                filename.to_string()
+                ExportTarget::File(filename.to_string())
             },
         });
 
         Ok(())
     }
 
-    /// Write the given results to all Exporters contained within this manager
-    pub fn write_results(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<()> {
+    /// Write the given results to all Exporters. The 'intermediate' flag specifies
+    /// whether this is being called while still performing benchmarks, or if this
+    /// is the final call after all benchmarks have been finished. In the former case,
+    /// results are written to all file targets (to always have them up to date, even
+    /// if a benchmark fails). In the latter case, we only print to stdout targets (in
+    /// order not to clutter the output of hyperfine with intermediate results).
+    pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
         for e in &self.exporters {
-            let file_content = e.exporter.serialize(results, unit)?;
-            write_to_file(&e.filename, &file_content)?;
+            let content = || e.exporter.serialize(results, self.time_unit);
+
+            match e.target {
+                ExportTarget::File(ref filename) => {
+                    if intermediate {
+                        write_to_file(filename, &content()?)?
+                    }
+                }
+                ExportTarget::Stdout => {
+                    if !intermediate {
+                        println!();
+                        println!("{}", String::from_utf8(content()?).unwrap());
+                    }
+                }
+            }
         }
         Ok(())
     }
diff --git a/src/main.rs b/src/main.rs
index 441991ada..a75d4bd07 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -34,13 +34,14 @@ fn run() -> Result<()> {
     let cli_arguments = get_cli_arguments(env::args_os());
     let options = Options::from_cli_arguments(&cli_arguments)?;
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
-    let export_manager = ExportManager::from_cli_arguments(&cli_arguments)?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.time_unit)?;
 
     options.validate_against_command_list(&commands)?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
     scheduler.run_benchmarks()?;
     scheduler.print_relative_speed_comparison();
+    scheduler.final_export()?;
 
     Ok(())
 }


Test patch:
diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs
index eb5765607..7930d8d30 100644
--- a/tests/integration_tests.rs
+++ b/tests/integration_tests.rs
@@ -345,3 +345,40 @@ fn performs_all_benchmarks_in_parameter_scan() {
                 .and(predicate::str::contains("Benchmark 5: sleep 50").not()),
         );
 }
+
+#[test]
+fn intermediate_results_are_not_exported_to_stdout() {
+    hyperfine_debug()
+        .arg("--style=none") // To only see the Markdown export on stdout
+        .arg("--export-markdown")
+        .arg("-")
+        .arg("sleep 1")
+        .arg("sleep 2")
+        .assert()
+        .success()
+        .stdout(
+            (predicate::str::contains("sleep 1").count(1))
+                .and(predicate::str::contains("sleep 2").count(1)),
+        );
+}
+
+#[test]
+#[cfg(unix)]
+fn exports_intermediate_results_to_file() {
+    use tempfile::tempdir;
+
+    let tempdir = tempdir().unwrap();
+    let export_path = tempdir.path().join("results.md");
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--export-markdown")
+        .arg(&export_path)
+        .arg("true")
+        .arg("false")
+        .assert()
+        .failure();
+
+    let contents = std::fs::read_to_string(export_path).unwrap();
+    assert!(contents.contains("true"));
+}