Repository: google/swift-benchmark
Pull request #39 (2020-06-10 15:22:15)

Add warmup column
This change introduces a new column that shows total time taken in warmup iterations. 

Currently it's always shown, but once #15 is fixed, we will be able to show it or not dynamically based on user preference. 


Patch:
diff --git a/Sources/Benchmark/BenchmarkReporter.swift b/Sources/Benchmark/BenchmarkReporter.swift
index d2af53c..7b9a770 100644
--- a/Sources/Benchmark/BenchmarkReporter.swift
+++ b/Sources/Benchmark/BenchmarkReporter.swift
@@ -27,6 +27,7 @@ struct PlainTextReporter<Target>: BenchmarkReporter where Target: TextOutputStre
         case std
         case iterations
         case counter(String)
+        case warmup
     }
 
     func title(_ column: Column) -> String {
@@ -36,6 +37,7 @@ struct PlainTextReporter<Target>: BenchmarkReporter where Target: TextOutputStre
         case .std: return "std"
         case .iterations: return "iterations"
         case .counter(let name): return name
+        case .warmup: return "warmup"
         }
     }
 
@@ -66,14 +68,19 @@ struct PlainTextReporter<Target>: BenchmarkReporter where Target: TextOutputStre
     }
 
     mutating func report(results: [BenchmarkResult]) {
+        var showWarmup = false
         var counters: Set<String> = Set()
         for result in results {
+            showWarmup = showWarmup || result.warmupMeasurements.count > 0
             for name in result.counters.keys {
                 counters.insert(name)
             }
         }
 
         var columns: [Column] = [.name, .time, .std, .iterations]
+        if showWarmup {
+            columns.append(.warmup)
+        }
         for counter in counters {
             columns.append(.counter(counter))
         }
@@ -96,6 +103,10 @@ struct PlainTextReporter<Target>: BenchmarkReporter where Target: TextOutputStre
                 .std: "± \(String(format: "%6.2f %%", (std / median) * 100))",
                 .iterations: "\(result.measurements.count)",
             ]
+            if showWarmup {
+                let warmup = result.warmupMeasurements
+                row[.warmup] = warmup.count > 0 ? "\(warmup.sum) ns" : ""
+            }
             for counter in counters {
                 if let value = result.counters[counter] {
                     row[.counter(counter)] = "\(value)"
@@ -128,7 +139,7 @@ struct PlainTextReporter<Target>: BenchmarkReporter where Target: TextOutputStre
                 case _ where index == 0,
                     .name, .std:
                     return string.rightPadding(toLength: width, withPad: " ")
-                case .time, .iterations, .counter(_):
+                case .time, .iterations, .counter(_), .warmup:
                     return string.leftPadding(toLength: width, withPad: " ")
                 }
             }
diff --git a/Sources/Benchmark/BenchmarkResult.swift b/Sources/Benchmark/BenchmarkResult.swift
index 763c003..3d860bc 100644
--- a/Sources/Benchmark/BenchmarkResult.swift
+++ b/Sources/Benchmark/BenchmarkResult.swift
@@ -16,14 +16,17 @@ public struct BenchmarkResult {
     public let benchmarkName: String
     public let suiteName: String
     public let measurements: [Double]
+    public let warmupMeasurements: [Double]
     public let counters: [String: Double]
 
     public init(
-        benchmarkName: String, suiteName: String, measurements: [Double], counters: [String: Double]
+        benchmarkName: String, suiteName: String, measurements: [Double],
+        warmupMeasurements: [Double], counters: [String: Double]
     ) {
         self.benchmarkName = benchmarkName
         self.suiteName = suiteName
         self.measurements = measurements
+        self.warmupMeasurements = warmupMeasurements
         self.counters = counters
     }
 }
diff --git a/Sources/Benchmark/BenchmarkRunner.swift b/Sources/Benchmark/BenchmarkRunner.swift
index b25d733..045171b 100644
--- a/Sources/Benchmark/BenchmarkRunner.swift
+++ b/Sources/Benchmark/BenchmarkRunner.swift
@@ -53,8 +53,9 @@ public struct BenchmarkRunner {
         reporter.report(running: benchmark.name, suite: suite.name)
         let totalStart = now()
 
+        var warmupState: BenchmarkState? = nil
         if let n = settings.warmupIterations {
-            let _ = doNIterations(n, benchmark: benchmark, suite: suite, settings: settings)
+            warmupState = doNIterations(n, benchmark: benchmark, suite: suite, settings: settings)
         }
 
         var state: BenchmarkState
@@ -75,6 +76,7 @@ public struct BenchmarkRunner {
             benchmarkName: benchmark.name,
             suiteName: suite.name,
             measurements: state.measurements,
+            warmupMeasurements: warmupState != nil ? warmupState!.measurements : [],
             counters: state.counters)
         results.append(result)
     }


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkReporterTests.swift b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
index 454880e..2c2ace4 100644
--- a/Tests/BenchmarkTests/BenchmarkReporterTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
@@ -34,10 +34,14 @@ final class BenchmarkReporterTests: XCTestCase {
     func testPlainTextReporter() throws {
         let results: [BenchmarkResult] = [
             BenchmarkResult(
-                benchmarkName: "fast", suiteName: "My Suite", measurements: [1_000, 2_000],
+                benchmarkName: "fast", suiteName: "My Suite",
+                measurements: [1_000, 2_000],
+                warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
-                benchmarkName: "slow", suiteName: "My Suite", measurements: [1_000_000, 2_000_000],
+                benchmarkName: "slow", suiteName: "My Suite",
+                measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
                 counters: [:]),
         ]
         let expected = #"""
@@ -47,16 +51,17 @@ final class BenchmarkReporterTests: XCTestCase {
             My Suite: slow 1500000.0 ns ±  47.14 %          2
             """#
         assertIsPrintedAs(results, expected)
-
     }
 
     func testCountersAreReported() throws {
         let results: [BenchmarkResult] = [
             BenchmarkResult(
                 benchmarkName: "fast", suiteName: "My Suite", measurements: [1_000, 2_000],
+                warmupMeasurements: [],
                 counters: ["n": 7]),
             BenchmarkResult(
                 benchmarkName: "slow", suiteName: "My Suite", measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
                 counters: [:]),
         ]
         let expected = #"""
@@ -66,11 +71,31 @@ final class BenchmarkReporterTests: XCTestCase {
             My Suite: slow 1500000.0 ns ±  47.14 %          2
             """#
         assertIsPrintedAs(results, expected)
+    }
+
+    func testWarmupReported() throws {
+        let results: [BenchmarkResult] = [
+            BenchmarkResult(
+                benchmarkName: "fast", suiteName: "My Suite", measurements: [1_000, 2_000],
+                warmupMeasurements: [10, 20, 30],
+                counters: [:]),
+            BenchmarkResult(
+                benchmarkName: "slow", suiteName: "My Suite", measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
+                counters: [:]),
+        ]
+        let expected = #"""
+            name           time         std        iterations warmup
+            ---------------------------------------------------------
+            My Suite: fast    1500.0 ns ±  47.14 %          2 60.0 ns
+            My Suite: slow 1500000.0 ns ±  47.14 %          2
+            """#
         assertIsPrintedAs(results, expected)
     }
 
     static var allTests = [
         ("testPlainTextReporter", testPlainTextReporter),
         ("testCountersAreReported", testCountersAreReported),
+        ("testWarmupReported", testWarmupReported),
     ]
 }
diff --git a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
index 2c5791d..5c3e505 100644
--- a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
@@ -39,6 +39,21 @@ final class BenchmarkRunnerTests: XCTestCase {
             runBenchmarks(settings: settings))
     }
 
+    func testWarmupMeasurements() throws {
+        let suite = BenchmarkSuite(name: "Suite")
+
+        suite.benchmark("no warmup") {
+        }
+        suite.benchmark("needs warmup", settings: WarmupIterations(10)) {
+        }
+
+        let results = try run(suites: [suite], settings: [Iterations(100)])
+        XCTAssertEqual(results.count, 2)
+
+        let counts = results.map { $0.warmupMeasurements.count }
+        XCTAssertEqual(counts, [0, 10])
+    }
+
     func testCustomMeasurements() throws {
         let suite = BenchmarkSuite(name: "Suite")
 
@@ -101,6 +116,7 @@ final class BenchmarkRunnerTests: XCTestCase {
         ("testFilterBenchmarksSuiteName", testFilterBenchmarksSuiteName),
         ("testFilterBenchmarksFullName", testFilterBenchmarksFullName),
         ("testAutomaticallyDetectIterations", testAutomaticallyDetectIterations),
+        ("testWarmupMeasurements", testWarmupMeasurements),
         ("testCustomMeasurements", testCustomMeasurements),
         ("testCustomCounters", testCustomCounters),
     ]