Repository: ordo-one/package-benchmark (swift)

Issue:
naming for iterations and/or duration
Working through writing a few sample benchmarks, I was exploring `Benchmark.defaultConfiguration.desiredIterations`, `Benchmark.defaultConfiguration.desiredDuration`, and how they relate to each other.

I do think the names could be improved by renaming them to `maxIterations` and `maxDuration`, respectively. I'm also thinking that as we cobble the documentation, expanding on `Benchmark.Configuration` (and maybe the article `WritingBenchmarks.md`), it would be worth calling out specifically that the running will go until the first of these two are hit.

And just to double check - if you specify warmupIterations on the configuration, does that take place before either the iterations vs. duration markers are measured?

Pull request:
## Description

This fixes #65, it makes the point of use for these settings clearer.

## How Has This Been Tested?

Manual testing.

## Minimal checklist:

- [x] I have performed a self-review of my own code 
- [x] I have added `DocC` code-level documentation for any public interfaces exported by the package
- [ ] I have added unit and/or integration tests that prove my fix is effective or that my feature works

Patch:
diff --git a/Benchmarks/Basic/BenchmarkRunner.swift b/Benchmarks/Basic/BenchmarkRunner.swift
index 9f2fc0c6..054d1c6c 100644
--- a/Benchmarks/Basic/BenchmarkRunner.swift
+++ b/Benchmarks/Basic/BenchmarkRunner.swift
@@ -15,8 +15,8 @@ extension BenchmarkRunner {}
 // swiftlint disable: attributes
 @_dynamicReplacement(for: registerBenchmarks)
 func benchmarks() {
-    Benchmark.defaultConfiguration = .init(desiredDuration: .milliseconds(10),
-                                           desiredIterations: .giga(1))
+    Benchmark.defaultConfiguration = .init(maxDuration: .milliseconds(10),
+                                           maxIterations: .giga(1))
 
     Benchmark("Basic",
               configuration: .init(metrics: [.wallClock, .throughput])) { _ in
diff --git a/Benchmarks/DateTime/DateTime.swift b/Benchmarks/DateTime/DateTime.swift
index d14ca224..c05e40d2 100644
--- a/Benchmarks/DateTime/DateTime.swift
+++ b/Benchmarks/DateTime/DateTime.swift
@@ -16,8 +16,8 @@ func benchmarks() {
     Benchmark.defaultConfiguration = .init(metrics: [.throughput, .wallClock],
                                            warmupIterations: 10,
                                            throughputScalingFactor: .kilo,
-                                           desiredDuration: .seconds(1),
-                                           desiredIterations: .kilo(10))
+                                           maxDuration: .seconds(1),
+                                           maxIterations: .kilo(10))
 
     Benchmark("InternalUTCClock-now") { benchmark in
         for _ in benchmark.throughputIterations {
diff --git a/Benchmarks/Histogram/HistogramBenchmark.swift b/Benchmarks/Histogram/HistogramBenchmark.swift
index 17610af0..8ab44d70 100644
--- a/Benchmarks/Histogram/HistogramBenchmark.swift
+++ b/Benchmarks/Histogram/HistogramBenchmark.swift
@@ -17,8 +17,8 @@ extension BenchmarkRunner {}
 @_dynamicReplacement(for: registerBenchmarks)
 func benchmarks() {
     Benchmark.defaultConfiguration = .init(throughputScalingFactor: .mega,
-                                           desiredDuration: .seconds(1),
-                                           desiredIterations: .kilo(1))
+                                           maxDuration: .seconds(1),
+                                           maxIterations: .kilo(1))
 
     Benchmark("Record",
               configuration: .init(metrics: [.wallClock, .throughput] + BenchmarkMetric.memory)) { benchmark in
diff --git a/Documentation/Workflows.md b/Documentation/Workflows.md
index b3c42935..f3665549 100644
--- a/Documentation/Workflows.md
+++ b/Documentation/Workflows.md
@@ -3,9 +3,9 @@
 ## Local developer workflow
 Typical workflow for a developer who wants to track performance metrics on the local machine while during performance work, would be to store one or more baselines (either default or named baselines), with e.g:
 
-`swift package --allow-writing-to-package-directory benchmark update-baseline`
+`swift package --allow-writing-to-package-directory benchmark baseline update`
 or
-`swift package --allow-writing-to-package-directory benchmark update-baseline alpha`
+`swift package --allow-writing-to-package-directory benchmark baseline update alpha`
 
 then while working, simply compare current state of local repo with the previously recorded baseline:
 
@@ -15,7 +15,7 @@ or
 
 If you have stored multiple baselines (for example for different approaches to solving a given performance issue), you can easily compare the two approaches by using named baselines for each and then compare them:
 
-`swift package benchmark compare alpha beta`
+`swift package benchmark baseline alpha --compare beta`
 
 ### Debugging crashing benchmarks
 The benchmark executables are set up to automatically run all tests when run standalone with simple debug output - this is to enable workflows where the benchmark is run in the Xcode debugger or with Instruments if desired - or with `lldb` on the command line on Linux to support debugging in problematic performance tests.
diff --git a/Documentation/WritingBenchmarks.md b/Documentation/WritingBenchmarks.md
index a6e3b44c..16fdfa26 100644
--- a/Documentation/WritingBenchmarks.md
+++ b/Documentation/WritingBenchmarks.md
@@ -89,9 +89,9 @@ public extension Benchmark {
         /// `benchmark.throughputScalingFactor.rawvalue` for the number of iterations.
         public var throughputScalingFactor: StatisticsUnits
         /// The target wall clock runtime for the benchmark, currenty defaults to `.seconds(1)` if not set
-        public var desiredDuration: Duration
+        public var maxDuration: Duration
         /// The target number of iterations for the benchmark., currently defaults to 100K iterations if not set
-        public var desiredIterations: Int
+        public var maxIterations: Int
         /// Whether to skip this test (convenience for not having to comment out tests that have issues)
         public var skip = false
         /// Customized CI failure thresholds for a given metric for the Benchmark
diff --git a/README.md b/README.md
index 692b2b55..54e8fafa 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ func benchmarks() {
 
     Benchmark("All metrics, full concurrency, async",
               metrics: BenchmarkMetric.all,
-              desiredDuration: .seconds(10)) { benchmark in
+              maxDuration: .seconds(10)) { benchmark in
         let _ = await withTaskGroup(of: Void.self, returning: Void.self, body: { taskGroup in
             for _ in 0..< 80  {
                 taskGroup.addTask {
diff --git a/Sources/Benchmark/Benchmark.swift b/Sources/Benchmark/Benchmark.swift
index 8a12673d..638fb972 100644
--- a/Sources/Benchmark/Benchmark.swift
+++ b/Sources/Benchmark/Benchmark.swift
@@ -55,8 +55,8 @@ public final class Benchmark: Codable, Hashable {
                                                                   timeUnits: .automatic,
                                                                   warmupIterations: 3,
                                                                   throughputScalingFactor: .count,
-                                                                  desiredDuration: .seconds(1),
-                                                                  desiredIterations: 100_000,
+                                                                  maxDuration: .seconds(1),
+                                                                  maxIterations: 10_000,
                                                                   skip: false,
                                                                   thresholds: nil)
 
@@ -236,10 +236,10 @@ public extension Benchmark {
         /// E.g. `.kilo`will scale results with 1000. Any iteration done in the benchmark should use
         /// `benchmark.throughputScalingFactor.rawvalue` for the number of iterations.
         public var throughputScalingFactor: StatisticsUnits
-        /// The target wall clock runtime for the benchmark, currenty defaults to `.seconds(1)` if not set
-        public var desiredDuration: Duration
-        /// The target number of iterations for the benchmark., currently defaults to 100K iterations if not set
-        public var desiredIterations: Int
+        /// The maximum wall clock runtime for the benchmark, currenty defaults to `.seconds(1)` if not set
+        public var maxDuration: Duration
+        /// The maximum number of iterations for the benchmark., currently defaults to 10K iterations if not set
+        public var maxIterations: Int
         /// Whether to skip this test (convenience for not having to comment out tests that have issues)
         public var skip = false
         /// Customized CI failure thresholds for a given metric for the Benchmark
@@ -249,8 +249,8 @@ public extension Benchmark {
                     timeUnits: BenchmarkTimeUnits = defaultConfiguration.timeUnits,
                     warmupIterations: Int = defaultConfiguration.warmupIterations,
                     throughputScalingFactor: StatisticsUnits = defaultConfiguration.throughputScalingFactor,
-                    desiredDuration: Duration = defaultConfiguration.desiredDuration,
-                    desiredIterations: Int = defaultConfiguration.desiredIterations,
+                    maxDuration: Duration = defaultConfiguration.maxDuration,
+                    maxIterations: Int = defaultConfiguration.maxIterations,
                     skip: Bool = defaultConfiguration.skip,
                     thresholds: [BenchmarkMetric: BenchmarkResult.PercentileThresholds]? =
                         defaultConfiguration.thresholds) {
@@ -258,8 +258,8 @@ public extension Benchmark {
             self.timeUnits = timeUnits
             self.warmupIterations = warmupIterations
             self.throughputScalingFactor = throughputScalingFactor
-            self.desiredDuration = desiredDuration
-            self.desiredIterations = desiredIterations
+            self.maxDuration = maxDuration
+            self.maxIterations = maxIterations
             self.skip = skip
             self.thresholds = thresholds
         }
@@ -270,8 +270,8 @@ public extension Benchmark {
             case timeUnits
             case warmupIterations
             case throughputScalingFactor
-            case desiredDuration
-            case desiredIterations
+            case maxDuration
+            case maxIterations
             case thresholds
         }
     }
diff --git a/Sources/BenchmarkSupport/BenchmarkRunner.swift b/Sources/BenchmarkSupport/BenchmarkRunner.swift
index 012d23dd..f6dbd217 100644
--- a/Sources/BenchmarkSupport/BenchmarkRunner.swift
+++ b/Sources/BenchmarkSupport/BenchmarkRunner.swift
@@ -252,7 +252,7 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
 
                     if quiet == false {
                         let progressString = "| \(benchmarkToRun.target):\(benchmarkToRun.name)"
-                        progressBar = ProgressBar(count: benchmark.configuration.desiredIterations,
+                        progressBar = ProgressBar(count: benchmark.configuration.maxIterations,
                                                   configuration: [ProgressPercent(),
                                                                   ProgressBarLine(barLength: 60),
                                                                   ProgressTimeEstimates(),
@@ -265,11 +265,11 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
                     var currentPercentage = 0
 
                     // Run the benchmark at a minimum the desired iterations/runtime --
-                    while iterations <= benchmark.configuration.desiredIterations ||
-                        accummulatedRuntime <= benchmark.configuration.desiredDuration {
+                    while iterations <= benchmark.configuration.maxIterations ||
+                        accummulatedRuntime <= benchmark.configuration.maxDuration {
                         // and at a maximum the same...
-                        guard accummulatedRuntime < benchmark.configuration.desiredDuration,
-                              iterations < benchmark.configuration.desiredIterations
+                        guard accummulatedRuntime < benchmark.configuration.maxDuration,
+                              iterations < benchmark.configuration.maxIterations
                         else {
                             break
                         }
@@ -287,24 +287,24 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
 
                         if var progressBar {
                             let iterationsPercentage = 100.0 * Double(iterations) /
-                                Double(benchmark.configuration.desiredIterations)
+                                Double(benchmark.configuration.maxIterations)
 
                             let timePercentage = 100.0 * (accummulatedRuntime /
-                                benchmark.configuration.desiredDuration)
+                                benchmark.configuration.maxDuration)
 
                             let maxPercentage = max(iterationsPercentage, timePercentage)
 
                             if Int(maxPercentage) > currentPercentage {
                                 currentPercentage = Int(maxPercentage)
                                 progressBar.setValue(Int((maxPercentage / 100) *
-                                        Double(benchmark.configuration.desiredIterations)))
+                                        Double(benchmark.configuration.maxIterations)))
                                 fflush(stdout)
                             }
                         }
                     }
 
                     if var progressBar {
-                        progressBar.setValue(benchmark.configuration.desiredIterations)
+                        progressBar.setValue(benchmark.configuration.maxIterations)
                         fflush(stdout)
                     }
 


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
index 61334b59..0efda0f1 100644
--- a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
@@ -16,11 +16,11 @@ import XCTest
 
 @_dynamicReplacement(for: registerBenchmarks) // Register benchmarks
 func benchmarks() {
-    Benchmark("Minimal benchmark", configuration: .init(metrics: BenchmarkMetric.all, desiredIterations: 1)) { _ in
+    Benchmark("Minimal benchmark", configuration: .init(metrics: BenchmarkMetric.all, maxIterations: 1)) { _ in
     }
-    Benchmark("Minimal benchmark 2", configuration: .init(warmupIterations: 0, desiredIterations: 2)) { _ in
+    Benchmark("Minimal benchmark 2", configuration: .init(warmupIterations: 0, maxIterations: 2)) { _ in
     }
-    Benchmark("Minimal benchmark 3", configuration: .init(timeUnits: .seconds, desiredIterations: 3)) { _ in
+    Benchmark("Minimal benchmark 3", configuration: .init(timeUnits: .seconds, maxIterations: 3)) { _ in
     }
 }