Repository: google/swift-benchmark
Pull request #53 (2020-06-25 15:39:36)

Add support for custom default settings
Since #45, cli settings have highest priority and can not be used to pass sane global defaults, because they would override benchmark and suite-level settings.  We add a new optional argument to `main` to address this.


Patch:
diff --git a/Sources/Benchmark/BenchmarkArguments.swift b/Sources/Benchmark/BenchmarkArguments.swift
index 3fc8c86..b2afee5 100644
--- a/Sources/Benchmark/BenchmarkArguments.swift
+++ b/Sources/Benchmark/BenchmarkArguments.swift
@@ -138,3 +138,8 @@ public struct BenchmarkArguments: ParsableArguments {
         return "Value provided via \(flag) must be a non-negative \(type)."
     }
 }
+
+public func parseArguments() -> [BenchmarkSetting] {
+    let command = BenchmarkCommand.parseOrExit()
+    return command.arguments.settings
+}
diff --git a/Sources/Benchmark/BenchmarkMain.swift b/Sources/Benchmark/BenchmarkMain.swift
index f92f784..431d2e6 100644
--- a/Sources/Benchmark/BenchmarkMain.swift
+++ b/Sources/Benchmark/BenchmarkMain.swift
@@ -12,18 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-public func main(_ suites: [BenchmarkSuite]) {
-    let command = BenchmarkCommand.parseOrExit()
-    main(suites, settings: command.arguments.settings)
-}
-
-public func main(_ suites: [BenchmarkSuite], settings: [BenchmarkSetting]) {
+public func main(
+    _ suites: [BenchmarkSuite] = [defaultBenchmarkSuite],
+    settings: [BenchmarkSetting] = parseArguments(),
+    customDefaults: [BenchmarkSetting] = defaultSettings
+) {
     var runner = BenchmarkRunner(
         suites: suites,
-        settings: settings)
+        settings: settings,
+        customDefaults: customDefaults)
     try! runner.run()
 }
-
-public func main() {
-    main([defaultBenchmarkSuite])
-}
diff --git a/Sources/Benchmark/BenchmarkRunner.swift b/Sources/Benchmark/BenchmarkRunner.swift
index 42f1e13..4cac1ac 100644
--- a/Sources/Benchmark/BenchmarkRunner.swift
+++ b/Sources/Benchmark/BenchmarkRunner.swift
@@ -15,16 +15,23 @@
 public struct BenchmarkRunner {
     let suites: [BenchmarkSuite]
     let settings: [BenchmarkSetting]
+    let customDefaults: [BenchmarkSetting]
     let globalSettings: BenchmarkSettings
     var progress: ProgressReporter
     var reporter: BenchmarkReporter
     var results: [BenchmarkResult] = []
 
-    init(suites: [BenchmarkSuite], settings: [BenchmarkSetting]) {
+    init(
+        suites: [BenchmarkSuite],
+        settings: [BenchmarkSetting],
+        customDefaults: [BenchmarkSetting] = []
+    ) {
         self.suites = suites
         self.settings = settings
+        self.customDefaults = customDefaults
         self.globalSettings = BenchmarkSettings([
             defaultSettings,
+            self.customDefaults,
             self.settings,
         ])
         switch self.globalSettings.format {
@@ -61,6 +68,7 @@ public struct BenchmarkRunner {
     mutating func run(benchmark: AnyBenchmark, suite: BenchmarkSuite) throws {
         let settings = BenchmarkSettings([
             defaultSettings,
+            self.customDefaults,
             suite.settings,
             benchmark.settings,
             self.settings,
diff --git a/Sources/Benchmark/BenchmarkSetting.swift b/Sources/Benchmark/BenchmarkSetting.swift
index 42097ac..6bdbcfc 100644
--- a/Sources/Benchmark/BenchmarkSetting.swift
+++ b/Sources/Benchmark/BenchmarkSetting.swift
@@ -226,7 +226,7 @@ public struct BenchmarkSettings {
     }
 }
 
-let defaultSettings: [BenchmarkSetting] = [
+public let defaultSettings: [BenchmarkSetting] = [
     MaxIterations(1_000_000),
     MinTime(seconds: 1.0),
     TimeUnit(.ns),
diff --git a/Sources/Benchmark/BenchmarkSuite.swift b/Sources/Benchmark/BenchmarkSuite.swift
index dda724b..e3ada7f 100644
--- a/Sources/Benchmark/BenchmarkSuite.swift
+++ b/Sources/Benchmark/BenchmarkSuite.swift
@@ -87,4 +87,4 @@ public class BenchmarkSuite {
     }
 }
 
-var defaultBenchmarkSuite = BenchmarkSuite(name: "")
+public let defaultBenchmarkSuite = BenchmarkSuite(name: "")


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkSettingTests.swift b/Tests/BenchmarkTests/BenchmarkSettingTests.swift
index 8551a6a..833020a 100644
--- a/Tests/BenchmarkTests/BenchmarkSettingTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkSettingTests.swift
@@ -21,11 +21,13 @@ final class BenchmarkSettingTests: XCTestCase {
     func assertNumberOfIterations(
         suite: BenchmarkSuite,
         counts expected: [Int],
-        cli: [BenchmarkSetting]
+        cli: [BenchmarkSetting],
+        customDefaults: [BenchmarkSetting] = []
     ) throws {
         var settings: [BenchmarkSetting] = [Format(.none)]
         settings.append(contentsOf: cli)
-        var runner = BenchmarkRunner(suites: [suite], settings: settings)
+        var runner = BenchmarkRunner(
+            suites: [suite], settings: settings, customDefaults: customDefaults)
 
         try runner.run()
         XCTAssertEqual(runner.results.count, expected.count)
@@ -124,6 +126,54 @@ final class BenchmarkSettingTests: XCTestCase {
             cli: [Iterations(1)])
     }
 
+    func testCustomDefaults() throws {
+        let suite = BenchmarkSuite(name: "Test") { suite in
+            suite.benchmark("a") {}
+            suite.benchmark("b") {}
+        }
+        try assertNumberOfIterations(
+            suite: suite,
+            counts: [1, 1],
+            cli: [],
+            customDefaults: [Iterations(1)])
+    }
+
+    func testCustomDafaultsOverridenBySuite() throws {
+        let suite = BenchmarkSuite(name: "Test", settings: Iterations(3)) { suite in
+            suite.benchmark("a") {}
+            suite.benchmark("b") {}
+        }
+        try assertNumberOfIterations(
+            suite: suite,
+            counts: [3, 3],
+            cli: [],
+            customDefaults: [Iterations(1)])
+    }
+
+    func testCustomDafaultsOverridenByBenchmark() throws {
+        let suite = BenchmarkSuite(name: "Test", settings: Iterations(3)) { suite in
+            suite.benchmark("a") {}
+            suite.benchmark("b", settings: Iterations(4)) {}
+        }
+        try assertNumberOfIterations(
+            suite: suite,
+            counts: [3, 4],
+            cli: [],
+            customDefaults: [Iterations(1)])
+    }
+
+    func testCustomDafaultsOverridenByCli() throws {
+        let suite = BenchmarkSuite(name: "Test", settings: Iterations(3)) { suite in
+            suite.benchmark("a") {}
+            suite.benchmark("b", settings: Iterations(4)) {}
+        }
+        try assertNumberOfIterations(
+            suite: suite,
+            counts: [5, 5],
+            cli: [Iterations(5)],
+            customDefaults: [Iterations(1)])
+    }
+
     static var allTests = [
         ("testDefaultSetting", testDefaultSetting),
         ("testSuiteSetting", testSuiteSetting),
@@ -133,5 +183,9 @@ final class BenchmarkSettingTests: XCTestCase {
         ("testCliOverridesSuiteSetting", testCliOverridesSuiteSetting),
         ("testCliOverridesBenchmarkSetting", testCliOverridesBenchmarkSetting),
         ("testCliOverridesBenchmarkAndSuiteSetting", testCliOverridesBenchmarkAndSuiteSetting),
+        ("testCustomDefaults", testCustomDefaults),
+        ("testCustomDafaultsOverridenBySuite", testCustomDafaultsOverridenBySuite),
+        ("testCustomDafaultsOverridenByBenchmark", testCustomDafaultsOverridenByBenchmark),
+        ("testCustomDafaultsOverridenByCli", testCustomDafaultsOverridenByCli),
     ]
 }