Repository: ordo-one/package-benchmark (swift)

Issue:
Detect and handle multiple identical Benchmark definitions
Easy to do mistake when copy/pasting benchmarks, we should fatalError() so it's not silently discarded as it currently is.

Pull request description:
Fixes https://github.com/ordo-one/package-benchmark/issues/18
Fixes https://github.com/ordo-one/package-benchmark/issues/19

```
@_dynamicReplacement(for: registerBenchmarks)
func benchmarks() {

    Benchmark.defaultBenchmarkTimeUnits = .nanoseconds

    Benchmark("Foundation Date()") {
...
    }
```

Patch:
diff --git a/Sources/Benchmark/Benchmark.swift b/Sources/Benchmark/Benchmark.swift
index db5a4d43..61e47ab7 100644
--- a/Sources/Benchmark/Benchmark.swift
+++ b/Sources/Benchmark/Benchmark.swift
@@ -68,6 +68,9 @@ public class Benchmark: Codable, Hashable {
     // Hook for custom metrics capturing
     public var customMetricMeasurement: BenchmarkCustomMetricMeasurement?
 
+    public static var defaultBenchmarkTimeUnits: BenchmarkTimeUnits = .automatic
+    internal static var testSkipBenchmarkRegistrations = false // true in test to avoid bench registration fail
+
     var lock: pthread_mutex_t = .init()
     var condition: pthread_cond_t = .init()
     var shutdown = false
@@ -112,7 +115,7 @@ public class Benchmark: Codable, Hashable {
     @discardableResult
     public init?(_ name: String,
                  metrics: [BenchmarkMetric] = BenchmarkMetric.default,
-                 timeUnits: BenchmarkTimeUnits? = .automatic,
+                 timeUnits: BenchmarkTimeUnits? = defaultBenchmarkTimeUnits,
                  warmup: Bool = true,
                  throughputScalingFactor: StatisticsUnits = .count,
                  desiredDuration: TimeDuration? = nil,
@@ -133,7 +136,13 @@ public class Benchmark: Codable, Hashable {
         self.thresholds = thresholds
         self.closure = closure
 
-        Self.benchmarks.append(self)
+        if Self.testSkipBenchmarkRegistrations == false {
+            guard Self.benchmarks.contains(self) == false else {
+                fatalError("Duplicate registration of benchmark '\(self.name)', name must be unique.")
+            }
+
+            Self.benchmarks.append(self)
+        }
 
         self.thresholds?.forEach { thresholdMetric, _ in
             if self.metrics.contains(thresholdMetric) == false {
@@ -167,7 +176,7 @@ public class Benchmark: Codable, Hashable {
     @discardableResult
     public init?(_ name: String,
                  metrics: [BenchmarkMetric] = BenchmarkMetric.default,
-                 timeUnits: BenchmarkTimeUnits? = .automatic,
+                 timeUnits: BenchmarkTimeUnits? = defaultBenchmarkTimeUnits,
                  warmup: Bool = true,
                  throughputScalingFactor: StatisticsUnits = .count,
                  desiredDuration: TimeDuration? = nil,


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
index 1c2b7d3f..35d0cbae 100644
--- a/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkRunnerTests.swift
@@ -36,6 +36,7 @@ final class BenchmarkRunnerTests: XCTestCase, BenchmarkRunnerReadWrite {
 
     internal func read() throws -> BenchmarkCommandRequest {
         //      print("read request")
+        Benchmark.testSkipBenchmarkRegistrations = true
         let benchmark = Benchmark("Minimal benchmark") { _ in
         }
         let benchmark2 = Benchmark("Minimal benchmark 2") { _ in
diff --git a/Tests/BenchmarkTests/BenchmarkTests.swift b/Tests/BenchmarkTests/BenchmarkTests.swift
index a726e28d..50d678e3 100644
--- a/Tests/BenchmarkTests/BenchmarkTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkTests.swift
@@ -14,7 +14,7 @@ import XCTest
 
 final class BenchmarkTests: XCTestCase {
     func testBenchmarkRun() throws {
-        let benchmark = Benchmark("Minimal benchmark") { _ in
+        let benchmark = Benchmark("testBenchmarkRun benchmark") { _ in
         }
         XCTAssertNotNil(benchmark)
         benchmark?.run()
@@ -22,7 +22,7 @@ final class BenchmarkTests: XCTestCase {
 
     func testBenchmarkRunAsync() throws {
         func asyncFunc() async {}
-        let benchmark = Benchmark("Minimal async benchmark") { _ in
+        let benchmark = Benchmark("testBenchmarkRunAsync benchmark") { _ in
             await asyncFunc()
         }
         XCTAssertNotNil(benchmark)
@@ -30,7 +30,8 @@ final class BenchmarkTests: XCTestCase {
     }
 
     func testBenchmarkRunCustomMetric() throws {
-        let benchmark = Benchmark("Minimal benchmark", metrics: [.custom("customMetric")]) { benchmark in
+        let benchmark = Benchmark("testBenchmarkRunCustomMetric benchmark",
+                                  metrics: [.custom("customMetric")]) { benchmark in
             for measurement in 1 ... 100 {
                 benchmark.measurement(.custom("customMetric"), measurement)
             }
@@ -40,18 +41,16 @@ final class BenchmarkTests: XCTestCase {
     }
 
     func testBenchmarkEqualityAndDifference() throws {
-        let benchmark = Benchmark("Minimal benchmark") { _ in
+        let benchmark = Benchmark("testBenchmarkEqualityAndDifference benchmark") { _ in
         }
-        let benchmark2 = Benchmark("Another minimal benchmark") { _ in
-        }
-        let benchmark3 = Benchmark("Minimal benchmark") { _ in
+        let benchmark2 = Benchmark("testBenchmarkEqualityAndDifference benchmark 2") { _ in
         }
         XCTAssert(benchmark != benchmark2)
-        XCTAssert(benchmark == benchmark3)
     }
 
     func testBenchmarkRunFailure() throws {
-        let benchmark = Benchmark("Minimal benchmark", metrics: [.custom("customMetric")]) { benchmark in
+        let benchmark = Benchmark("testBenchmarkRunFailure benchmark",
+                                  metrics: [.custom("customMetric")]) { benchmark in
             benchmark.error("Benchmark failed")
         }
         XCTAssertNotNil(benchmark)
@@ -61,7 +60,7 @@ final class BenchmarkTests: XCTestCase {
     }
 
     func testBenchmarkRunMoreParameters() throws {
-        let benchmark = Benchmark("Minimal benchmark",
+        let benchmark = Benchmark("testBenchmarkRunMoreParameters benchmark",
                                   metrics: BenchmarkMetric.all,
                                   timeUnits: .milliseconds,
                                   warmup: false,