Repository: ordo-one/package-benchmark (swift)

Issue:
Feature: Custom benchmark tags
I want to test the waters on whether the community would be receptive to a PR that adds the ability to apply custom tags to a benchmark.
This specifically targets a use case with Influx where I want to run the same benchmark multiple times under different constraints/scenarios.

For example:
```
for size in [10, 100, 1000] {
    Benchmark("TestFoo", tags: [ "size": size ]) {
        foo(size)
    }
}
```

This would allow me to get better dimensionality on a benchmark that, when exported to Influx, is more easily queried etc.
This would require some amendments to the Influx format exporter to include these custom tags for the particular benchmark.

We (myself and/or @ORyanHampton) are happy to take this PR on - just don't want to spend time on it if it's not a direction that the community finds appropriate.

Pull request description:
## Description

Sometimes it is convenient to run the same benchmark multiple times under slightly different scenarios. For example, benchmarking `func foo(_ n: Int)` for various parameterizations of `n`. This adds support for specifying the parameters of a benchmark but makes no functional changes beyond the name/identifier of a benchmark now including a parameter list when one was provided.

This unlocks the potential for the benchmark exporters (such as the Influx exporter) to leverage these parameters (by emitting appropriate tags/fields in the case of Influx) in their output.

Resolves https://github.com/ordo-one/package-benchmark/issues/267

## How Has This Been Tested?

Added a new test to verify the output of `Benchmark.name`.
Also ran the remaining tests to make sure no other behavior changed.

## Minimal checklist:

- [x] I have performed a self-review of my own code 
- [x] I have added `DocC` code-level documentation for any public interfaces exported by the package
- [x] I have added unit and/or integration tests that prove my fix is effective or that my feature works

Patch:
diff --git a/Plugins/BenchmarkTool/BenchmarkTool.swift b/Plugins/BenchmarkTool/BenchmarkTool.swift
index af26196b..068409ae 100644
--- a/Plugins/BenchmarkTool/BenchmarkTool.swift
+++ b/Plugins/BenchmarkTool/BenchmarkTool.swift
@@ -268,7 +268,7 @@ struct BenchmarkTool: AsyncParsableCommand {
 
         // run each benchmark for the target as a separate process
         try benchmarks.forEach { benchmark in
-            if try shouldIncludeBenchmark(benchmark.name) {
+            if try shouldIncludeBenchmark(benchmark.baseName) {
                 let results = try runChild(benchmarkPath: benchmark.executablePath!,
                                            benchmarkCommand: command,
                                            benchmark: benchmark) { [self] result in
diff --git a/Sources/Benchmark/Benchmark.swift b/Sources/Benchmark/Benchmark.swift
index 94ba1287..79f2be93 100644
--- a/Sources/Benchmark/Benchmark.swift
+++ b/Sources/Benchmark/Benchmark.swift
@@ -10,10 +10,10 @@
 
 import Dispatch
 
-// swiftlint: disable file_length
+// swiftlint:disable file_length
 
 /// Defines a benchmark
-public final class Benchmark: Codable, Hashable {
+public final class Benchmark: Codable, Hashable { // swiftlint:disable:this type_body_length
     #if swift(>=5.8)
         @_documentation(visibility: internal)
     #endif
@@ -69,8 +69,28 @@ public final class Benchmark: Codable, Hashable {
     #endif
     public static var benchmarks: [Benchmark] = [] // Bookkeeping of all registered benchmarks
 
+    /// The name of the benchmark without any of the tags appended
+    public var baseName: String
+
     /// The name used for display purposes of the benchmark (also used for matching when comparing to baselines)
-    public var name: String
+    public var name: String {
+        get {
+            if configuration.tags.isEmpty {
+                return baseName
+            } else {
+                return baseName
+                    + " ("
+                    + configuration.tags
+                        .sorted(by: { $0.key < $1.key })
+                        .map({ "\($0.key): \($0.value)" })
+                        .joined(separator: ", ")
+                    + ")"
+            }
+        }
+        set {
+            baseName = newValue
+        }
+    }
 
     /// The reason for a benchmark failure, not set if successful
     public var failureReason: String?
@@ -119,6 +139,7 @@ public final class Benchmark: Codable, Hashable {
 
     /// Hook for setting defaults for a whole benchmark suite
     public static var defaultConfiguration: Configuration = .init(metrics: BenchmarkMetric.default,
+                                                                  tags: [:],
                                                                   timeUnits: .automatic,
                                                                   warmupIterations: 1,
                                                                   scalingFactor: .one,
@@ -131,7 +152,7 @@ public final class Benchmark: Codable, Hashable {
     var measurementCompleted = false // Keep track so we skip multiple 'end of measurement'
 
     enum CodingKeys: String, CodingKey {
-        case name
+        case baseName = "name"
         case target
         case executablePath
         case configuration
@@ -168,7 +189,7 @@ public final class Benchmark: Codable, Hashable {
             return nil
         }
         target = ""
-        self.name = name
+        self.baseName = name
         self.configuration = configuration
         self.closure = closure
         self.setup = setup
@@ -193,7 +214,7 @@ public final class Benchmark: Codable, Hashable {
             return nil
         }
         target = ""
-        self.name = name
+        self.baseName = name
         self.configuration = configuration
         asyncClosure = closure
         self.setup = setup
@@ -362,6 +383,8 @@ public extension Benchmark {
     struct Configuration: Codable {
         /// Defines the metrics that should be measured for the benchmark
         public var metrics: [BenchmarkMetric]
+        /// Specifies the parameters used to define the benchmark.
+        public var tags: [String: String]
         /// Override the automatic detection of timeunits for metrics related to time to a specific
         /// one (auto should work for most use cases)
         public var timeUnits: BenchmarkTimeUnits
@@ -386,6 +409,7 @@ public extension Benchmark {
         public var teardown: BenchmarkTeardownHook?
 
         public init(metrics: [BenchmarkMetric] = defaultConfiguration.metrics,
+                    tags: [String: String] = defaultConfiguration.tags,
                     timeUnits: BenchmarkTimeUnits = defaultConfiguration.timeUnits,
                     warmupIterations: Int = defaultConfiguration.warmupIterations,
                     scalingFactor: BenchmarkScalingFactor = defaultConfiguration.scalingFactor,
@@ -397,6 +421,7 @@ public extension Benchmark {
                     setup: BenchmarkSetupHook? = nil,
                     teardown: BenchmarkTeardownHook? = nil) {
             self.metrics = metrics
+            self.tags = tags
             self.timeUnits = timeUnits
             self.warmupIterations = warmupIterations
             self.scalingFactor = scalingFactor
@@ -411,6 +436,7 @@ public extension Benchmark {
         // swiftlint:disable nesting
         enum CodingKeys: String, CodingKey {
             case metrics
+            case tags
             case timeUnits
             case warmupIterations
             case scalingFactor
@@ -447,3 +473,4 @@ public extension Benchmark {
     @_optimize(none) // Used after tip here: https://forums.swift.org/t/compiler-swallows-blackhole/64305/10 - see also https://github.com/apple/swift/commit/1fceeab71e79dc96f1b6f560bf745b016d7fcdcf
     static func blackHole(_: some Any) {}
 }
+// swiftlint:enable file_length
diff --git a/Sources/Benchmark/BenchmarkExecutor.swift b/Sources/Benchmark/BenchmarkExecutor.swift
index 8b2abb96..f1da29f2 100644
--- a/Sources/Benchmark/BenchmarkExecutor.swift
+++ b/Sources/Benchmark/BenchmarkExecutor.swift
@@ -444,6 +444,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length
                                                      scalingFactor: benchmark.configuration.scalingFactor,
                                                      warmupIterations: benchmark.configuration.warmupIterations,
                                                      thresholds: benchmark.configuration.thresholds?[metric],
+                                                     tags: benchmark.configuration.tags,
                                                      statistics: value)
                         results.append(result)
                     }
@@ -457,6 +458,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length
                                                      scalingFactor: benchmark.configuration.scalingFactor,
                                                      warmupIterations: benchmark.configuration.warmupIterations,
                                                      thresholds: benchmark.configuration.thresholds?[metric],
+                                                     tags: benchmark.configuration.tags,
                                                      statistics: value)
                         results.append(result)
                     }
diff --git a/Sources/Benchmark/BenchmarkResult.swift b/Sources/Benchmark/BenchmarkResult.swift
index a0ba6255..09a6a855 100644
--- a/Sources/Benchmark/BenchmarkResult.swift
+++ b/Sources/Benchmark/BenchmarkResult.swift
@@ -161,12 +161,14 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
                 scalingFactor: BenchmarkScalingFactor,
                 warmupIterations: Int,
                 thresholds: BenchmarkThresholds? = nil,
+                tags: [String: String] = [:],
                 statistics: Statistics) {
         self.metric = metric
         self.timeUnits = timeUnits == .automatic ? BenchmarkTimeUnits(statistics.units()) : timeUnits
         self.scalingFactor = scalingFactor
         self.warmupIterations = warmupIterations
         self.thresholds = thresholds
+        self.tags = tags
         self.statistics = statistics
     }
 
@@ -175,6 +177,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
     public var scalingFactor: BenchmarkScalingFactor
     public var warmupIterations: Int
     public var thresholds: BenchmarkThresholds?
+    public var tags: [String: String]
     public var statistics: Statistics
 
     public var scaledTimeUnits: BenchmarkTimeUnits {


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkTests.swift b/Tests/BenchmarkTests/BenchmarkTests.swift
index fd9049b0..43e9769b 100644
--- a/Tests/BenchmarkTests/BenchmarkTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkTests.swift
@@ -73,4 +73,17 @@ final class BenchmarkTests: XCTestCase {
         XCTAssertNotNil(benchmark)
         benchmark?.run()
     }
+
+    func testBenchmarkParameterizedDescription() throws {
+        let benchmark = Benchmark("testBenchmarkParameterizedDescription benchmark",
+                                  configuration: .init(
+                                    tags: [
+                                        "foo": "bar",
+                                        "bin": String(42),
+                                        "pi": String(3.14)
+                                    ]
+                                  )) { _ in }
+        XCTAssertNotNil(benchmark)
+        XCTAssertEqual(benchmark?.name, "testBenchmarkParameterizedDescription benchmark (bin: 42, foo: bar, pi: 3.14)")
+    }
 }