Repository: google/swift-benchmark
Pull request #57 (2020-06-30 12:10:32)

Add support for mixed columns
This change makes it possible to define which columns are reported on per-benchmark and per-suite basis:

* In columnar output (Console, CSV), a superset of all requested columns is shown, where only requested benchmark/column cells are non-blank.
* In JSON output, only the columns which were requested are shown in the output. 


Patch:
diff --git a/Package.swift b/Package.swift
index e85c526..800f3fc 100644
--- a/Package.swift
+++ b/Package.swift
@@ -36,6 +36,9 @@ let package = Package(
         .target(
             name: "BenchmarkSuiteExample",
             dependencies: ["Benchmark"]),
+        .target(
+            name: "BenchmarkSuiteExampleMain",
+            dependencies: ["Benchmark", "BenchmarkSuiteExample"]),
         .testTarget(
             name: "BenchmarkTests",
             dependencies: ["Benchmark", "BenchmarkSuiteExample"]),
diff --git a/README.md b/README.md
index 976ac1b..68d5d31 100644
--- a/README.md
+++ b/README.md
@@ -27,18 +27,26 @@ Example:
 ```terminal
 $ swift run -c release BenchmarkMinimalExample -h
 [3/3] Linking BenchmarkMinimalExample
-USAGE: benchmark-command [--allow-debug-build] [--filter <filter>] [--iterations <iterations>] [--warmup-iterations <warmup-iterations>] [--min-time <min-time>] [--max-iterations <max-iterations>]
+USAGE: benchmark-command [--allow-debug-build] [--filter <filter>] [--filter-not <filter-not>] [--iterations <iterations>] [--warmup-iterations <warmup-iterations>] [--min-time <min-time>] [--max-iterations <max-iterations>] [--time-unit <time-unit>] [--inverse-time-unit <inverse-time-unit>] [--columns <columns>] [--format <format>]
 
 OPTIONS:
   --allow-debug-build     Overrides check to verify optimized build.
   --filter <filter>       Run only benchmarks whose names match the regular expression.
+  --filter-not <filter-not>
+                          Exclude benchmarks whose names match the regular expression.
   --iterations <iterations>
                           Number of iterations to run.
   --warmup-iterations <warmup-iterations>
                           Number of warm-up iterations to run.
   --min-time <min-time>   Minimal time to run when automatically detecting number iterations.
   --max-iterations <max-iterations>
-                          Maximum number of iterations to run when automatically detecting number iterations.
+                          Maximum number of iterations to run when automatically detecting number
+                          iterations.
+  --time-unit <time-unit> Time unit used to report the timing results.
+  --inverse-time-unit <inverse-time-unit>
+                          Inverse time unit used to report throughput results.
+  --columns <columns>     Comma-separated list of column names to show.
+  --format <format>       Output format (valid values are: json, csv, console, none).
   -h, --help              Show help information.
 
 $ swift run -c release BenchmarkMinimalExample
diff --git a/Sources/Benchmark/BenchmarkArguments.swift b/Sources/Benchmark/BenchmarkArguments.swift
index b2afee5..e800a44 100644
--- a/Sources/Benchmark/BenchmarkArguments.swift
+++ b/Sources/Benchmark/BenchmarkArguments.swift
@@ -23,6 +23,9 @@ public struct BenchmarkArguments: ParsableArguments {
     @Option(help: "Run only benchmarks whose names match the regular expression.")
     var filter: String?
 
+    @Option(help: "Exclude benchmarks whose names match the regular expression.")
+    var filterNot: String?
+
     @Option(help: "Number of iterations to run.")
     var iterations: Int?
 
@@ -57,6 +60,9 @@ public struct BenchmarkArguments: ParsableArguments {
         if let value = filter {
             result.append(Filter(value))
         }
+        if let value = filterNot {
+            result.append(FilterNot(value))
+        }
         if let value = iterations {
             result.append(Iterations(value))
         }
diff --git a/Sources/Benchmark/BenchmarkColumn.swift b/Sources/Benchmark/BenchmarkColumn.swift
index 86e0bed..bbef06b 100644
--- a/Sources/Benchmark/BenchmarkColumn.swift
+++ b/Sources/Benchmark/BenchmarkColumn.swift
@@ -49,7 +49,7 @@ public struct BenchmarkColumn: Hashable {
         value: @escaping (BenchmarkResult) -> Double,
         unit: Unit = .none,
         alignment: Alignment = .right,
-        formatter optionalFormatter: Formatter? = nil 
+        formatter optionalFormatter: Formatter? = nil
     ) {
         self.name = name
         self.value = value
@@ -214,39 +214,31 @@ public struct BenchmarkColumn: Hashable {
         return columns
     }
 
-    /// Evaluate all cells for all columns over all results, using
-    /// columns defined in settings or defaults otherwise.
-    static func evaluate(results: [BenchmarkResult], settings: BenchmarkSettings, pretty: Bool) -> (
-        [Row], [BenchmarkColumn]
-    ) {
-        var columns: [BenchmarkColumn] = []
-        if let names = settings.columns {
-            for name in names {
-                columns.append(BenchmarkColumn.registry[name]!)
-            }
-        } else {
-            columns = BenchmarkColumn.defaults(results: results)
-        }
-
-        let rows = BenchmarkColumn.evaluate(columns: columns, results: results, pretty: pretty)
-
-        return (rows, columns)
-    }
-
     /// Evaluate all cells for all columns over all results. 
     /// Pretty argument specifies if output is meant to be human-readable.
-    static func evaluate(columns: [BenchmarkColumn], results: [BenchmarkResult], pretty: Bool)
-        -> [Row]
+    static func evaluate(results: [BenchmarkResult], pretty: Bool)
+        -> ([Row], [BenchmarkColumn])
     {
+        var allColumns: [BenchmarkColumn] = []
         var header: Row = [:]
-        for column in columns {
-            header[column] = column.name
-        }
-
-        var rows: [Row] = [header]
+        var rows: [Row] = []
         for result in results {
+            var columns: [BenchmarkColumn] = []
+            if let names = result.settings.columns {
+                for name in names {
+                    columns.append(BenchmarkColumn.registry[name]!)
+                }
+            } else {
+                columns = BenchmarkColumn.defaults(results: results)
+            }
+
             var row: Row = [:]
             for column in columns {
+                if header[column] == nil {
+                    header[column] = column.name
+                    allColumns.append(column)
+                }
+
                 var content: String
                 if column.name == "name" {
                     if result.suiteName != "" {
@@ -286,6 +278,8 @@ public struct BenchmarkColumn: Hashable {
             rows.append(row)
         }
 
-        return rows
+        var result: [Row] = [header]
+        result.append(contentsOf: rows)
+        return (result, allColumns)
     }
 }
diff --git a/Sources/Benchmark/BenchmarkFilter.swift b/Sources/Benchmark/BenchmarkFilter.swift
index 301f99d..a0097f7 100644
--- a/Sources/Benchmark/BenchmarkFilter.swift
+++ b/Sources/Benchmark/BenchmarkFilter.swift
@@ -15,9 +15,11 @@
 import Foundation
 
 internal struct BenchmarkFilter {
+    let negate: Bool
     let underlying: NSRegularExpression?
 
-    init(_ regularExpression: String?) throws {
+    init(_ regularExpression: String?, negate: Bool) throws {
+        self.negate = negate
         if regularExpression != nil {
             self.underlying = try NSRegularExpression(
                 pattern: regularExpression!,
@@ -33,7 +35,9 @@ internal struct BenchmarkFilter {
         } else {
             let str = "\(suiteName).\(benchmarkName)"
             let range = NSRange(location: 0, length: str.utf16.count)
-            return underlying!.firstMatch(in: str, range: range) != nil
+            var result = underlying!.firstMatch(in: str, range: range) != nil
+            result = negate ? !result : result
+            return result
         }
     }
 }
diff --git a/Sources/Benchmark/BenchmarkReporter.swift b/Sources/Benchmark/BenchmarkReporter.swift
index 7b77ab1..161ff3f 100644
--- a/Sources/Benchmark/BenchmarkReporter.swift
+++ b/Sources/Benchmark/BenchmarkReporter.swift
@@ -20,7 +20,7 @@ protocol ProgressReporter {
 }
 
 protocol BenchmarkReporter {
-    mutating func report(results: [BenchmarkResult], settings: BenchmarkSettings)
+    mutating func report(results: [BenchmarkResult])
 }
 
 struct VerboseProgressReporter<Output: FlushableTextOutputStream>: ProgressReporter {
@@ -51,7 +51,7 @@ struct VerboseProgressReporter<Output: FlushableTextOutputStream>: ProgressRepor
 struct QuietReporter: ProgressReporter, BenchmarkReporter {
     mutating func report(running name: String, suite: String) {}
     mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64) {}
-    mutating func report(results: [BenchmarkResult], settings: BenchmarkSettings) {}
+    mutating func report(results: [BenchmarkResult]) {}
 }
 
 struct ConsoleReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
@@ -61,9 +61,8 @@ struct ConsoleReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
         self.output = output
     }
 
-    mutating func report(results: [BenchmarkResult], settings: BenchmarkSettings) {
-        let (rows, columns) = BenchmarkColumn.evaluate(
-            results: results, settings: settings, pretty: true)
+    mutating func report(results: [BenchmarkResult]) {
+        let (rows, columns) = BenchmarkColumn.evaluate(results: results, pretty: true)
 
         let widths: [BenchmarkColumn: Int] = Dictionary(
             uniqueKeysWithValues:
@@ -80,7 +79,12 @@ struct ConsoleReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
         print("", to: &output)
         for (index, row) in rows.enumerated() {
             let components: [String] = columns.compactMap { column in
-                let string = row[column]!
+                var string: String
+                if let value = row[column] {
+                    string = value
+                } else {
+                    string = ""
+                }
                 let width = widths[column]!
                 let alignment = index == 0 ? .left : column.alignment
                 switch alignment {
@@ -108,12 +112,17 @@ struct CSVReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
         self.output = output
     }
 
-    mutating func report(results: [BenchmarkResult], settings: BenchmarkSettings) {
-        let (rows, columns) = BenchmarkColumn.evaluate(
-            results: results, settings: settings, pretty: false)
+    mutating func report(results: [BenchmarkResult]) {
+        let (rows, columns) = BenchmarkColumn.evaluate(results: results, pretty: false)
 
         for row in rows {
-            let components: [String] = columns.compactMap { row[$0]! }
+            let components: [String] = columns.compactMap {
+                if let value = row[$0] {
+                    return value
+                } else {
+                    return ""
+                }
+            }
             let escaped = components.map { component -> String in
                 if component.contains(",") || component.contains("\"") || component.contains("\n") {
                     let escaped = component.replacingOccurrences(of: "\"", with: "\"\"")
@@ -135,9 +144,8 @@ struct JSONReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
         self.output = output
     }
 
-    mutating func report(results: [BenchmarkResult], settings: BenchmarkSettings) {
-        let (rows, columns) = BenchmarkColumn.evaluate(
-            results: results, settings: settings, pretty: false)
+    mutating func report(results: [BenchmarkResult]) {
+        let (rows, columns) = BenchmarkColumn.evaluate(results: results, pretty: false)
 
         print("{", to: &output)
         print("  \"benchmarks\": [", to: &output)
@@ -151,13 +159,22 @@ struct JSONReporter<Output: FlushableTextOutputStream>: BenchmarkReporter {
                     // performance to ensure that output properly
                     // escapes special characters that could be
                     // present within the benchmark name.
-                    let name = row[column]!
+                    let name: String
+                    if let value = row[column] {
+                        name = value
+                    } else {
+                        name = ""
+                    }
                     let data = try! JSONSerialization.data(withJSONObject: [name])
                     var encoded = String(data: data, encoding: .utf8)!
                     encoded = String(encoded.dropFirst().dropLast())
                     rhs = encoded
                 } else {
-                    rhs = String(row[column]!)
+                    if let value = row[column] {
+                        rhs = String(value)
+                    } else {
+                        continue
+                    }
                 }
                 let suffix = columnIndex != columns.count - 1 ? "," : ""
                 print("      \"\(column.name)\": \(rhs)\(suffix)", to: &output)
diff --git a/Sources/Benchmark/BenchmarkRunner.swift b/Sources/Benchmark/BenchmarkRunner.swift
index 4cac1ac..883740e 100644
--- a/Sources/Benchmark/BenchmarkRunner.swift
+++ b/Sources/Benchmark/BenchmarkRunner.swift
@@ -16,7 +16,6 @@ public struct BenchmarkRunner {
     let suites: [BenchmarkSuite]
     let settings: [BenchmarkSetting]
     let customDefaults: [BenchmarkSetting]
-    let globalSettings: BenchmarkSettings
     var progress: ProgressReporter
     var reporter: BenchmarkReporter
     var results: [BenchmarkResult] = []
@@ -29,18 +28,18 @@ public struct BenchmarkRunner {
         self.suites = suites
         self.settings = settings
         self.customDefaults = customDefaults
-        self.globalSettings = BenchmarkSettings([
+        let globalSettings = BenchmarkSettings([
             defaultSettings,
             self.customDefaults,
             self.settings,
         ])
-        switch self.globalSettings.format {
+        switch globalSettings.format {
         case .none:
             self.progress = QuietReporter()
         default:
             self.progress = VerboseProgressReporter(output: StderrOutputStream())
         }
-        switch self.globalSettings.format {
+        switch globalSettings.format {
         case .none:
             self.reporter = QuietReporter()
         case .console:
@@ -56,7 +55,7 @@ public struct BenchmarkRunner {
         for suite in suites {
             try run(suite: suite)
         }
-        reporter.report(results: results, settings: globalSettings)
+        reporter.report(results: results)
     }
 
     mutating func run(suite: BenchmarkSuite) throws {
@@ -74,11 +73,16 @@ public struct BenchmarkRunner {
             self.settings,
         ])
 
-        let filter = try BenchmarkFilter(settings.filter)
+        let filter = try BenchmarkFilter(settings.filter, negate: false)
         if !filter.matches(suiteName: suite.name, benchmarkName: benchmark.name) {
             return
         }
 
+        let filterNot = try BenchmarkFilter(settings.filterNot, negate: true)
+        if !filterNot.matches(suiteName: suite.name, benchmarkName: benchmark.name) {
+            return
+        }
+
         progress.report(running: benchmark.name, suite: suite.name)
         let totalStart = now()
 
diff --git a/Sources/Benchmark/BenchmarkSetting.swift b/Sources/Benchmark/BenchmarkSetting.swift
index 6bdbcfc..cb349bf 100644
--- a/Sources/Benchmark/BenchmarkSetting.swift
+++ b/Sources/Benchmark/BenchmarkSetting.swift
@@ -54,6 +54,14 @@ public struct Filter: BenchmarkSetting {
     }
 }
 
+/// A regex string used to exclude benchmarks that should be run.
+public struct FilterNot: BenchmarkSetting {
+    public var value: String
+    public init(_ value: String) {
+        self.value = value
+    }
+}
+
 /// A minimal (total) time that iterations has to run
 /// to be considered significant.
 public struct MinTime: BenchmarkSetting {
@@ -184,6 +192,11 @@ public struct BenchmarkSettings {
         return self[Filter.self]?.value
     }
 
+    /// Convenience accessor for the Filter setting.
+    public var filterNot: String? {
+        return self[FilterNot.self]?.value
+    }
+
     /// Convenience accessor for the MinTime setting.
     public var minTime: Double {
         if let value = self[MinTime.self]?.value {


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkColumnTests.swift b/Tests/BenchmarkTests/BenchmarkColumnTests.swift
index 6657653..6c517d7 100644
--- a/Tests/BenchmarkTests/BenchmarkColumnTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkColumnTests.swift
@@ -55,7 +55,7 @@ final class BenchmarkColumnTests: XCTestCase {
 
     func testRenamed() {
         let time = BenchmarkColumn.registry["time"]!
-        XCTAssertEqual(time.name, "time") 
+        XCTAssertEqual(time.name, "time")
         let mytime = time.renamed("mytime")
         XCTAssertEqual(mytime.name, "mytime")
     }
diff --git a/Tests/BenchmarkTests/BenchmarkCommandTests.swift b/Tests/BenchmarkTests/BenchmarkCommandTests.swift
index 0a15b5f..a4981e6 100644
--- a/Tests/BenchmarkTests/BenchmarkCommandTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkCommandTests.swift
@@ -42,19 +42,34 @@ final class BenchmarkCommandTests: XCTestCase {
 
     func testParseFilter() throws {
         AssertParse(["--filter", "bar", "--allow-debug-build"]) { settings in
-            let filter = try! BenchmarkFilter(settings.filter)
+            let filter = try! BenchmarkFilter(settings.filter, negate: false)
             XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "baz"))
             XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "bar"))
         }
 
         AssertParse(["--filter", "foo.bar", "--allow-debug-build"]) { settings in
-            let filter = try! BenchmarkFilter(settings.filter)
+            let filter = try! BenchmarkFilter(settings.filter, negate: false)
             XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "baz"))
             XCTAssertFalse(filter.matches(suiteName: "foobar", benchmarkName: "baz"))
             XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "bar"))
         }
     }
 
+    func testParseFilterNot() throws {
+        AssertParse(["--filter-not", "bar", "--allow-debug-build"]) { settings in
+            let filter = try! BenchmarkFilter(settings.filterNot, negate: true)
+            XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "baz"))
+            XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "bar"))
+        }
+
+        AssertParse(["--filter-not", "foo.bar", "--allow-debug-build"]) { settings in
+            let filter = try! BenchmarkFilter(settings.filterNot, negate: true)
+            XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "baz"))
+            XCTAssert(filter.matches(suiteName: "foobar", benchmarkName: "baz"))
+            XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "bar"))
+        }
+    }
+
     func testParseIterations() throws {
         AssertParse(["--iterations", "42", "--allow-debug-build"]) { settings in
             XCTAssertEqual(settings.iterations, 42)
@@ -113,6 +128,7 @@ final class BenchmarkCommandTests: XCTestCase {
         ("testAllowDebugBuild", testAllowDebugBuild),
         ("testDebugBuildError", testDebugBuildError),
         ("testParseFilter", testParseFilter),
+        ("testParseFilterNot", testParseFilterNot),
         ("testParseIterations", testParseIterations),
         ("testParseZeroIterations", testParseZeroIterations),
         ("testParseWarmupIterations", testParseWarmupIterations),
diff --git a/Tests/BenchmarkTests/BenchmarkReporterTests.swift b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
index dfed2f4..9b0cba2 100644
--- a/Tests/BenchmarkTests/BenchmarkReporterTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
@@ -17,33 +17,26 @@ import XCTest
 @testable import Benchmark
 
 final class BenchmarkReporterTests: XCTestCase {
-    func assertConsoleReported(
-        _ results: [BenchmarkResult], _ expected: String,
-        settings: BenchmarkSettings = BenchmarkSettings()
-    ) {
+    func assertConsoleReported(_ results: [BenchmarkResult], _ expected: String) {
         let output = MockTextOutputStream()
         var reporter = ConsoleReporter(output: output)
-        reporter.report(results: results, settings: settings)
+        reporter.report(results: results)
         assertReported(output.result(), expected)
     }
 
-    func assertJSONReported(
-        _ results: [BenchmarkResult], _ expected: String,
-        settings: BenchmarkSettings = BenchmarkSettings()
-    ) {
+    func assertJSONReported(_ results: [BenchmarkResult], _ expected: String) {
         let output = MockTextOutputStream()
         var reporter = JSONReporter(output: output)
-        reporter.report(results: results, settings: settings)
+        reporter.report(results: results)
         assertReported(output.result(), expected)
     }
 
     func assertCSVReported(
-        _ results: [BenchmarkResult], _ expected: String,
-        settings: BenchmarkSettings = BenchmarkSettings()
+        _ results: [BenchmarkResult], _ expected: String
     ) {
         let output = MockTextOutputStream()
         var reporter = CSVReporter(output: output)
-        reporter.report(results: results, settings: settings)
+        reporter.report(results: results)
         assertReported(output.result(), expected)
     }
 
@@ -168,6 +161,30 @@ final class BenchmarkReporterTests: XCTestCase {
         assertConsoleReported(results, expected)
     }
 
+    func testConsoleMixedColumns() throws {
+        let results: [BenchmarkResult] = [
+            BenchmarkResult(
+                benchmarkName: "fast", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "min"])]]),
+                measurements: [1_000, 2_000],
+                warmupMeasurements: [10, 20, 30],
+                counters: [:]),
+            BenchmarkResult(
+                benchmarkName: "slow", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "max"])]]),
+                measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
+                counters: [:]),
+        ]
+        let expected = #"""
+            name         min       max         
+            -----------------------------------
+            MySuite.fast 1000.0 ns             
+            MySuite.slow           2000000.0 ns
+            """#
+        assertConsoleReported(results, expected)
+    }
+
     func testJSONEmpty() {
         let results: [BenchmarkResult] = []
         let expected = #"""
@@ -221,7 +238,7 @@ final class BenchmarkReporterTests: XCTestCase {
             results.append(
                 BenchmarkResult(
                     benchmarkName: name, suiteName: "",
-                    settings: BenchmarkSettings(),
+                    settings: BenchmarkSettings([defaultSettings, [Columns(["name"])]]),
                     measurements: [],
                     warmupMeasurements: [],
                     counters: [:]))
@@ -244,33 +261,40 @@ final class BenchmarkReporterTests: XCTestCase {
               ]
             }
             """#
-        let settings = BenchmarkSettings([Columns(["name"])])
-        assertJSONReported(results, expected, settings: settings)
+        assertJSONReported(results, expected)
     }
 
     func testJSONTimeUnit() throws {
         let results: [BenchmarkResult] = [
             BenchmarkResult(
                 benchmarkName: "ns", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.ns)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.ns), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "us", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.us)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.us), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "ms", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.ms)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.ms), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "s", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.s)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.s), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
@@ -297,15 +321,44 @@ final class BenchmarkReporterTests: XCTestCase {
               ]
             }
             """#
-        let settings = BenchmarkSettings([Columns(["name", "time"])])
-        assertJSONReported(results, expected, settings: settings)
+        assertJSONReported(results, expected)
     }
 
-    func testCSVEmpty() {
-        let results: [BenchmarkResult] = []
+    func testJSONMixedColumns() throws {
+        let results: [BenchmarkResult] = [
+            BenchmarkResult(
+                benchmarkName: "fast", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "min"])]]),
+                measurements: [1_000, 2_000],
+                warmupMeasurements: [10, 20, 30],
+                counters: [:]),
+            BenchmarkResult(
+                benchmarkName: "slow", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "max"])]]),
+                measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
+                counters: [:]),
+        ]
         let expected = #"""
-            name,time,std,iterations
+            {
+              "benchmarks": [
+                {
+                  "name": "MySuite.fast",
+                  "min": 1000.0,
+                },
+                {
+                  "name": "MySuite.slow",
+                  "max": 2000000.0
+                }
+              ]
+            }
             """#
+        assertJSONReported(results, expected)
+    }
+
+    func testCSVEmpty() {
+        let results: [BenchmarkResult] = []
+        let expected = ""
         assertCSVReported(results, expected)
     }
 
@@ -338,7 +391,7 @@ final class BenchmarkReporterTests: XCTestCase {
             results.append(
                 BenchmarkResult(
                     benchmarkName: name, suiteName: "",
-                    settings: BenchmarkSettings(),
+                    settings: BenchmarkSettings([defaultSettings, [Columns(["name", "time"])]]),
                     measurements: [1_000, 2_000],
                     warmupMeasurements: [],
                     counters: [:]))
@@ -350,33 +403,40 @@ final class BenchmarkReporterTests: XCTestCase {
             "
             ",1500.0
             """#
-        let settings = BenchmarkSettings([Columns(["name", "time"])])
-        assertCSVReported(results, expected, settings: settings)
+        assertCSVReported(results, expected)
     }
 
     func testCSVTimeUnit() throws {
         let results: [BenchmarkResult] = [
             BenchmarkResult(
                 benchmarkName: "ns", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.ns)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.ns), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "us", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.us)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.us), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "ms", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.ms)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.ms), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
             BenchmarkResult(
                 benchmarkName: "s", suiteName: "MySuite",
-                settings: BenchmarkSettings([TimeUnit(.s)]),
+                settings: BenchmarkSettings([
+                    defaultSettings, [TimeUnit(.s), Columns(["name", "time"])],
+                ]),
                 measurements: [123_456_789],
                 warmupMeasurements: [],
                 counters: [:]),
@@ -388,8 +448,30 @@ final class BenchmarkReporterTests: XCTestCase {
             MySuite.ms,123.456789
             MySuite.s,0.123456789
             """#
-        let settings = BenchmarkSettings([Columns(["name", "time"])])
-        assertCSVReported(results, expected, settings: settings)
+        assertCSVReported(results, expected)
+    }
+
+    func testCSVMixedColumns() throws {
+        let results: [BenchmarkResult] = [
+            BenchmarkResult(
+                benchmarkName: "fast", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "min"])]]),
+                measurements: [1_000, 2_000],
+                warmupMeasurements: [10, 20, 30],
+                counters: [:]),
+            BenchmarkResult(
+                benchmarkName: "slow", suiteName: "MySuite",
+                settings: BenchmarkSettings([defaultSettings, [Columns(["name", "max"])]]),
+                measurements: [1_000_000, 2_000_000],
+                warmupMeasurements: [],
+                counters: [:]),
+        ]
+        let expected = #"""
+            name,min,max
+            MySuite.fast,1000.0,
+            MySuite.slow,,2000000.0
+            """#
+        assertCSVReported(results, expected)
     }
 
     static var allTests = [
@@ -397,13 +479,16 @@ final class BenchmarkReporterTests: XCTestCase {
         ("testConsoleCounters", testConsoleCounters),
         ("testConsoleWarmup", testConsoleWarmup),
         ("testConsoleTimeUnit", testConsoleTimeUnit),
+        ("testConsoleMixedColumns", testConsoleMixedColumns),
         ("testJSONEmpty", testJSONEmpty),
         ("testJSONBasic", testJSONBasic),
         ("testJSONEscape", testJSONEscape),
         ("testJSONTimeUnit", testJSONTimeUnit),
+        ("testJSONMixedColumns", testJSONMixedColumns),
         ("testCSVEmpty", testCSVEmpty),
         ("testCSVBasic", testCSVBasic),
         ("testCSVEscape", testCSVEscape),
         ("testCSVTimeUnit", testCSVTimeUnit),
+        ("testCSVMixedColumns", testCSVMixedColumns),
     ]
 }