Repository: google/swift-benchmark
Pull request #81 (2020-11-03 12:29:15)

Make time formatter produce uniform decimals
By non-uniformly showing decimals (e.g. `.500`) only for some of the benchmarks, while omitting them for others, swift-benchmark makes it needlessly hard to visually compare times:

At a quick glance, is `bar` faster or slower than `foo` and/or `baz`?

```terminal
running foo... done! (1722.98 ms)
running bar... done! (2296.11 ms)
running baz... done! (1617.17 ms)

name time            std        iterations
------------------------------------------
foo     426280194 ns ±   0.33 %          3
bar  78679670.500 ns ±   1.15 %         18
baz     404978073 ns ±   0.91 %          3
```

Glancing over these benchmarks one might easily come to the wrong impression that `bar` is several orders of magnitude slower than the others.

If the benchmark times were printed with uniform decimals instead, such a wrong interpretation would be rather unlikely:

```terminal
foo 426280194.000 ns ±   0.33 %          3
bar  78679670.500 ns ±   1.15 %         18
baz 404978073.000 ns ±   0.91 %          3
```

---

While **uniform decimals** help in quickly **recognizing relative differences** the addition of **thousands separators** can further improve the **recognizability of absolute values**:

```terminal
foo 426,280,194.000 ns ±   0.33 %        3
bar  78,679,670.500 ns ±   1.15 %       18
baz 404,978,073.000 ns ±   0.91 %        3
```

Swift's native string formatting however does not provide a way to add thousands separators, so **this PR only implements the former**.

shabalind: Needs to be updated to resolve the conflicting changes in `BenchmarkReporterTests.swift`.
regexident: Resolved. 👍 

Patch:
diff --git a/Sources/Benchmark/BenchmarkColumn.swift b/Sources/Benchmark/BenchmarkColumn.swift
index 43795a9..335cf4c 100644
--- a/Sources/Benchmark/BenchmarkColumn.swift
+++ b/Sources/Benchmark/BenchmarkColumn.swift
@@ -64,7 +64,7 @@ public struct BenchmarkColumn: Hashable {
             case .inverseTime:
                 self.formatter = BenchmarkFormatter.inverseTime
             case .none:
-                self.formatter = BenchmarkFormatter.number
+                self.formatter = BenchmarkFormatter.real
             }
         }
     }
@@ -98,7 +98,8 @@ public struct BenchmarkColumn: Hashable {
             formatter: BenchmarkFormatter.stdPercentage)
         result["iterations"] = BenchmarkColumn(
             name: "iterations",
-            value: { Double($0.measurements.count) })
+            value: { Double($0.measurements.count) },
+            formatter: BenchmarkFormatter.integer)
         result["warmup"] = BenchmarkColumn(
             name: "warmup",
             value: { $0.warmupMeasurements.sum },
@@ -208,7 +209,8 @@ public struct BenchmarkColumn: Hashable {
                             return 0
                         }
                     },
-                    alignment: .right))
+                    alignment: .right,
+                    formatter: BenchmarkFormatter.integer))
         }
 
         return columns
diff --git a/Sources/Benchmark/BenchmarkFormatter.swift b/Sources/Benchmark/BenchmarkFormatter.swift
index 59b5738..d7ab4f2 100644
--- a/Sources/Benchmark/BenchmarkFormatter.swift
+++ b/Sources/Benchmark/BenchmarkFormatter.swift
@@ -16,25 +16,25 @@
 public enum BenchmarkFormatter {
     public typealias Formatter = (Double, BenchmarkSettings) -> String
 
-    /// Just show a number, stripping ".0" if it's integer.
-    public static let number: Formatter = { (value, settings) in
-        let string = String(value)
-        if string.hasSuffix(".0") {
-            return String(string.dropLast(2))
-        } else {
-            return String(format: "%.3f", value)
-        }
+    /// Show an integer number without decimals.
+    public static let integer: Formatter = { (value, settings) in
+        return String(format: "%.0f", value)
+    }
+
+    /// Show a real number with decimals.
+    public static let real: Formatter = { (value, settings) in
+        return String(format: "%.3f", value)
     }
 
     /// Show number with the corresponding time unit.
     public static let time: Formatter = { (value, settings) in
-        let num = number(value, settings)
+        let num = real(value, settings)
         return "\(num) \(settings.timeUnit)"
     }
 
     /// Show number with the corresponding inverse time unit.
     public static let inverseTime: Formatter = { (value, settings) in
-        let num = number(value, settings)
+        let num = real(value, settings)
         return "\(num) /\(settings.inverseTimeUnit)"
     }
 
@@ -45,7 +45,7 @@ public enum BenchmarkFormatter {
 
     /// Show value as plus or minus standard deviation.
     public static let std: Formatter = { (value, settings) in
-        let num = number(value, settings)
+        let num = real(value, settings)
         return "± \(num)"
     }
 
diff --git a/Sources/Benchmark/BenchmarkSetting.swift b/Sources/Benchmark/BenchmarkSetting.swift
index def6cec..cd988fb 100644
--- a/Sources/Benchmark/BenchmarkSetting.swift
+++ b/Sources/Benchmark/BenchmarkSetting.swift
@@ -77,11 +77,24 @@ public struct TimeUnit: BenchmarkSetting {
     public init(_ value: Value) {
         self.value = value
     }
-    public enum Value: String, ExpressibleByArgument {
+    public enum Value: String, ExpressibleByArgument, CustomStringConvertible {
         case ns
         case us
         case ms
         case s
+
+        public var description: String {
+            switch self {
+            case .ns:
+                return "ns"
+            case .us:
+                return "us"
+            case .ms:
+                return "ms"
+            case .s:
+                return " s"
+            }
+        }
     }
 }
 


Test patch:
diff --git a/Tests/BenchmarkTests/BenchmarkReporterTests.swift b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
index 78577ca..f3916c7 100644
--- a/Tests/BenchmarkTests/BenchmarkReporterTests.swift
+++ b/Tests/BenchmarkTests/BenchmarkReporterTests.swift
@@ -63,13 +63,18 @@ final class BenchmarkReporterTests: XCTestCase {
         file: StaticString = #filePath,
         line: UInt = #line
     ) {
+        func trimmingTrailingWhitespace(_ string: String) -> String {
+            String(string.reversed().drop(while: { $0.isWhitespace }).reversed())
+        }
         let lines = Array(got.split(separator: "\n").map { String($0) })
         let expectedLines = expected.split(separator: "\n").map { String($0) }
         let actual = lines.map { $0.trimmingCharacters(in: .newlines) }
             .filter { !$0.isEmpty }
         XCTAssertEqual(expectedLines.count, actual.count, message(), file: file, line: line)
         for (expectedLine, actualLine) in zip(expectedLines, actual) {
-            XCTAssertEqual(expectedLine, actualLine, message(), file: file, line: line)
+            let trimmedExpectedLine = trimmingTrailingWhitespace(expectedLine)
+            let trimmedActualLine = trimmingTrailingWhitespace(actualLine)
+            XCTAssertEqual(trimmedExpectedLine, trimmedActualLine, message(), file: file, line: line)
         }
     }
 
@@ -89,10 +94,10 @@ final class BenchmarkReporterTests: XCTestCase {
                 counters: [:]),
         ]
         let expected = #"""
-            name         time       std        iterations
-            ---------------------------------------------
-            MySuite.fast    1500 ns ±  47.14 %          2
-            MySuite.slow 1500000 ns ±  47.14 %          2
+            name         time           std        iterations
+            -------------------------------------------------
+            MySuite.fast    1500.000 ns ±  47.14 %          2
+            MySuite.slow 1500000.000 ns ±  47.14 %          2
             """#
         assertConsoleReported(results, expected)
     }
@@ -113,10 +118,10 @@ final class BenchmarkReporterTests: XCTestCase {
                 counters: [:]),
         ]
         let expected = #"""
-            name         time       std        iterations foo
-            -------------------------------------------------
-            MySuite.fast    1500 ns ±  47.14 %          2   7
-            MySuite.slow 1500000 ns ±  47.14 %          2   0
+            name         time           std        iterations foo
+            -----------------------------------------------------
+            MySuite.fast    1500.000 ns ±  47.14 %          2   7
+            MySuite.slow 1500000.000 ns ±  47.14 %          2   0
             """#
         assertConsoleReported(results, expected)
     }
@@ -137,10 +142,10 @@ final class BenchmarkReporterTests: XCTestCase {
                 counters: [:]),
         ]
         let expected = #"""
-            name         time       std        iterations warmup
-            ----------------------------------------------------
-            MySuite.fast    1500 ns ±  47.14 %          2  60 ns
-            MySuite.slow 1500000 ns ±  47.14 %          2   0 ns
+            name         time           std        iterations warmup
+            -----------------------------------------------------------
+            MySuite.fast    1500.000 ns ±  47.14 %          2 60.000 ns
+            MySuite.slow 1500000.000 ns ±  47.14 %          2  0.000 ns
             """#
         assertConsoleReported(results, expected)
     }
@@ -173,12 +178,12 @@ final class BenchmarkReporterTests: XCTestCase {
                 counters: [:]),
         ]
         let expected = #"""
-            name       time          std        iterations
-            ----------------------------------------------
-            MySuite.ns  123456789 ns ±   0.00 %          1
-            MySuite.us 123456.789 us ±   0.00 %          1
-            MySuite.ms    123.457 ms ±   0.00 %          1
-            MySuite.s        0.123 s ±   0.00 %          1
+            name       time             std        iterations
+            -------------------------------------------------
+            MySuite.ns 123456789.000 ns ±   0.00 %          1
+            MySuite.us    123456.789 us ±   0.00 %          1
+            MySuite.ms       123.457 ms ±   0.00 %          1
+            MySuite.s          0.123  s ±   0.00 %          1
             """#
         assertConsoleReported(results, expected)
     }
@@ -199,10 +204,10 @@ final class BenchmarkReporterTests: XCTestCase {
                 counters: [:]),
         ]
         let expected = #"""
-            name         min     max       
-            -------------------------------
-            MySuite.fast 1000 ns           
-            MySuite.slow         2000000 ns
+            name         min         max
+            ---------------------------------------
+            MySuite.fast 1000.000 ns                          
+            MySuite.slow             2000000.000 ns
             """#
         assertConsoleReported(results, expected)
     }