feat(minor): Various outstanding fixes (#204)

Fixes for the following issues:
#202
#201
#195
#194
#191
#198

Benchmarks/Basic/BenchmarkRunner+Basic.swift

@@ -40,6 +40,7 @@ let benchmarks = {
     enum CustomMetrics {
         static var one: BenchmarkMetric { .custom("CustomMetricOne") }
         static var two: BenchmarkMetric { .custom("CustomMetricTwo", polarity: .prefersLarger, useScalingFactor: true) }
+        static var three: BenchmarkMetric { .custom("CustomMetricThree", polarity: .prefersLarger, useScalingFactor: false) }
     }
 
     Benchmark("Basic",
@@ -52,12 +53,33 @@ let benchmarks = {
     Benchmark("Noop2", configuration: .init(metrics: [.wallClock] + .arc)) { _ in
     }
 
-    Benchmark("Scaled metrics",
+    Benchmark("Scaled metrics One",
+              configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one],
+                                   scalingFactor: .one)) { benchmark in
+        for _ in benchmark.scaledIterations {
+            blackHole(Int.random(in: 1 ... 1_000))
+        }
+        benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000))
+        benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000))
+    }
+
+    Benchmark("Scaled metrics K",
               configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one],
                                    scalingFactor: .kilo)) { benchmark in
+        for _ in benchmark.scaledIterations {
+            blackHole(Int.random(in: 1 ... 1_000))
+        }
+        benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000))
+        benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000))
+    }
+
+    Benchmark("Scaled metrics M",
+              configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one, CustomMetrics.three],
+                                   scalingFactor: .mega)) { benchmark in
         for _ in benchmark.scaledIterations {
             blackHole(Int.random(in: benchmark.scaledIterations))
         }
+        benchmark.measurement(CustomMetrics.three, Int.random(in: 1 ... 1_000_000_000))
         benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000))
         benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000))
     }

Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift

@@ -47,14 +47,13 @@ let help =
     --skip-target <skip-target>
                           Benchmark targets matching the regexp filter that should be skipped
     --format <format>       The output format to use, one of: ["text", "markdown", "influx", "jmh", "histogramEncoded", "histogram", "histogramSamples", "histogramPercentiles", "metricP90AbsoluteThresholds"], default is 'text'
-    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks, valid values are: ["cpuUser", "cpuSystem", "cpuTotal", "wallClock", "throughput", "peakMemoryResident", "peakMemoryVirtual", "mallocCountSmall", "mallocCountLarge",
-                          "mallocCountTotal", "allocatedResidentMemory", "memoryLeaked", "syscalls", "contextSwitches", "threads", "threadsRunning", "readSyscalls", "writeSyscalls", "readBytesLogical", "writeBytesLogical", "readBytesPhysical", "writeBytesPhysical", "retainCount", "releaseCount",
-                          "retainReleaseDelta", "custom"]
+    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks, valid values are: ["cpuUser", "cpuSystem", "cpuTotal", "wallClock", "throughput", "peakMemoryResident",
+                          "peakMemoryResidentDelta", "peakMemoryVirtual", "mallocCountSmall", "mallocCountLarge", "mallocCountTotal", "allocatedResidentMemory", "memoryLeaked", "syscalls", "contextSwitches", "threads", "threadsRunning", "readSyscalls",
+                          "writeSyscalls", "readBytesLogical", "writeBytesLogical", "readBytesPhysical", "writeBytesPhysical", "retainCount", "releaseCount", "retainReleaseDelta", "custom"]
     --path <path>           The path where exported data is stored, default is the current directory (".").
     --quiet                 Specifies that output should be suppressed (useful for if you just want to check return code)
     --scale                 Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output)
-    --check-absolute
-                          Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
+    --check-absolute        Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
                           This is used for CI workflows when you want to validate the thresholds vs. a persisted benchmark baseline
                           rather than comparing PR vs main or vs a current run. This is useful to cut down the build matrix needed
                           for those wanting to validate performance of e.g. toolchains or OS:s as well (or have other reasons for wanting

Plugins/BenchmarkHelpGenerator/BenchmarkHelpGenerator.swift

@@ -20,6 +20,7 @@ let availableMetrics = [
     "wallClock",
     "throughput",
     "peakMemoryResident",
+    "peakMemoryResidentDelta",
     "peakMemoryVirtual",
     "mallocCountSmall",
     "mallocCountLarge",

Plugins/BenchmarkTool/BenchmarkTool+Operations.swift

@@ -147,12 +147,14 @@ extension BenchmarkTool {
                         return
                     }
                     if let benchmarkPath = checkAbsolutePath { // load statically defined threshods for .p90
+                        var thresholdsFound = false
                         benchmarks.forEach { benchmark in
                             let thresholds = BenchmarkTool.makeBenchmarkThresholds(path: benchmarkPath,
                                                                                    moduleName: benchmark.target,
                                                                                    benchmarkName: benchmark.name)
                             var transformed: [BenchmarkMetric: BenchmarkThresholds] = [:]
                             if let thresholds {
+                                thresholdsFound = true
                                 thresholds.forEach { key, value in
                                     if let metric = BenchmarkMetric(argument: key) {
                                         let absoluteThreshold: BenchmarkThresholds.AbsoluteThresholds = [.p90: value]
@@ -164,6 +166,11 @@ extension BenchmarkTool {
                                 }
                             }
                         }
+                        if !thresholdsFound  {
+                            print("")
+                            failBenchmark("Could not find any matching absolute thresholds at path [\(benchmarkPath)], failing threshold check.",
+                                          exitCode: .thresholdRegression)
+                        }
                     }
                     print("")
                     let currentBaseline = benchmarkBaselines[0]

Plugins/BenchmarkTool/BenchmarkTool+PrettyPrinting.swift

@@ -92,14 +92,15 @@ extension BenchmarkTool {
 
         var scaledResults: [ScaledResults] = []
         results.forEach { result in
-            let description: String
-            let metrics = result.metrics
-            let percentiles = metrics.statistics.percentiles()
             var resultPercentiles = ScaledResults.Percentiles()
 
-            var adjustmentFunction: (Int) -> Int
+            let metrics = result.metrics
+            let percentiles = metrics.statistics.percentiles()
+            let shouldScale = self.scale == false && result.metrics.metric.useScalingFactor
+            let adjustmentFunction: (Int) -> Int
+            let description: String
 
-            if self.scale, result.metrics.metric.useScalingFactor {
+            if shouldScale {
                 description = useGroupingDescription ? "\(result.description) \(result.metrics.scaledUnitDescriptionPretty)"
                     : "\(result.metrics.metric.description) \(result.metrics.scaledUnitDescriptionPretty)"
                 adjustmentFunction = result.metrics.scale
@@ -145,7 +146,7 @@ extension BenchmarkTool {
             metrics.forEach { metric in
                 width = max(width, metric.description.count)
             }
-            width = min(maxDescriptionWidth, width + " (M)".count)
+            width = min(maxDescriptionWidth, width + " (ms)".count)
 
             baseline.targets.forEach { target in
                 let separator = String(repeating: "=", count: "\(target)".count)
@@ -173,7 +174,7 @@ extension BenchmarkTool {
             baseline.benchmarkIdentifiers.forEach { identifier in
                 width = max(width, "\(identifier.target):\(identifier.name)".count)
             }
-            width = min(maxDescriptionWidth, width + " (M)".count)
+            width = min(maxDescriptionWidth, width + " (ms)".count)
 
             baseline.benchmarkMetrics.forEach { metric in
 
@@ -240,7 +241,12 @@ extension BenchmarkTool {
                                 }
                             }
 
-                            let title = "\(result.metric.description) \(result.unitDescriptionPretty)"
+                            let displayBaseScaled = self.scale == false && base.metric.useScalingFactor
+                            let displayResultScaled = self.scale == false && result.metric.useScalingFactor
+                            let title = displayBaseScaled ?
+                                "\(result.metric.description) \(result.scaledUnitDescriptionPretty)" :
+                                "\(result.metric.description) \(result.unitDescriptionPretty)"
+
                             let width = 40
                             let table = TextTable<ScaledResults> {
                                 [Column(title: title, value: "\($0.description)", width: width, align: .center),
@@ -267,7 +273,7 @@ extension BenchmarkTool {
                             var adjustmentFunction: (Int) -> Int
                             let samples = result.statistics.measurementCount - base.statistics.measurementCount
 
-                            if self.scale, base.metric.useScalingFactor {
+                            if displayBaseScaled {
                                 adjustmentFunction = base.scale
                             } else {
                                 adjustmentFunction = base.normalize
@@ -285,7 +291,7 @@ extension BenchmarkTool {
                                                                percentiles: basePercentiles,
                                                                samples: base.statistics.measurementCount))
 
-                            if self.scale, result.metric.useScalingFactor {
+                            if displayResultScaled {
                                 adjustmentFunction = result.scale
                             } else {
                                 adjustmentFunction = result.normalize

Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift

@@ -43,7 +43,7 @@ extension BenchmarkTool {
             path = cwdPath
         }
 
-        var p90Thresholds: [String: BenchmarkThresholds.AbsoluteThreshold]?
+        var p90Thresholds: [String: BenchmarkThresholds.AbsoluteThreshold]? = nil
 
         do {
             let fileDescriptor = try FileDescriptor.open(path, .readOnly, options: [], permissions: .ownerRead)

Sources/Benchmark/BenchmarkExecutor+Extensions.swift

@@ -39,6 +39,8 @@ extension BenchmarkExecutor {
             return true
         case .peakMemoryResident:
             return true
+        case .peakMemoryResidentDelta:
+            return true
         case .peakMemoryVirtual:
             return true
         case .syscalls:

Sources/Benchmark/BenchmarkExecutor.swift

@@ -100,11 +100,12 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
         var iterations = 0
         let initialStartTime = BenchmarkClock.now
 
-        // 'Warmup' to remove initial mallocs from stats in p100, also used as base for some metrics
+        // 'Warmup' to remove initial mallocs from stats in p100
         _ = MallocStatsProducer.makeMallocStats() // baselineMallocStats
 
         // Calculate typical sys call check overhead and deduct that to get 'clean' stats for the actual benchmark
         var operatingSystemStatsOverhead = OperatingSystemStats()
+        var baselinePeakMemoryResidentDelta = 0
         if operatingSystemStatsRequested {
             let statsOne = operatingSystemStatsProducer.makeOperatingSystemStats()
             let statsTwo = operatingSystemStatsProducer.makeOperatingSystemStats()
@@ -219,6 +220,9 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
                 delta = stopOperatingSystemStats.peakMemoryResident
                 statistics[.peakMemoryResident]?.add(Int(delta))
 
+                delta = stopOperatingSystemStats.peakMemoryResident - baselinePeakMemoryResidentDelta
+                statistics[.peakMemoryResidentDelta]?.add(Int(delta))
+
                 delta = stopOperatingSystemStats.peakMemoryVirtual
                 statistics[.peakMemoryVirtual]?.add(Int(delta))
 
@@ -266,11 +270,20 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
             statistics[metric]?.add(value)
         }
 
+        if arcStatsRequested {
+            ARCStatsProducer.hook()
+        }
+
         if benchmark.configuration.metrics.contains(.threads) ||
             benchmark.configuration.metrics.contains(.threadsRunning) ||
             benchmark.configuration.metrics.contains(.peakMemoryResident) ||
+            benchmark.configuration.metrics.contains(.peakMemoryResidentDelta) ||
             benchmark.configuration.metrics.contains(.peakMemoryVirtual) {
             operatingSystemStatsProducer.startSampling(5_000) // ~5 ms
+
+            if benchmark.configuration.metrics.contains(.peakMemoryResidentDelta) {
+                baselinePeakMemoryResidentDelta = operatingSystemStatsProducer.makeOperatingSystemStats().peakMemoryResident
+            }
         }
 
         var progressBar: ProgressBar?
@@ -290,10 +303,6 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
 
         var nextPercentageToUpdateProgressBar = 0
 
-        if arcStatsRequested {
-            ARCStatsProducer.hook()
-        }
-
         #if canImport(OSLog)
             let benchmarkInterval = signPost.beginInterval("Benchmark", id: signpostID, "\(benchmark.name)")
         #endif
@@ -340,10 +349,6 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
             signPost.endInterval("Benchmark", benchmarkInterval, "\(iterations)")
         #endif
 
-        if arcStatsRequested {
-            ARCStatsProducer.unhook()
-        }
-
         if var progressBar {
             progressBar.setValue(100)
         }
@@ -353,6 +358,10 @@ final class BenchmarkExecutor { // swiftlint:disable:this type_body_length
             operatingSystemStatsProducer.stopSampling()
         }
 
+        if arcStatsRequested {
+            ARCStatsProducer.unhook()
+        }
+
         // construct metric result array
         var results: [BenchmarkResult] = []
         statistics.forEach { key, value in

Sources/Benchmark/BenchmarkMetric+Defaults.swift

@@ -39,6 +39,7 @@ public extension BenchmarkMetric {
     /// A collection of memory benchmarks.
     static var memory: [BenchmarkMetric] {
         [.peakMemoryResident,
+         .peakMemoryResidentDelta,
          .peakMemoryVirtual,
          .mallocCountSmall,
          .mallocCountLarge,
@@ -83,6 +84,7 @@ public extension BenchmarkMetric {
          .wallClock,
          .throughput,
          .peakMemoryResident,
+         .peakMemoryResidentDelta,
          .peakMemoryVirtual,
          .mallocCountSmall,
          .mallocCountLarge,

Sources/Benchmark/BenchmarkMetric.swift

@@ -25,6 +25,8 @@ public enum BenchmarkMetric: Hashable, Equatable, Codable, CustomStringConvertib
     case throughput
     /// Measure resident memory usage - sampled during runtime
     case peakMemoryResident
+    /// Measure resident memory usage - sampled during runtime (subtracting start of benchmark baseline resident amount)
+    case peakMemoryResidentDelta
     /// Measure virtual memory usage - sampled during runtime
     case peakMemoryVirtual
     /// Number of small malloc calls
@@ -122,7 +124,7 @@ public extension BenchmarkMetric {
             return true
         case .mallocCountLarge, .mallocCountSmall, .mallocCountTotal, .memoryLeaked:
             return true
-        case .syscalls, .throughput:
+        case .syscalls:
             return true
         case .readSyscalls, .readBytesLogical, .readBytesPhysical:
             return true
@@ -163,6 +165,8 @@ public extension BenchmarkMetric {
             return "Throughput (# / s)"
         case .peakMemoryResident:
             return "Memory (resident peak)"
+        case .peakMemoryResidentDelta:
+            return "Memory Δ (resident peak)"
         case .peakMemoryVirtual:
             return "Memory (virtual peak)"
         case .mallocCountSmall:
@@ -231,6 +235,8 @@ public extension BenchmarkMetric {
             return "throughput"
         case .peakMemoryResident:
             return "peakMemoryResident"
+        case .peakMemoryResidentDelta:
+            return "peakMemoryResidentDelta"
         case .peakMemoryVirtual:
             return "peakMemoryVirtual"
         case .mallocCountSmall:
@@ -301,6 +307,8 @@ public extension BenchmarkMetric {
             self = BenchmarkMetric.throughput
         case "peakMemoryResident":
             self = BenchmarkMetric.peakMemoryResident
+        case "peakMemoryResidentDelta":
+            self = BenchmarkMetric.peakMemoryResidentDelta
         case "peakMemoryVirtual":
             self = BenchmarkMetric.peakMemoryVirtual
         case "mallocCountSmall":

Sources/Benchmark/BenchmarkResult.swift

@@ -266,6 +266,9 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
     }
 
     public var unitDescriptionPretty: String {
+        if metric == .throughput {
+            return "(\(scaledScalingFactor.description))"
+        }
         if metric.countable {
             let statisticsUnit = Statistics.Units(timeUnits)
             if statisticsUnit == .count {

Sources/Benchmark/Documentation.docc/BenchmarkMetric.md

@@ -34,6 +34,7 @@
 ### Memory Metrics
 
 - ``BenchmarkMetric/peakMemoryResident``
+- ``BenchmarkMetric/peakMemoryResidentDelta``
 - ``BenchmarkMetric/peakMemoryVirtual``
 - ``BenchmarkMetric/mallocCountSmall``
 - ``BenchmarkMetric/mallocCountLarge``

Sources/Benchmark/Documentation.docc/Metrics.md

@@ -16,6 +16,7 @@ Currently supported metrics are:
 - term `wallClock`: Wall clock time for running the test
 - term `throughput`: The throughput in operations / second
 - term `peakMemoryResident`: The resident memory usage - sampled during runtime
+- term `peakMemoryResidentDelta`: The resident memory usage - sampled during runtime (excluding start of benchmark baseline) 
 - term `peakMemoryVirtual`:  The virtual memory usage - sampled during runtime
 - term `mallocCountSmall`: The number of small malloc calls according to jemalloc
 - term `mallocCountLarge`: The number of large malloc calls according to jemalloc