Skip to content

Commit

Permalink
feat(minor): Add Benchmark tags
Browse files Browse the repository at this point in the history
Sometimes it is convenient to run the same benchmark multiple times under slightly different scenarios.
For example, benchmarking `func foo(_ n: Int)` for various parameterizations of `n`.
This adds support for specifying tags for a benchmark but makes no functional changes beyond the name/identifier of a benchmark now including a parameter list when one was provided.

This unlocks the potential for the benchmark exporters (such as the Influx exporter) to leverage these tags (by emitting appropriate tags/fields in the case of Influx) in their output.
  • Loading branch information
Jairon Terrero committed Sep 30, 2024
1 parent 75e8622 commit e327a91
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 4 deletions.
34 changes: 30 additions & 4 deletions Sources/Benchmark/Benchmark.swift
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,28 @@ public final class Benchmark: Codable, Hashable {
#endif
public static var benchmarks: [Benchmark] = [] // Bookkeeping of all registered benchmarks

/// The default name used for display purposes of the benchmark when no parameters exist.
private var baseName: String

/// The name used for display purposes of the benchmark (also used for matching when comparing to baselines)
public var name: String
public var name: String {
get {
if configuration.tags.isEmpty {
return baseName
} else {
return baseName
+ " ("
+ configuration.tags
.sorted(by: { $0.key < $1.key })
.map({ "\($0.key): \($0.value)" })
.joined(separator: ", ")
+ ")"
}
}
set {
baseName = newValue
}
}

/// The reason for a benchmark failure, not set if successful
public var failureReason: String?
Expand Down Expand Up @@ -119,6 +139,7 @@ public final class Benchmark: Codable, Hashable {

/// Hook for setting defaults for a whole benchmark suite
public static var defaultConfiguration: Configuration = .init(metrics: BenchmarkMetric.default,
tags: [:],
timeUnits: .automatic,
warmupIterations: 1,
scalingFactor: .one,
Expand All @@ -131,7 +152,7 @@ public final class Benchmark: Codable, Hashable {
var measurementCompleted = false // Keep track so we skip multiple 'end of measurement'

enum CodingKeys: String, CodingKey {
case name
case baseName = "name"
case target
case executablePath
case configuration
Expand Down Expand Up @@ -168,7 +189,7 @@ public final class Benchmark: Codable, Hashable {
return nil
}
target = ""
self.name = name
self.baseName = name
self.configuration = configuration
self.closure = closure
self.setup = setup
Expand All @@ -193,7 +214,7 @@ public final class Benchmark: Codable, Hashable {
return nil
}
target = ""
self.name = name
self.baseName = name
self.configuration = configuration
asyncClosure = closure
self.setup = setup
Expand Down Expand Up @@ -362,6 +383,8 @@ public extension Benchmark {
struct Configuration: Codable {
/// Defines the metrics that should be measured for the benchmark
public var metrics: [BenchmarkMetric]
/// Specifies the parameters used to define the benchmark.
public var tags: [String: String]
/// Override the automatic detection of timeunits for metrics related to time to a specific
/// one (auto should work for most use cases)
public var timeUnits: BenchmarkTimeUnits
Expand All @@ -386,6 +409,7 @@ public extension Benchmark {
public var teardown: BenchmarkTeardownHook?

public init(metrics: [BenchmarkMetric] = defaultConfiguration.metrics,
tags: [String: String] = defaultConfiguration.tags,
timeUnits: BenchmarkTimeUnits = defaultConfiguration.timeUnits,
warmupIterations: Int = defaultConfiguration.warmupIterations,
scalingFactor: BenchmarkScalingFactor = defaultConfiguration.scalingFactor,
Expand All @@ -397,6 +421,7 @@ public extension Benchmark {
setup: BenchmarkSetupHook? = nil,
teardown: BenchmarkTeardownHook? = nil) {
self.metrics = metrics
self.tags = tags
self.timeUnits = timeUnits
self.warmupIterations = warmupIterations
self.scalingFactor = scalingFactor
Expand All @@ -411,6 +436,7 @@ public extension Benchmark {
// swiftlint:disable nesting
enum CodingKeys: String, CodingKey {
case metrics
case tags
case timeUnits
case warmupIterations
case scalingFactor
Expand Down
2 changes: 2 additions & 0 deletions Sources/Benchmark/BenchmarkExecutor.swift
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length
scalingFactor: benchmark.configuration.scalingFactor,
warmupIterations: benchmark.configuration.warmupIterations,
thresholds: benchmark.configuration.thresholds?[metric],
tags: benchmark.configuration.tags,
statistics: value)
results.append(result)
}
Expand All @@ -457,6 +458,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length
scalingFactor: benchmark.configuration.scalingFactor,
warmupIterations: benchmark.configuration.warmupIterations,
thresholds: benchmark.configuration.thresholds?[metric],
tags: benchmark.configuration.tags,
statistics: value)
results.append(result)
}
Expand Down
3 changes: 3 additions & 0 deletions Sources/Benchmark/BenchmarkResult.swift
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,14 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
scalingFactor: BenchmarkScalingFactor,
warmupIterations: Int,
thresholds: BenchmarkThresholds? = nil,
tags: [String: String] = [:],
statistics: Statistics) {
self.metric = metric
self.timeUnits = timeUnits == .automatic ? BenchmarkTimeUnits(statistics.units()) : timeUnits
self.scalingFactor = scalingFactor
self.warmupIterations = warmupIterations
self.thresholds = thresholds
self.tags = tags
self.statistics = statistics
}

Expand All @@ -175,6 +177,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
public var scalingFactor: BenchmarkScalingFactor
public var warmupIterations: Int
public var thresholds: BenchmarkThresholds?
public var tags: [String: String]
public var statistics: Statistics

public var scaledTimeUnits: BenchmarkTimeUnits {
Expand Down
13 changes: 13 additions & 0 deletions Tests/BenchmarkTests/BenchmarkTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,17 @@ final class BenchmarkTests: XCTestCase {
XCTAssertNotNil(benchmark)
benchmark?.run()
}

func testBenchmarkParameterizedDescription() throws {
let benchmark = Benchmark("testBenchmarkParameterizedDescription benchmark",
configuration: .init(
tags: [
"foo": "bar",
"bin": String(42),
"pi": String(3.14)
]
)) { _ in }
XCTAssertNotNil(benchmark)
XCTAssertEqual(benchmark?.name, "testBenchmarkParameterizedDescription benchmark (bin: 42, foo: bar, pi: 3.14)")
}
}

0 comments on commit e327a91

Please sign in to comment.