Skip to content

Commit

Permalink
refactor: BenchmarkBaseline.Profile
Browse files Browse the repository at this point in the history 
Groups BenchmarkBaseline results by "profile" - meaning an execution of a specific benchmark and its various results (for each execution).
This allows us to store information pertinent to the entire benchmark in one place rather than repeating it for each BenchmarkResult.
  • Loading branch information
@CrownedPhoenix
Jairon Terrero authored and CrownedPhoenix committed Dec 5, 2024
1 parent 7706927 commit 97a55c1
Show file tree
Hide file tree
Showing 6 changed files with 64 additions and 50 deletions.
60 changes: 35 additions & 25 deletions Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,28 +84,38 @@ struct BenchmarkBaseline: Codable {
var metrics: BenchmarkResult
}

init(baselineName: String, machine: BenchmarkMachine, results: [BenchmarkIdentifier: [BenchmarkResult]]) {
init(baselineName: String, machine: BenchmarkMachine, results: [BenchmarkIdentifier: Profile]) {
self.baselineName = baselineName
self.machine = machine
self.results = results
self.profiles = results
}

// @discardableResult
mutating func merge(_ otherBaseline: BenchmarkBaseline) -> BenchmarkBaseline {
if machine != otherBaseline.machine {
print("Warning: Merging baselines from two different machine configurations")
}
results.merge(otherBaseline.results) { first, _ in first }
profiles.merge(otherBaseline.profiles) { first, _ in first }

return self
}

var baselineName: String
var machine: BenchmarkMachine
var results: BenchmarkResultsByIdentifier
var profiles: [BenchmarkIdentifier: Profile]

/// Represents a particular execution of a specific benchmark
/// and its set of results.
struct Profile: Codable {
var results: [BenchmarkResult]

init(results: [BenchmarkResult] = []) {
self.results = results
}
}

var benchmarkIdentifiers: [BenchmarkIdentifier] {
Array(results.keys).sorted(by: { ($0.target, $0.name) < ($1.target, $1.name) })
Array(profiles.keys).sorted(by: { ($0.target, $0.name) < ($1.target, $1.name) })
}

var targets: [String] {
Expand All @@ -118,8 +128,8 @@ struct BenchmarkBaseline: Codable {

var benchmarkMetrics: [BenchmarkMetric] {
var results: [BenchmarkMetric] = []
self.results.forEach { _, resultVector in
resultVector.forEach {
self.profiles.forEach { _, profile in
profile.results.forEach {
results.append($0.metric)
}
}
Expand All @@ -129,8 +139,8 @@ struct BenchmarkBaseline: Codable {

func resultEntriesMatching(_ closure: (BenchmarkIdentifier, BenchmarkResult) -> (Bool, String)) -> [ResultsEntry] {
var results: [ResultsEntry] = []
self.results.forEach { identifier, resultVector in
resultVector.forEach {
self.profiles.forEach { identifier, profile in
profile.results.forEach {
let (include, description) = closure(identifier, $0)
if include {
results.append(ResultsEntry(description: description, metrics: $0))
Expand All @@ -143,8 +153,8 @@ struct BenchmarkBaseline: Codable {

func metricsMatching(_ closure: (BenchmarkIdentifier, BenchmarkResult) -> Bool) -> [BenchmarkMetric] {
var results: [BenchmarkMetric] = []
self.results.forEach { identifier, resultVector in
resultVector.forEach {
self.profiles.forEach { identifier, profile in
profile.results.forEach {
if closure(identifier, $0) {
results.append($0.metric)
}
Expand All @@ -156,8 +166,8 @@ struct BenchmarkBaseline: Codable {

func resultsMatching(_ closure: (BenchmarkIdentifier, BenchmarkResult) -> Bool) -> [BenchmarkResult] {
var results: [BenchmarkResult] = []
self.results.forEach { identifier, resultVector in
resultVector.forEach {
self.profiles.forEach { identifier, profile in
profile.results.forEach {
if closure(identifier, $0) {
results.append($0)
}
Expand All @@ -168,8 +178,8 @@ struct BenchmarkBaseline: Codable {
}

func resultsByTarget(_ target: String) -> [String: [BenchmarkResult]] {
let filteredResults = results.filter { $0.key.target == target }.sorted(by: { $0.key.name < $1.key.name })
let resultsPerTarget = Dictionary(uniqueKeysWithValues: filteredResults.map { key, value in (key.name, value) })
let filteredResults = profiles.filter { $0.key.target == target }.sorted(by: { $0.key.name < $1.key.name })
let resultsPerTarget = Dictionary(uniqueKeysWithValues: filteredResults.map { key, value in (key.name, value.results) })

return resultsPerTarget
}
Expand Down Expand Up @@ -425,10 +435,10 @@ extension BenchmarkBaseline: Equatable {
var warningPrinted = false
var allDeviationResults = BenchmarkResult.ThresholdDeviations()

for (lhsBenchmarkIdentifier, lhsBenchmarkResults) in lhs.results {
for lhsBenchmarkResult in lhsBenchmarkResults {
if let rhsResults = rhs.results.first(where: { $0.key == lhsBenchmarkIdentifier }) {
if let rhsBenchmarkResult = rhsResults.value.first(where: { $0.metric == lhsBenchmarkResult.metric }) {
for (lhsBenchmarkIdentifier, lhsBenchmarkProfiles) in lhs.profiles {
for lhsBenchmarkResult in lhsBenchmarkProfiles.results {
if let rhsProfile = rhs.profiles.first(where: { $0.key == lhsBenchmarkIdentifier }) {
if let rhsBenchmarkResult = rhsProfile.value.results.first(where: { $0.metric == lhsBenchmarkResult.metric }) {
let thresholds = thresholdsForBenchmarks(benchmarks,
name: lhsBenchmarkIdentifier.name,
target: lhsBenchmarkIdentifier.target,
Expand Down Expand Up @@ -462,8 +472,8 @@ extension BenchmarkBaseline: Equatable {
[BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]]) -> BenchmarkResult.ThresholdDeviations {
var allDeviationResults = BenchmarkResult.ThresholdDeviations()

for (lhsBenchmarkIdentifier, lhsBenchmarkResults) in results {
for lhsBenchmarkResult in lhsBenchmarkResults {
for (lhsBenchmarkIdentifier, lhsBenchmarkProfile) in profiles {
for lhsBenchmarkResult in lhsBenchmarkProfile.results {
let thresholds = thresholdsForBenchmarks(benchmarks,
name: lhsBenchmarkIdentifier.name,
target: lhsBenchmarkIdentifier.target,
Expand Down Expand Up @@ -492,10 +502,10 @@ extension BenchmarkBaseline: Equatable {
return false
}

for (lhsBenchmarkIdentifier, lhsBenchmarkResults) in lhs.results {
for lhsBenchmarkResult in lhsBenchmarkResults {
if let rhsResults = rhs.results.first(where: { $0.key == lhsBenchmarkIdentifier }) {
if let rhsBenchmarkResult = rhsResults.value.first(where: { $0.metric == lhsBenchmarkResult.metric }) {
for (lhsBenchmarkIdentifier, lhsBenchmarkProfile) in lhs.profiles {
for lhsBenchmarkResult in lhsBenchmarkProfile.results {
if let rhsProfile = rhs.profiles.first(where: { $0.key == lhsBenchmarkIdentifier }) {
if let rhsBenchmarkResult = rhsProfile.value.results.first(where: { $0.metric == lhsBenchmarkResult.metric }) {
if lhsBenchmarkResult != rhsBenchmarkResult {
return false
}
Expand Down
8 changes: 4 additions & 4 deletions Plugins/BenchmarkTool/BenchmarkTool+Export+InfluxCSVFormatter.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ extension BenchmarkTool {
baseline.targets.forEach { key in
let exportStruct = saveExportableResults(BenchmarkBaseline(baselineName: baseline.baselineName,
machine: benchmarkMachine(),
results: baseline.results),
results: baseline.profiles),
target: key)

let formatter = InfluxCSVFormatter(exportableBenchmark: exportStruct)
Expand All @@ -128,14 +128,14 @@ extension BenchmarkTool {
}

func saveExportableResults(_ benchmarks: BenchmarkBaseline, target: String) -> ExportableBenchmark {
var keys = benchmarks.results.keys.sorted(by: { $0.name < $1.name })
var keys = benchmarks.profiles.keys.sorted(by: { $0.name < $1.name })
var testList: [TestData] = []
keys.removeAll(where: { $0.target != target })

keys.forEach { test in
if let value = benchmarks.results[test] {
if let profile = benchmarks.profiles[test] {
var allResults: [BenchmarkResult] = []
value.forEach { result in
profile.results.forEach { result in
allResults.append(result)
}

Expand Down
20 changes: 10 additions & 10 deletions Plugins/BenchmarkTool/BenchmarkTool+Export.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,8 @@ extension BenchmarkTool {
try write(exportData: "\(convertToInflux(baseline))",
fileName: "\(baselineName).influx.csv")
case .histogram:
try baseline.results.forEach { key, results in
try results.forEach { values in
try baseline.profiles.forEach { key, profile in
try profile.results.forEach { values in
let outputString = values.statistics.histogram
let description = values.metric.rawDescription
try write(exportData: "\(outputString)",
Expand All @@ -169,10 +169,10 @@ extension BenchmarkTool {
try write(exportData: "\(convertToJMH(baseline))",
fileName: cleanupStringForShellSafety("\(baselineName).jmh.json"))
case .histogramSamples:
try baseline.results.forEach { key, results in
try baseline.profiles.forEach { key, profile in
var outputString = ""

try results.forEach { values in
try profile.results.forEach { values in
let histogram = values.statistics.histogram

outputString += "\(values.metric.description) \(values.unitDescriptionPretty)\n"
Expand All @@ -189,10 +189,10 @@ extension BenchmarkTool {
}
}
case .histogramEncoded:
try baseline.results.forEach { key, results in
try baseline.profiles.forEach { key, profile in
let encoder = JSONEncoder()

try results.forEach { values in
try profile.results.forEach { values in
let histogram = values.statistics.histogram
let jsonData = try encoder.encode(histogram)
let description = values.metric.rawDescription
Expand All @@ -207,8 +207,8 @@ extension BenchmarkTool {
case .histogramPercentiles:
var outputString = ""

try baseline.results.forEach { key, results in
try results.forEach { values in
try baseline.profiles.forEach { key, profile in
try profile.results.forEach { values in
let histogram = values.statistics.histogram

outputString += "Percentile\t" + "\(values.metric.description) \(values.unitDescriptionPretty)\n"
Expand All @@ -224,12 +224,12 @@ extension BenchmarkTool {
}
}
case .metricP90AbsoluteThresholds:
try baseline.results.forEach { key, results in
try baseline.profiles.forEach { key, profile in
let jsonEncoder = JSONEncoder()
jsonEncoder.outputFormatting = [.prettyPrinted, .sortedKeys]

var outputResults: [String: BenchmarkThresholds.AbsoluteThreshold] = [:]
results.forEach { values in
profile.results.forEach { values in
outputResults[values.metric.rawDescription] = Int(values.statistics.histogram.valueAtPercentile(90.0))
}

Expand Down
6 changes: 3 additions & 3 deletions Plugins/BenchmarkTool/BenchmarkTool+Operations.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ extension BenchmarkTool {
}

if benchmarks.isEmpty { // if we read from baseline and didn't run them, we put in some fake entries for the compare
currentBaseline.results.keys.forEach { baselineKey in
currentBaseline.profiles.keys.forEach { baselineKey in
if let benchmark: Benchmark = .init(baselineKey.name, closure:{_ in}) {
benchmark.target = baselineKey.target
benchmarks.append(benchmark)
Expand Down Expand Up @@ -230,7 +230,7 @@ extension BenchmarkTool {
let baseline = benchmarkBaselines[0]
if let baselineName = self.baseline.first {
try baseline.targets.forEach { target in
let results = baseline.results.filter { $0.key.target == target }
let results = baseline.profiles.filter { $0.key.target == target }
let subset = BenchmarkBaseline(baselineName: baselineName,
machine: baseline.machine,
results: results)
Expand All @@ -257,7 +257,7 @@ extension BenchmarkTool {
}

if benchmarks.isEmpty { // if we read from baseline and didn't run them, we put in some fake entries for the compare
currentBaseline.results.keys.forEach { baselineKey in
currentBaseline.profiles.keys.forEach { baselineKey in
if let benchmark: Benchmark = .init(baselineKey.name, closure:{_ in}) {
benchmark.target = baselineKey.target
benchmarks.append(benchmark)
Expand Down
10 changes: 5 additions & 5 deletions Plugins/BenchmarkTool/BenchmarkTool+PrettyPrinting.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,15 +207,15 @@ extension BenchmarkTool {
let baseBaselineName = currentBaseline.baselineName
let comparisonBaselineName = baseline.baselineName

var keys = baseline.results.keys.sorted(by: { $0.name < $1.name })
var keys = baseline.profiles.keys.sorted(by: { $0.name < $1.name })

keys.removeAll(where: { $0.target != target })

var firstOutput = true

keys.forEach { key in
if let value = baseline.results[key] {
guard let baselineComparison = currentBaseline.results[key] else {
if let profile = baseline.profiles[key] {
guard let baselineComparison = currentBaseline.profiles[key] else {
// print("No baseline to compare with for `\(key.target):\(key.name)`.")
return
}
Expand All @@ -234,9 +234,9 @@ extension BenchmarkTool {
printText("----------------------------------------------------------------------------------------------------------------------------")
print("")

value.forEach { currentResult in
profile.results.forEach { currentResult in
var result = currentResult
if let base = baselineComparison.first(where: { $0.metric == result.metric }) {
if let base = baselineComparison.results.first(where: { $0.metric == result.metric }) {
let hideResults = result.deviationsComparedWith(base, thresholds: result.thresholds ?? BenchmarkThresholds.none).regressions.isEmpty

// We hide the markdown results if they are better than baseline to cut down noise
Expand Down
10 changes: 7 additions & 3 deletions Plugins/BenchmarkTool/BenchmarkTool.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ struct BenchmarkTool: AsyncParsableCommand {
"Running Benchmarks".printAsHeader()
}

var benchmarkResults: BenchmarkResultsByIdentifier = [:]
var benchmarkResults: [BenchmarkIdentifier: BenchmarkBaseline.Profile] = [:]

benchmarks.sort { ($0.target, $0.name) < ($1.target, $1.name) }

Expand All @@ -303,8 +303,12 @@ struct BenchmarkTool: AsyncParsableCommand {
printChildRunError(error: result, benchmarkExecutablePath: benchmark.executablePath!)
}
}

benchmarkResults = benchmarkResults.merging(results) { _, new in new }

for result in results {
benchmarkResults[result.key] = BenchmarkBaseline.Profile(
results: result.value
)
}
}
}

Expand Down

0 comments on commit 97a55c1

Please sign in to comment.