feat(minor): Add more thorough setup/teardown hooks (#153)

Adding support for global, shared and local setup/teardown hooks in a systemic manner. Fixes #142 Co-authored-by: dimlio <[email protected]>
ordo-one · Apr 21, 2023 · eb5c4b2 · eb5c4b2
1 parent c838d01
commit eb5c4b2
Show file tree

Hide file tree

Showing 7 changed files with 243 additions and 27 deletions.
diff --git a/Benchmarks/Basic/Basic+SetupTeardown.swift b/Benchmarks/Basic/Basic+SetupTeardown.swift
@@ -0,0 +1,45 @@
+//
+//  File.swift
+//  
+//
+//  Created by Joakim Hassila on 2023-04-21.
+//
+
+import Benchmark
+
+func sharedSetup() {
+//    print("Shared setup hook")
+}
+
+func sharedTeardown() {
+//    print("Shared teardown hook")
+}
+
+func testSetUpTearDown() {
+//    Benchmark.setup = { print("Global setup hook") }
+//    Benchmark.teardown = { print("Global teardown hook") }
+
+    Benchmark("SetupTeardown",
+              configuration: .init(setup: sharedSetup, teardown: sharedTeardown)) { _ in
+    } setup: {
+//        print("Local setup hook")
+    } teardown: {
+//        print("Local teardown hook")
+    }
+
+    Benchmark("SetupTeardown2",
+              configuration: .init(setup: sharedSetup, teardown: sharedTeardown)) { _ in
+    }
+
+    Benchmark("SetupTeardown3",
+              configuration: .init(setup: sharedSetup)) { _ in
+    } teardown: {
+//        print("Local teardown hook")
+    }
+
+    Benchmark("SetupTeardown4",
+              configuration: .init(setup: sharedSetup)) { _ in
+    } setup: {
+//        print("Local setup hook")
+    }
+}
diff --git a/Benchmarks/Basic/BenchmarkRunner+Basic.swift b/Benchmarks/Basic/BenchmarkRunner+Basic.swift
@@ -34,8 +34,8 @@ let benchmarks = {
                                            maxIterations: Int.max,
                                            thresholds: thresholds)
 
-    // Benchmark.startupHook = { print("Startup hook") }
-    // Benchmark.shutdownHook = { print("Shutdown hook") }
+    testSetUpTearDown()
+
     // A way to define custom metrics fairly compact
     enum CustomMetrics {
         static var one: BenchmarkMetric { .custom("CustomMetricOne") }

diff --git a/...s/DateTime/BenchmarkRunner+DateTime.swift → Benchmarks/DateTime/DateTime.swift b/...s/DateTime/BenchmarkRunner+DateTime.swift → Benchmarks/DateTime/DateTime.swift
diff --git a/...Histogram/BenchmarkRunner+Histogram.swift → Benchmarks/Histogram/Histogram.swift b/...Histogram/BenchmarkRunner+Histogram.swift → Benchmarks/Histogram/Histogram.swift
diff --git a/Sources/Benchmark/Benchmark.swift b/Sources/Benchmark/Benchmark.swift
@@ -41,12 +41,23 @@ public final class Benchmark: Codable, Hashable {
 
     /// Alias for closures used to hook into setup / teardown
     public typealias BenchmarkHook = () async throws -> Void
+    public typealias BenchmarkSetupTeardownHook = BenchmarkHook
+
+    #if swift(>=5.8)
+        @_documentation(visibility: internal)
+    #endif
+    public static var startupHook: BenchmarkSetupTeardownHook? // Should be removed when going to 2.0, just kept for API compatiblity
+
+    #if swift(>=5.8)
+        @_documentation(visibility: internal)
+    #endif
+    public static var shutdownHook: BenchmarkSetupTeardownHook? // Should be removed when going to 2.0, just kept for API compatiblity
 
     /// This closure if set, will be run before a targets benchmarks are run, but after they are registered
-    public static var startupHook: BenchmarkHook?
+    public static var setup: BenchmarkSetupTeardownHook?
 
     /// This closure if set, will be run after a targets benchmarks run, but after they are registered
-    public static var shutdownHook: BenchmarkHook?
+    public static var teardown: BenchmarkSetupTeardownHook?
 
     /// Set to true if this benchmark results should be compared with an absolute threshold when `--check-absolute` is
     /// specified on the command line. An implementation can then choose to configure thresholds differently for
@@ -83,6 +94,9 @@ public final class Benchmark: Codable, Hashable {
     var closure: BenchmarkClosure? // The actual benchmark to run
     /// asyncClosure: The actual benchmark (async) closure that will be measured
     var asyncClosure: BenchmarkAsyncClosure? // The actual benchmark to run
+    // setup/teardown hooks for the instance
+    var setup: BenchmarkSetupTeardownHook?
+    var teardown: BenchmarkSetupTeardownHook?
 
     // Hooks for benchmark infrastructure to capture metrics of actual measurement() block without preamble:
     #if swift(>=5.8)
@@ -144,14 +158,18 @@ public final class Benchmark: Codable, Hashable {
     @discardableResult
     public init?(_ name: String,
                  configuration: Benchmark.Configuration = Benchmark.defaultConfiguration,
-                 closure: @escaping BenchmarkClosure) {
+                 closure: @escaping BenchmarkClosure,
+                 setup: BenchmarkSetupTeardownHook? = nil,
+                 teardown: BenchmarkSetupTeardownHook? = nil) {
         if configuration.skip {
             return nil
         }
         target = ""
         self.name = name
         self.configuration = configuration
         self.closure = closure
+        self.setup = setup
+        self.teardown = teardown
 
         benchmarkRegistration()
     }
@@ -165,14 +183,18 @@ public final class Benchmark: Codable, Hashable {
     @discardableResult
     public init?(_ name: String,
                  configuration: Benchmark.Configuration = Benchmark.defaultConfiguration,
-                 closure: @escaping BenchmarkAsyncClosure) {
+                 closure: @escaping BenchmarkAsyncClosure,
+                 setup: BenchmarkSetupTeardownHook? = nil,
+                 teardown: BenchmarkSetupTeardownHook? = nil) {
         if configuration.skip {
             return nil
         }
         target = ""
         self.name = name
         self.configuration = configuration
         asyncClosure = closure
+        self.setup = setup
+        self.teardown = teardown
 
         benchmarkRegistration()
     }
@@ -186,14 +208,16 @@ public final class Benchmark: Codable, Hashable {
     @discardableResult
     public convenience init?(_ name: String,
                              configuration: Benchmark.Configuration = Benchmark.defaultConfiguration,
-                             closure: @escaping BenchmarkThrowingClosure) {
-        self.init(name, configuration: configuration) { benchmark in
+                             closure: @escaping BenchmarkThrowingClosure,
+                             setup: BenchmarkSetupTeardownHook? = nil,
+                             teardown: BenchmarkSetupTeardownHook? = nil) {
+        self.init(name, configuration: configuration, closure: { benchmark in
             do {
                 try closure(benchmark)
             } catch {
                 benchmark.error("Benchmark \(name) failed with \(error)")
             }
-        }
+        }, setup: setup, teardown: teardown)
     }
 
     /// Definition of an async throwing Benchmark
@@ -205,14 +229,16 @@ public final class Benchmark: Codable, Hashable {
     @discardableResult
     public convenience init?(_ name: String,
                              configuration: Benchmark.Configuration = Benchmark.defaultConfiguration,
-                             closure: @escaping BenchmarkAsyncThrowingClosure) {
-        self.init(name, configuration: configuration) { benchmark in
+                             closure: @escaping BenchmarkAsyncThrowingClosure,
+                             setup: BenchmarkSetupTeardownHook? = nil,
+                             teardown: BenchmarkSetupTeardownHook? = nil) {
+        self.init(name, configuration: configuration, closure: { benchmark in
             do {
                 try await closure(benchmark)
             } catch {
                 benchmark.error("Benchmark \(name) failed with \(error)")
             }
-        }
+        }, setup: setup, teardown: teardown)
     }
 
     // Shared between sync/async actual benchmark registration
@@ -343,6 +369,10 @@ public extension Benchmark {
         public var skip = false
         /// Customized CI failure thresholds for a given metric for the Benchmark
         public var thresholds: [BenchmarkMetric: BenchmarkThresholds]?
+        /// Optional per-benchmark specific setup done before warmup and all iterations
+        public var setup: BenchmarkSetupTeardownHook?
+        /// Optional per-benchmark specific teardown done after final run is done
+        public var teardown: BenchmarkSetupTeardownHook?
 
         public init(metrics: [BenchmarkMetric] = defaultConfiguration.metrics,
                     timeUnits: BenchmarkTimeUnits = defaultConfiguration.timeUnits,
@@ -352,7 +382,9 @@ public extension Benchmark {
                     maxIterations: Int = defaultConfiguration.maxIterations,
                     skip: Bool = defaultConfiguration.skip,
                     thresholds: [BenchmarkMetric: BenchmarkThresholds]? =
-                        defaultConfiguration.thresholds) {
+                        defaultConfiguration.thresholds,
+                    setup: BenchmarkSetupTeardownHook? = nil,
+                    teardown: BenchmarkSetupTeardownHook? = nil) {
             self.metrics = metrics
             self.timeUnits = timeUnits
             self.warmupIterations = warmupIterations
@@ -361,6 +393,8 @@ public extension Benchmark {
             self.maxIterations = maxIterations
             self.skip = skip
             self.thresholds = thresholds
+            self.setup = setup
+            self.teardown = teardown
         }
 
         // swiftlint:disable nesting

diff --git a/Sources/Benchmark/BenchmarkRunner.swift b/Sources/Benchmark/BenchmarkRunner.swift
@@ -113,13 +113,6 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
         var benchmark: Benchmark?
         var results: [BenchmarkResult] = []
 
-        do {
-            try await Benchmark.startupHook?()
-        } catch {
-            try channel.write(.error("Benchmark.startupHook failed: \(error)"))
-            return
-        }
-
         while true {
             if debug { // in debug mode we run all benchmarks matching filter/skip specified
                 var benchmark: Benchmark?
@@ -174,8 +167,40 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
 
                     benchmark.target = benchmarkToRun.target
 
+                    do {
+                        try await Benchmark.startupHook?()
+                        try await Benchmark.setup?()
+
+                        if let setup = benchmark.configuration.setup {
+                            try await setup()
+                        }
+
+                        if let setup = benchmark.setup {
+                            try await setup()
+                        }
+                    } catch {
+                        try channel.write(.error("Benchmark.setup or local benchmark setup failed: \(error)"))
+                        return
+                    }
+
                     results = benchmarkExecutor.run(benchmark)
 
+                    do {
+                        if let teardown = benchmark.teardown {
+                            try await teardown()
+                        }
+
+                        if let teardown = benchmark.configuration.teardown {
+                            try await teardown()
+                        }
+
+                        try await Benchmark.shutdownHook?()
+                        try await Benchmark.teardown?()
+                    } catch {
+                        try channel.write(.error("Benchmark.teardown or local benchmark teardown failed: \(error)"))
+                        return
+                    }
+
                     guard benchmark.failureReason == nil else {
                         try channel.write(.error(benchmark.failureReason!))
                         return
@@ -202,13 +227,6 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
                     print("Internal error: Couldn't find specified benchmark '\(benchmarkToRun.name)' to run.")
                 }
 
-                do {
-                    try await Benchmark.shutdownHook?()
-                } catch {
-                    try channel.write(.error("Benchmark.shutdownHook failed: \(error)"))
-                    return
-                }
-
                 try channel.write(.end)
             case .end:
                 return

diff --git a/Sources/Benchmark/Documentation.docc/WritingBenchmarks.md b/Sources/Benchmark/Documentation.docc/WritingBenchmarks.md
@@ -216,6 +216,125 @@ Benchmark.defaultConfiguration = .init(...)
 
 There are a number of convenience methods in `BenchmarkThreshold+Defaults.swift`.
 
+### Customize startup/teardown code for benchmarks
+
+There are multiple ways to setup shared benchmark setup/teardown code.
+
+First off, one can easily setup global shared state that can be reused by just defining it in the closure:
+
+```swift
+import Benchmark
+
+let benchmarks = {
+    let mySharedSetup = [1, 2, 3, 4]
+
+    Benchmark("Minimal benchmark") { benchmark in
+    // Some work to measure here, use mySharedSetup
+    }
+
+    Benchmark("Minimal benchmark 2") { benchmark in
+    // Some work to measure here, use mySharedSetup here too
+    }
+}
+```
+
+Secondly, one can have setup/teardown closures that are shared across all benchmarks:
+
+```swift
+import Benchmark
+
+let benchmarks = {
+    Benchmark.setup = { print("global setup closure, used for all benchmarks") }
+    Benchmark.teardown = { print("global teardown closure, used for all benchmarks") }
+
+    Benchmark("Minimal benchmark") { benchmark in
+    // Some work to measure here, use mySharedSetup
+    }
+
+    Benchmark("Minimal benchmark 2") { benchmark in
+    // Some work to measure here, use mySharedSetup here too
+    }
+}
+```
+
+Thirdly, one can have setup/teardown closures as part of the configuration for a subset of benchmarks
+
+```swift
+import Benchmark
+
+func setupFunction() {
+}
+
+func teardownFunction() {
+}
+
+let benchmarks = {
+
+// only shared setup
+Benchmark("Minimal benchmark", 
+    configuration: .init(setup: setupFunction, teardown: teardownFunction)) { benchmark in 
+    // Some work to measure here
+}
+
+Benchmark("Minimal benchmark 2", 
+    configuration: .init(setup: setupFunction, teardown: teardownFunction)) { benchmark in
+    // Some work to measure here
+    }
+}
+```
+
+Finally, one can have setup/teardown closures that are specific to a given benchmark
+
+```swift
+import Benchmark
+
+let benchmarks = {
+
+Benchmark("Minimal benchmark") { benchmark in 
+    } setup: {
+     // do setup for this benchmark here
+    } teardown: {
+     // do teardown here
+    }
+}
+```
+
+All of these setup/teardown hooks can be combined, the order of execution is:
+
+* Global setup
+* Configuration provided setup
+* Closure provided setup
+
+with teardown in reverse order.
+
+So to use all hooks at the same time:
+
+```swift
+import Benchmark
+
+func sharedSetup() {
+}
+
+func sharedTeardown() {
+}
+
+let benchmarks = {
+
+    Benchmark.setup = { print("global setup closure, used for all benchmarks") }
+    Benchmark.teardown = { print("global teardown closure, used for all benchmarks") }
+
+
+    Benchmark("Minimal benchmark", 
+    configuration: .init(setup: setupFunction, teardown: teardownFunction)) { benchmark in 
+    } setup: {
+    // do setup for this benchmark here
+    } teardown: {
+    // do teardown here
+    }
+}
+```
+
+
 ### Async vs Sync
 
 The framework supports both synchronous and asynchronous benchmark closures, it should transparently "just work".