Remove outliers in throughput benchmark

ogxd · Nov 23, 2023 · 6dca688 · 6dca688
1 parent aa49a98
commit 6dca688
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 7 deletions.
diff --git a/benches/throughput/main.rs b/benches/throughput/main.rs
@@ -103,8 +103,7 @@ fn benchmark<F, S>(processor: &mut dyn ResultProcessor, data: &[u8], name: &str,
         // Warmup
         black_box(time(ITERATIONS, &|| delegate(&data[..len], S::default()))); 
 
-        let mut total_duration: Duration = Duration::ZERO;
-        let mut runs: usize = 0;
+        let mut durations_s = vec![];
         let now = Instant::now();
         while now.elapsed() < MAX_RUN_DURATION {
             // Make seed unpredictable to prevent optimizations
@@ -117,10 +116,11 @@ fn benchmark<F, S>(processor: &mut dyn ResultProcessor, data: &[u8], name: &str,
             let end = start + len;
             let slice = &data[start..end];
             // Execute method for a new iterations
-            total_duration += time(ITERATIONS, &|| delegate(slice, S::default()));
-            runs += 1;
+            let duration = time(ITERATIONS, &|| delegate(slice, S::default()));
+            durations_s.push(duration.as_secs_f64());
         }
-        let throughput = (len as f64) / (1024f64 * 1024f64 * (total_duration.as_secs_f64() / runs as f64 / ITERATIONS as f64));
+        let average_duration_s = calculate_average_without_outliers(&mut durations_s);
+        let throughput = (len as f64) / (1024f64 * 1024f64 * (average_duration_s / ITERATIONS as f64));
 
         processor.on_result(len, throughput);
     }
@@ -151,4 +151,33 @@ fn execute_noinlining<F>(delegate: &F) -> u64
     where F: Fn() -> u64
 {
     delegate()
+}
+
+// Outliers are inevitable, especially on a low number of iterations
+// To avoid computing a huge number of iterations we can use the interquartile range
+fn calculate_average_without_outliers(timings: &mut Vec<f64>) -> f64 {
+    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+    let q1 = percentile(timings, 25.0);
+    let q3 = percentile(timings, 75.0);
+    let iqr = q3 - q1;
+
+    let lower_bound = q1 - 1.5 * iqr;
+    let upper_bound = q3 + 1.5 * iqr;
+
+    let filtered_timings: Vec<f64> = timings
+        .iter()
+        .filter(|&&x| x >= lower_bound && x <= upper_bound)
+        .cloned()
+        .collect();
+
+    let sum: f64 = filtered_timings.iter().sum();
+    let count = filtered_timings.len();
+
+    sum / count as f64
+}
+
+fn percentile(sorted_data: &Vec<f64>, percentile: f64) -> f64 {
+    let idx = (percentile / 100.0 * (sorted_data.len() - 1) as f64).round() as usize;
+    sorted_data[idx]
 }
diff --git a/benches/throughput/result_processor.rs b/benches/throughput/result_processor.rs
@@ -138,7 +138,7 @@ impl ResultProcessor for OutputPlot {
             .caption(format!("Throughput ({})", arch), ("sans-serif", (5).percent_height()))
             .set_label_area_size(LabelAreaPosition::Left, (14).percent())
             .set_label_area_size(LabelAreaPosition::Bottom, (10).percent())
-            .margin((1).percent())
+            .margin_right((5).percent())
             .build_cartesian_2d(
                 (x_min..x_max)
                     .log_scale()
@@ -170,7 +170,7 @@ impl ResultProcessor for OutputPlot {
         chart
             .configure_series_labels()
             .border_style(BLACK)
-            .background_style(RGBColor(255, 255, 255))
+            .background_style(RGBAColor(255, 255, 255, 0.7f64))
             .draw().unwrap();
 
         // To avoid the IO failure being ignored silently, we manually call the present function