From 77d6de04c86a6d616c84132d499940301ea6205c Mon Sep 17 00:00:00 2001
From: Olivier Giniaux <oginiaux@smartadserver.com>
Date: Thu, 23 Nov 2023 09:46:23 +0100
Subject: [PATCH] Remove outliers in throughput benchmark

---
 benches/throughput/main.rs             | 41 ++++++++++++++++++++++----
 benches/throughput/result_processor.rs |  6 ++--
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/benches/throughput/main.rs b/benches/throughput/main.rs
index 8ca9e8b..3b78fa7 100644
--- a/benches/throughput/main.rs
+++ b/benches/throughput/main.rs
@@ -103,12 +103,11 @@ fn benchmark<F, S>(processor: &mut dyn ResultProcessor, data: &[u8], name: &str,
         // Warmup
         black_box(time(ITERATIONS, &|| delegate(&data[..len], S::default()))); 
 
-        let mut total_duration: Duration = Duration::ZERO;
-        let mut runs: usize = 0;
+        let mut durations_s = vec![];
         let now = Instant::now();
         while now.elapsed() < MAX_RUN_DURATION {
             // Make seed unpredictable to prevent optimizations
-            let seed = S::try_from(total_duration.as_nanos())
+            let seed = S::try_from(now.elapsed().as_nanos())
                 .unwrap_or_else(|_| panic!("Something went horribly wrong!"));
             // Offset slice by an unpredictable amount to prevent optimization (pre caching)
             // and make the benchmark use both aligned and unaligned data
@@ -117,10 +116,11 @@ fn benchmark<F, S>(processor: &mut dyn ResultProcessor, data: &[u8], name: &str,
             let end = start + len;
             let slice = &data[start..end];
             // Execute method for a new iterations
-            total_duration += time(ITERATIONS, &|| delegate(slice, S::default()));
-            runs += 1;
+            let duration = time(ITERATIONS, &|| delegate(slice, S::default()));
+            durations_s.push(duration.as_secs_f64());
         }
-        let throughput = (len as f64) / (1024f64 * 1024f64 * (total_duration.as_secs_f64() / runs as f64 / ITERATIONS as f64));
+        let average_duration_s = calculate_average_without_outliers(&mut durations_s);
+        let throughput = (len as f64) / (1024f64 * 1024f64 * (average_duration_s / ITERATIONS as f64));
 
         processor.on_result(len, throughput);
     }
@@ -151,4 +151,33 @@ fn execute_noinlining<F>(delegate: &F) -> u64
     where F: Fn() -> u64
 {
     delegate()
+}
+
+// Outliers are inevitable, especially on a low number of iterations
+// To avoid computing a huge number of iterations we can use the interquartile range
+fn calculate_average_without_outliers(timings: &mut Vec<f64>) -> f64 {
+    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+    let q1 = percentile(timings, 25.0);
+    let q3 = percentile(timings, 75.0);
+    let iqr = q3 - q1;
+
+    let lower_bound = q1 - 1.5 * iqr;
+    let upper_bound = q3 + 1.5 * iqr;
+
+    let filtered_timings: Vec<f64> = timings
+        .iter()
+        .filter(|&&x| x >= lower_bound && x <= upper_bound)
+        .cloned()
+        .collect();
+
+    let sum: f64 = filtered_timings.iter().sum();
+    let count = filtered_timings.len();
+
+    sum / count as f64
+}
+
+fn percentile(sorted_data: &Vec<f64>, percentile: f64) -> f64 {
+    let idx = (percentile / 100.0 * (sorted_data.len() - 1) as f64).round() as usize;
+    sorted_data[idx]
 }
\ No newline at end of file
diff --git a/benches/throughput/result_processor.rs b/benches/throughput/result_processor.rs
index fbc7526..49bece6 100644
--- a/benches/throughput/result_processor.rs
+++ b/benches/throughput/result_processor.rs
@@ -138,12 +138,12 @@ impl ResultProcessor for OutputPlot {
             .caption(format!("Throughput ({})", arch), ("sans-serif", (5).percent_height()))
             .set_label_area_size(LabelAreaPosition::Left, (14).percent())
             .set_label_area_size(LabelAreaPosition::Bottom, (10).percent())
-            .margin((1).percent())
+            .margin_right((5).percent())
             .build_cartesian_2d(
                 (x_min..x_max)
                     .log_scale()
                     .with_key_points(self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).collect::<Vec<u32>>()),
-                    (y_min..y_max)
+                    y_min..y_max
                     //.log_scale(),
             ).unwrap();
 
@@ -170,7 +170,7 @@ impl ResultProcessor for OutputPlot {
         chart
             .configure_series_labels()
             .border_style(BLACK)
-            .background_style(RGBColor(255, 255, 255))
+            .background_style(RGBAColor(255, 255, 255, 0.7f64))
             .draw().unwrap();
 
         // To avoid the IO failure being ignored silently, we manually call the present function