Interleaved comparing test

scripts/aws-bench.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This scripts is automating experiment on an AWS virtual machine
+#
+# The goal of an experiment is to measure performance variance reported by both harnesses (tango/criterion).
+# UTF8 counting routine is used as a test function. The first one is counting up to 5000 characters in a string
+# the second is up to 4950. We are expecting to see 1% difference in performance of those two functions
+
+CRITERION=./target/criterion.txt
+TANGO=./target/tango.txt
+TANGO_FILTERED=./target/tango-filtered.txt
+
+# Building and exporting all benchmarks. Align feature is used to disable inlining and to force 32-byte aligning
+# of a tested functions. Without this trick the performance of the functions on Intel platform is heavily influenced
+# by code aligning.
+cargo +nightly export ./target/benchmarks -- bench --features=align --bench=criterion
+cargo +nightly export target/benchmarks -- bench --features=align --bench='tango-*'
+
+while :
+do
+    date | tee -a "${CRITERION}" | tee -a "${TANGO}" | tee -a "${TANGO_FILTERED}"
+
+    # Running criterion benchmarks
+    ./target/benchmarks/criterion --bench str_length_495 \
+        --warm-up-time 1 --measurement-time 1 | tee -a "${CRITERION}"
+    ./target/benchmarks/criterion --bench str_length_500 \
+        --warm-up-time 1 --measurement-time 1 | tee -a "${CRITERION}"
+
+    # Running tango benchmarks
+    ./target/benchmarks/tango_faster compare ./target/benchmarks/tango_slower \
+        -t 1000 -o -f 'str_length_limit' | tee -a "${TANGO}"
+    ./target/benchmarks/tango_faster compare ./target/benchmarks/tango_slower \
+        -t 1000 -f 'str_length_limit' | tee -a "${TANGO_FILTERED}"
+done

scripts/aws-results.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+CRITERION=./target/criterion.txt
+TANGO=./target/tango.txt
+TANGO_FILTERED=./target/tango-filtered.txt
+
+if [ "$1" == "tango" ]; then
+    cat "${TANGO}" | awk '{print $(NF)}' | egrep -o '(-|\+)[0-9]+\.[0-9]+'
+fi
+
+if [ "$1" == "tango-filtered" ]; then
+    cat "${TANGO_FILTERED}" | awk '{print $(NF)}' | egrep -o '(-|\+)[0-9]+\.[0-9]+'
+fi
+
+if [ "$1" == "criterion" ]; then
+    paste \
+        <(cat "${CRITERION}" | grep -A1 "str_length_5000" | grep 'time:' | awk '{print $5}') \
+        <(cat "${CRITERION}" | grep -A1 "str_length_4950" | grep 'time:' | awk '{print $5}') | \
+        awk '{print ($2 - $1) / $1 * 100}'
+fi

scripts/tango.sh

@@ -12,7 +12,7 @@ cargo export target/benchmarks -- bench --bench='tango-*'
 time (
     for i in {1..30}; do
         ./target/benchmarks/tango_faster compare ./target/benchmarks/tango_slower \
-            -t 1000 -f 'str_length_limit' >> "${FILE}"
+            -t 1000 -o -f 'str_length_limit' >> "${FILE}"
     done
 )