Speedup benches

.github/workflows/native.yml

@@ -195,13 +195,13 @@ jobs:
           wget https://github.com/sharkdp/hyperfine/releases/download/v1.6.0/hyperfine_1.6.0_amd64.deb
           sudo dpkg -i hyperfine_1.6.0_amd64.deb
       - name: Build benchmark
-        run: ./build_bench.sh
+        run: ./build_bench.sh debug
       - name: Test benchmark
         run: |
           chmod +x openfst-1.7.2/bin/fst*
           python -m pip install -e rustfst-python-bench
-          python rustfst-python-bench/rustfst_python_bench/bench_all.py rustfst-tests-data/fst_003/raw_vector.fst report.md
-          python rustfst-python-bench/rustfst_python_bench/bench_all_detailed.py rustfst-tests-data/fst_003/raw_vector.fst report2.md
+          python rustfst-python-bench/rustfst_python_bench/bench_all.py -w 1 -r 1 debug rustfst-tests-data/fst_003/raw_vector.fst report.md
+          python rustfst-python-bench/rustfst_python_bench/bench_all_detailed.py -w 1 -r 1 debug rustfst-tests-data/fst_003/raw_vector.fst report2.md
 
   rustfst-python:
     name: rustfst-python

build_bench.sh

@@ -1,15 +1,33 @@
 #!/usr/bin/env bash
-set -e
+set -ex
+
+mode=$1
+
+case $mode in
+  "release")
+    CPP_FLAGS="-O3"
+    RUST_FLAGS="--release"
+    ;;
+
+  "debug")
+    CPP_FLAGS=""
+    RUST_FLAGS=""
+    ;;
+
+  *)
+    echo "$mode not supported. Expected debug or release" && exit 1
+    ;;
+esac
 
 cd openfst_benchmark
 
 for file in `ls *.cpp`
 do
     echo "Building $file"
-    g++ -O3 -std=c++11 $file -I ../openfst-1.7.2/src/include/ ../openfst-1.7.2/lib/libfst.a -o "${file%.*}"
+    g++ $CPP_FLAGS -std=c++11 $file -I ../openfst-1.7.2/src/include/ ../openfst-1.7.2/lib/libfst.a -o "${file%.*}"
 done
 
 cd ..
 
-cargo build --release -p rustfst-cli
+cargo build $RUST_FLAGS -p rustfst-cli
 

rustfst-python-bench/rustfst_python_bench/bench_all.py

@@ -6,7 +6,7 @@
 import tempfile
 
 from rustfst_python_bench.algorithms.supported_algorithms import SupportedAlgorithms
-from rustfst_python_bench.constants import OPENFST_BINS, RUSTFST_CLI
+from rustfst_python_bench.constants import OPENFST_BINS, get_rusftfst_cli_dir
 from rustfst_python_bench.utils import header_report
 
 
@@ -15,6 +15,12 @@ def parse():
         description="Script to bench all CLIs of OpenFST and RustFST"
     )
 
+    parser.add_argument(
+        "compilation_mode",
+        type=str,
+        choices=["debug", "release"]
+    )
+
     parser.add_argument(
         "path_in_fst",
         type=str,
@@ -46,15 +52,16 @@ def parse():
     return args
 
 
-def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs, algo):
+def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs, algo, compilation_mode):
 
     openfst_cli = os.path.join(OPENFST_BINS, algo.openfst_cli())
+    rustfst_cli = get_rusftfst_cli_dir(compilation_mode)
 
     path_out_openfst = os.path.join(results_dir, f'{algo_name}_openfst.fst')
     path_out_rustfst = os.path.join(results_dir, f'{algo_name}_rustfst.fst')
 
     cmd_openfst = f"{openfst_cli} {algo.get_cli_args()} {path_in_fst} {path_out_openfst}"
-    cmd_rustfst = f"{RUSTFST_CLI} {algo.rustfst_subcommand()} {algo.get_cli_args()} {path_in_fst} {path_out_rustfst}"
+    cmd_rustfst = f"{rustfst_cli} {algo.rustfst_subcommand()} {algo.get_cli_args()} {path_in_fst} {path_out_rustfst}"
 
     cmd = f"hyperfine -w {warmup} -r {runs} '{cmd_openfst}' '{cmd_rustfst}'" \
           f" --export-markdown {path_report_md} --show-output"
@@ -63,7 +70,7 @@ def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs
     algo.check_correctness(path_out_openfst, path_out_rustfst)
 
 
-def bench(path_in_fst, path_report_md, warmup, runs):
+def bench(path_in_fst, path_report_md, warmup, runs, compilation_mode):
 
     with io.open(path_report_md, mode="w") as report_f:
         report_f.write("# Benchmark OpenFST CLI vs RustFST CLI\n")
@@ -76,7 +83,7 @@ def bench(path_in_fst, path_report_md, warmup, runs):
                     params = algo.get_parameters()
                     report_f.write(f"## {algoname.capitalize()}\n")
                     for param in params:
-                        bench_algo(algoname, path_in_fst, tmpdirname, report_path_temp, warmup, runs, param)
+                        bench_algo(algoname, path_in_fst, tmpdirname, report_path_temp, warmup, runs, param, compilation_mode)
 
                         with io.open(report_path_temp, mode="r") as f:
 
@@ -90,7 +97,7 @@ def bench(path_in_fst, path_report_md, warmup, runs):
 
 def main():
     args = parse()
-    bench(args.path_in_fst, args.path_report_md, args.warmup, args.runs)
+    bench(args.path_in_fst, args.path_report_md, args.warmup, args.runs, args.compilation_mode)
 
 
 if __name__ == '__main__':

rustfst-python-bench/rustfst_python_bench/bench_all_detailed.py

@@ -5,7 +5,7 @@
 import tempfile
 
 from rustfst_python_bench.algorithms.supported_algorithms import SupportedAlgorithms
-from rustfst_python_bench.constants import RUSTFST_CLI, BENCH_OPENFST_BINS
+from rustfst_python_bench.constants import get_rusftfst_cli_dir, BENCH_OPENFST_BINS
 from rustfst_python_bench.utils import header_report
 
 
@@ -14,6 +14,12 @@ def parse():
         description="Script to bench all CLIs of OpenFST and RustFST"
     )
 
+    parser.add_argument(
+        "compilation_mode",
+        type=str,
+        choices=["debug", "release"]
+    )
+
     parser.add_argument(
         "path_in_fst",
         type=str,
@@ -45,11 +51,13 @@ def parse():
     return args
 
 
-def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs, algo):
+def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs, algo, compilation_mode):
 
     path_out_rustfst = os.path.join(results_dir, f'{algo_name}_rustfst.fst')
 
-    cmd_rustfst = f"{RUSTFST_CLI} {algo.rustfst_subcommand()} {algo.get_cli_args()} {path_in_fst} {path_out_rustfst} " \
+    rustfst_cli = get_rusftfst_cli_dir(compilation_mode)
+
+    cmd_rustfst = f"{rustfst_cli} {algo.rustfst_subcommand()} {algo.get_cli_args()} {path_in_fst} {path_out_rustfst} " \
                   f"--bench --export-markdown {path_report_md} --n_iters {runs} --n_warm_ups {warmup}"
 
     subprocess.check_call([cmd_rustfst], shell=True)
@@ -76,7 +84,7 @@ def bench_algo(algo_name, path_in_fst, results_dir, path_report_md, warmup, runs
     algo.check_correctness(path_out_openfst, path_out_rustfst)
 
 
-def bench(path_in_fst, path_report_md, warmup, runs):
+def bench(compilation_mode: str, path_in_fst: str, path_report_md: str, warmup, runs):
 
     with io.open(path_report_md, mode="w") as report_f:
         report_f.write("# Benchmark OpenFST C++ functions vs RustFST Rust functions\n")
@@ -89,7 +97,7 @@ def bench(path_in_fst, path_report_md, warmup, runs):
                 params = algo_class.get_parameters()
                 report_f.write(f"## {algoname.capitalize()}\n")
                 for param in params:
-                    bench_algo(algoname, path_in_fst, tmpdirname, report_path_temp, warmup, runs, param)
+                    bench_algo(algoname, path_in_fst, tmpdirname, report_path_temp, warmup, runs, param, compilation_mode)
 
                     with io.open(report_path_temp, mode="r") as f:
 
@@ -102,7 +110,13 @@ def bench(path_in_fst, path_report_md, warmup, runs):
 
 def main():
     args = parse()
-    bench(args.path_in_fst, args.path_report_md, args.warmup, args.runs)
+    bench(
+        compilation_mode=args.compilation_mode,
+        path_in_fst=args.path_in_fst,
+        path_report_md=args.path_report_md,
+        warmup=args.warmup,
+        runs=args.runs
+    )
 
 
 if __name__ == '__main__':

rustfst-python-bench/rustfst_python_bench/constants.py

@@ -1,3 +1,6 @@
 OPENFST_BINS = './openfst-1.7.2/bin/'
 BENCH_OPENFST_BINS = './openfst_benchmark/'
-RUSTFST_CLI = './target/release/rustfst-cli'
+
+
+def get_rusftfst_cli_dir(compilation_mode: str) -> str:
+    return f"./target/{compilation_mode}/rustfst-cli"