feat(bench): add benchmarks crate for io-engine operations

Adds nexus creation benchmark via gRPC and directly.
The benchmarks are added on a different crate (io-engine-bench) because otherwise on every change
on the bench code every io-engine crate binaries has to be built which takes time.
The downside is that io-engine binaries release or debug need to be built prior to running the
benches.
(This can easily be changed if we want to)

The benchmark needs to run from the io-engine-bench folder because:
1- we have a runner to run it as root (required by the in-binary benchmark)
2- we have code in the runner to chown the results to $USER and copy them back and forth
from the io-engine-bench/results/criterion location and the target location.

To run the benchmark in release:
RUST_LOG=disable cargo bench -p io-engine-bench

To run the debug:
RUST_LOG=disable $RUST_NIGHTLY_PATH/bin/cargo bench -p io-engine-bench --profile=dev

The criterion results are placed in the cargo target folder. You may direct your browser to this
location to inspect the results.

Cargo.toml

@@ -6,6 +6,7 @@ members = [
 	"jsonrpc",
 	"libnvme-rs",
 	"io-engine",
+    "io-engine-bench",
 	"rpc",
 	"sysfs",
 	"composer",

ci.nix

@@ -52,6 +52,7 @@ mkShell {
     python3
     utillinux
     xfsprogs
+    gnuplot
   ] ++ (if (nospdk) then [ libspdk-dev.buildInputs ] else [ libspdk-dev ])
   ++ pkgs.lib.optional (!norust) channel.stable
   ++ pkgs.lib.optional (!norust) channel.nightly;

composer/src/lib.rs

@@ -119,6 +119,13 @@ impl Binary {
 
         Self::new(&format!("{}/target/debug/{}", srcdir, name), vec![])
     }
+    /// Setup local binary from target
+    pub fn from_target(build: &str, name: &str) -> Self {
+        let path = std::path::PathBuf::from(std::env!("CARGO_MANIFEST_DIR"));
+        let srcdir = path.parent().unwrap().to_string_lossy();
+
+        Self::new(&format!("{}/target/{}/{}", srcdir, build, name), vec![])
+    }
     /// Setup nix shell binary from path and arguments
     pub fn from_nix(name: &str) -> Self {
         Self::new(name, vec![])

io-engine-bench/.cargo/config

@@ -0,0 +1,7 @@
+# we need elevated privileges to run mayastor related tests
+# cargo will ask you nicely to type your password
+[target.x86_64-unknown-linux-gnu]
+runner = ".cargo/runner.sh"
+
+[target.aarch64-unknown-linux-gnu]
+runner = ".cargo/runner.sh"

io-engine-bench/.cargo/runner.sh

@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+
+# Grab the arguments passed to the runner.
+ARGS="${@}"
+
+if [[ $EUID -ne 0 ]]; then
+  MAYBE_SUDO='sudo -E'
+else
+  MAYBE_SUDO=''
+fi
+
+TARGET_CRITERION="$SRCDIR/target/criterion"
+GIT_CRITERION="$SRCDIR/io-engine-bench/results/criterion"
+
+if [ -d "$GIT_CRITERION" ]; then
+    mv "$GIT_CRITERION" "$TARGET_CRITERION"
+fi
+
+# Elevate to sudo so we can set some capabilities via `capsh`, then execute the args with the required capabilities:
+#
+# * Set `cap_setpcap` to be able to set [ambient capabilities](https://lwn.net/Articles/636533/) which can be inherited
+# by children.
+# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice` as they are required by the io-engine.
+${MAYBE_SUDO} capsh \
+  --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice+iep" \
+  --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice \
+  -- -c "${ARGS}"
+
+if [ -d "$TARGET_CRITERION" ]; then
+    ${MAYBE_SUDO} chown -R $USER "$TARGET_CRITERION" &>/dev/null | true
+    mv "$TARGET_CRITERION" "$GIT_CRITERION"
+fi

io-engine-bench/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "io-engine-bench"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dev-dependencies]
+tokio = { version = "1.12.0", features = [ "full" ] }
+chrono = "0.4.19"
+env_logger = "0.9.0"
+futures = "0.3.16"
+once_cell = "1.8.0"
+tonic = "0.5.2"
+tracing = "0.1.26"
+tracing-core = "0.1.19"
+tracing-futures = "0.2.5"
+tracing-subscriber = "0.2.20"
+url = "2.2.2"
+crossbeam = "0.8.1"
+uuid = { version = "0.8.2", features = ["v4"] }
+run_script = "0.8.0"
+rpc = { path = "../rpc" }
+io-engine = { path = "../io-engine" }
+composer = { path = "../composer" }
+spdk-rs = { path = "../spdk-rs" }
+criterion = { version = "0.3.5", features = [ "async_tokio" ] }
+
+[[bench]]
+name = "nexus"
+path = "src/nexus.rs"
+harness = false

io-engine-bench/README.md

@@ -0,0 +1,22 @@
+# Benchmarks
+
+This is where we keep benchmarks to various io-engine operations, example: how long does it take to create a nexus.
+
+## Prerequisites
+It's recommended to run the benchmarks from this folder because:
+1. we have a cargo runner to run it as root (required by the in-binary benchmark)
+
+2. we have code in the cargo runner to chown the results to $USER and copy them back and forth
+from the `io-engine-bench/results/criterion` location and the cargo target location.
+
+## Examples
+### To run the benchmark in release:
+> RUST_LOG=disable cargo bench -p io-engine-bench
+
+### To run the benchmark in debug:
+> RUST_LOG=disable $RUST_NIGHTLY_PATH/bin/cargo bench -p io-engine-bench --profile=dev
+
+## Results
+The criterion results are placed in the cargo target folder. You may direct your browser to this location to inspect the results.
+
+For "persistence", if the path `io-engine-bench/results/` exists, the benchmarks are moved to that location.

io-engine-bench/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    let profile = std::env::var("PROFILE").unwrap();
+    let spdk_rpath =
+        format!("{}/target/{}", std::env::var("SRCDIR").unwrap(), profile);
+    println!("cargo:rustc-link-search=native={}", spdk_rpath);
+    println!("cargo:rustc-link-arg=-Wl,-rpath={}", spdk_rpath);
+}

io-engine-bench/src/common

@@ -0,0 +1 @@
+../../io-engine/tests/common

io-engine-bench/src/lib.rs

@@ -0,0 +1 @@
+

io-engine-bench/src/nexus.rs

@@ -0,0 +1,216 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use io_engine::{
+    bdev::nexus::nexus_create,
+    core::MayastorCliArgs,
+    grpc::v1::nexus::nexus_destroy,
+};
+use rpc::mayastor::{BdevShareRequest, BdevUri, Null};
+use std::sync::Arc;
+
+#[allow(unused)]
+mod common;
+use common::compose::MayastorTest;
+use composer::{Binary, Builder, ComposeTest};
+
+/// Infer the build type from the `OUT_DIR` and `SRCDIR`.
+fn build_type() -> String {
+    let out_dir = env!("OUT_DIR");
+    let src_dir = env!("SRCDIR");
+    let prefix = format!("{}/target/", src_dir);
+    let target = out_dir.replace(&prefix, "");
+    let splits = target.split('/').take(1).collect::<Vec<_>>();
+    let build = splits.first().expect("build type not found");
+    assert!(!build.is_empty());
+    build.to_string()
+}
+
+/// Create a new compose test cluster.
+async fn new_compose() -> Arc<ComposeTest> {
+    let binary = Binary::from_target(&build_type(), "io-engine");
+
+    let builder = Builder::new()
+        .name("cargo-bench")
+        .network("10.1.0.0/16")
+        .add_container_bin("io-engine-1", binary.clone())
+        .add_container_bin("io-engine-2", binary.clone())
+        .add_container_bin("io-engine-3", binary.clone())
+        .add_container_bin("io-engine-4", binary)
+        .with_clean(true)
+        .build()
+        .await
+        .unwrap();
+    Arc::new(builder)
+}
+/// Create a new in-binary environment.
+fn new_environment<'a>() -> Arc<MayastorTest<'a>> {
+    Arc::new(MayastorTest::new(MayastorCliArgs::default()))
+}
+
+/// Get remote nvmf targets to use as nexus children.
+async fn get_children(compose: Arc<ComposeTest>) -> &'static Vec<String> {
+    static STATIC_TARGETS: tokio::sync::OnceCell<Vec<String>> =
+        tokio::sync::OnceCell::const_new();
+
+    STATIC_TARGETS
+        .get_or_init(|| async move {
+            // get the handles if needed, to invoke methods to the containers
+            let mut hdls = compose.grpc_handles().await.unwrap();
+            let mut children = Vec::with_capacity(hdls.len());
+
+            let disk_index = 0;
+            // create and share a bdev on each container
+            for h in &mut hdls {
+                h.bdev.list(Null {}).await.unwrap();
+                h.bdev
+                    .create(BdevUri {
+                        uri: format!("malloc:///disk{}?size_mb=20", disk_index),
+                    })
+                    .await
+                    .unwrap();
+                h.bdev
+                    .share(BdevShareRequest {
+                        name: format!("disk{}", disk_index),
+                        proto: "nvmf".into(),
+                    })
+                    .await
+                    .unwrap();
+
+                // create a nexus with the remote replica as its child
+                let child_uri = format!(
+                    "nvmf://{}:8420/nqn.2019-05.io.openebs:disk{}",
+                    h.endpoint.ip(),
+                    disk_index
+                );
+                children.push(child_uri);
+            }
+            children
+        })
+        .await
+}
+
+/// Created Nexus that is destroyed on drop.
+struct DirectNexus(Arc<MayastorTest<'static>>, String);
+impl Drop for DirectNexus {
+    fn drop(&mut self) {
+        let name = self.1.clone();
+        let io_engine = self.0.clone();
+        std::thread::spawn(|| {
+            tokio::runtime::Runtime::new()
+                .unwrap()
+                .block_on(async move {
+                    io_engine
+                        .spawn(async move {
+                            nexus_destroy(name.as_str()).await.unwrap();
+                        })
+                        .await;
+                });
+        })
+        .join()
+        .unwrap();
+    }
+}
+/// Create a new nexus in-binary and return it as droppable to be destroyed.
+async fn nexus_create_direct(
+    ms_environment: &Arc<MayastorTest<'static>>,
+    compose: &Arc<ComposeTest>,
+    nr_children: usize,
+) -> DirectNexus {
+    let uuid = uuid::Uuid::new_v4();
+    let nexus_name = format!("nexus-{}", uuid);
+    let name = nexus_name.clone();
+    let uuid_str = uuid.to_string();
+
+    let children = get_children(compose.clone())
+        .await
+        .iter()
+        .take(nr_children)
+        .cloned();
+
+    let name = ms_environment
+        .spawn(async move {
+            nexus_create(
+                &name,
+                10 * 1024 * 1024,
+                Some(uuid_str.as_str()),
+                &children.collect::<Vec<_>>(),
+            )
+            .await
+            .unwrap();
+            uuid_str
+        })
+        .await;
+    DirectNexus(ms_environment.clone(), name)
+}
+
+/// Created Grpc Nexus that is destroyed on drop.
+struct GrpcNexus(Arc<ComposeTest>, rpc::mayastor::Nexus);
+impl Drop for GrpcNexus {
+    fn drop(&mut self) {
+        let uuid = self.1.uuid.clone();
+        let compose = self.0.clone();
+        std::thread::spawn(|| {
+            tokio::runtime::Runtime::new()
+                .unwrap()
+                .block_on(async move {
+                    let mut hdls = compose.grpc_handles().await.unwrap();
+                    let nexus_hdl = &mut hdls.last_mut().unwrap();
+                    nexus_hdl
+                        .mayastor
+                        .destroy_nexus(rpc::mayastor::DestroyNexusRequest {
+                            uuid,
+                        })
+                        .await
+                        .unwrap();
+                });
+        })
+        .join()
+        .unwrap()
+    }
+}
+/// Create a new nexus via grpc and return it as droppable to be destroyed.
+async fn nexus_create_grpc(
+    compose: &Arc<ComposeTest>,
+    nr_children: usize,
+) -> GrpcNexus {
+    let children = get_children(compose.clone())
+        .await
+        .iter()
+        .take(nr_children)
+        .cloned();
+    let mut hdls = compose.grpc_handles().await.unwrap();
+
+    let nexus_hdl = &mut hdls.last_mut().unwrap();
+    let nexus = nexus_hdl
+        .mayastor
+        .create_nexus(rpc::mayastor::CreateNexusRequest {
+            uuid: uuid::Uuid::new_v4().to_string(),
+            size: 10 * 1024 * 1024,
+            children: children.collect::<Vec<_>>(),
+        })
+        .await
+        .unwrap();
+    GrpcNexus(compose.clone(), nexus.into_inner())
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let runtime = tokio::runtime::Runtime::new().unwrap();
+    let compose = runtime.block_on(async move { new_compose().await });
+    let ms_environment = new_environment();
+
+    let mut group = c.benchmark_group(format!("{}/nexus/create", build_type()));
+    group
+        // Benchmark nexus create in-binary
+        .bench_function("direct", |b| {
+            b.to_async(&runtime).iter_with_large_drop(|| {
+                nexus_create_direct(&ms_environment, &compose, 3)
+            })
+        })
+        // Benchmark nexus create via gRPC
+        .bench_function("grpc", |b| {
+            b.to_async(&runtime)
+                .iter_with_large_drop(|| nexus_create_grpc(&compose, 3))
+        });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

shell.nix

@@ -45,6 +45,7 @@ mkShell {
     pytest_inputs
     python3
     utillinux
+    gnuplot
   ] ++ (if (nospdk) then [ libspdk-dev.buildInputs ] else [ libspdk-dev ]);
 
   LIBCLANG_PATH = io-engine.LIBCLANG_PATH;