From 4647301dff25782f4ebd1f9afbb9bee8d674df97 Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Thu, 14 Apr 2022 09:32:11 +0100 Subject: [PATCH] feat(bench): add benchmarks crate for io-engine operations Adds nexus creation benchmark via gRPC and directly. The benchmarks are added on a different crate (io-engine-bench) because otherwise on every change on the bench code every io-engine crate binaries has to be built which takes time. The downside is that io-engine binaries release or debug need to be built prior to running the benches. (This can easily be changed if we want to) The benchmark needs to run from the io-engine-bench folder because: 1- we have a runner to run it as root (required by the in-binary benchmark) 2- we have code in the runner to chown the results to $USER and copy them back and forth from the io-engine-bench/results/criterion location and the target location. To run the benchmark in release: RUST_LOG=disable cargo bench -p io-engine-bench To run the debug: RUST_LOG=disable $RUST_NIGHTLY_PATH/bin/cargo bench -p io-engine-bench --profile=dev The criterion results are placed in the cargo target folder. You may direct your browser to this location to inspect the results. --- Cargo.lock | 318 ++++++++++++++++++++++++++++++- Cargo.toml | 1 + ci.nix | 1 + composer/src/lib.rs | 7 + io-engine-bench/.cargo/config | 7 + io-engine-bench/.cargo/runner.sh | 32 ++++ io-engine-bench/Cargo.toml | 32 ++++ io-engine-bench/README.md | 22 +++ io-engine-bench/build.rs | 7 + io-engine-bench/src/common | 1 + io-engine-bench/src/lib.rs | 1 + io-engine-bench/src/nexus.rs | 216 +++++++++++++++++++++ shell.nix | 1 + 13 files changed, 642 insertions(+), 4 deletions(-) create mode 100644 io-engine-bench/.cargo/config create mode 100755 io-engine-bench/.cargo/runner.sh create mode 100644 io-engine-bench/Cargo.toml create mode 100644 io-engine-bench/README.md create mode 100644 io-engine-bench/build.rs create mode 120000 io-engine-bench/src/common create mode 100644 io-engine-bench/src/lib.rs create mode 100644 io-engine-bench/src/nexus.rs diff --git a/Cargo.lock b/Cargo.lock index d08886b6b..5444476c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,12 +192,30 @@ dependencies = [ "serde_with", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "build_const" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byte-unit" version = "4.0.14" @@ -219,6 +237,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", +] + [[package]] name = "cc" version = "1.0.73" @@ -334,6 +361,44 @@ dependencies = [ "build_const", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "futures", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam" version = "0.8.1" @@ -419,6 +484,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.13.1" @@ -775,6 +862,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -813,7 +906,7 @@ checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.1", ] [[package]] @@ -860,7 +953,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.1", "pin-project-lite", "socket2", "tokio", @@ -1001,6 +1094,31 @@ dependencies = [ "version-info", ] +[[package]] +name = "io-engine-bench" +version = "0.1.0" +dependencies = [ + "chrono", + "composer", + "criterion", + "crossbeam", + "env_logger", + "futures", + "io-engine", + "once_cell", + "rpc", + "run_script", + "spdk-rs", + "tokio", + "tonic", + "tracing", + "tracing-core", + "tracing-futures", + "tracing-subscriber", + "url", + "uuid", +] + [[package]] name = "io-uring" version = "0.5.2" @@ -1035,12 +1153,27 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "jsonrpc" version = "1.0.0" @@ -1290,6 +1423,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1407,6 +1546,34 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -1579,6 +1746,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.11" @@ -1665,6 +1856,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -1677,12 +1877,27 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + [[package]] name = "serde" version = "1.0.136" @@ -1692,6 +1907,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.136" @@ -1709,7 +1934,7 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -1721,7 +1946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -2011,6 +2236,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -2403,6 +2638,17 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -2425,6 +2671,70 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2 1.0.36", + "quote 1.0.16", + "syn 1.0.89", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote 1.0.16", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2 1.0.36", + "quote 1.0.16", + "syn 1.0.89", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.2.5" diff --git a/Cargo.toml b/Cargo.toml index f6ddce422..89dc090ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "jsonrpc", "libnvme-rs", "io-engine", + "io-engine-bench", "rpc", "sysfs", "composer", diff --git a/ci.nix b/ci.nix index 3f898604e..0e0651368 100644 --- a/ci.nix +++ b/ci.nix @@ -52,6 +52,7 @@ mkShell { python3 utillinux xfsprogs + gnuplot ] ++ (if (nospdk) then [ libspdk-dev.buildInputs ] else [ libspdk-dev ]) ++ pkgs.lib.optional (!norust) channel.stable ++ pkgs.lib.optional (!norust) channel.nightly; diff --git a/composer/src/lib.rs b/composer/src/lib.rs index ac7f1aa8f..a8edff5db 100644 --- a/composer/src/lib.rs +++ b/composer/src/lib.rs @@ -119,6 +119,13 @@ impl Binary { Self::new(&format!("{}/target/debug/{}", srcdir, name), vec![]) } + /// Setup local binary from target + pub fn from_target(build: &str, name: &str) -> Self { + let path = std::path::PathBuf::from(std::env!("CARGO_MANIFEST_DIR")); + let srcdir = path.parent().unwrap().to_string_lossy(); + + Self::new(&format!("{}/target/{}/{}", srcdir, build, name), vec![]) + } /// Setup nix shell binary from path and arguments pub fn from_nix(name: &str) -> Self { Self::new(name, vec![]) diff --git a/io-engine-bench/.cargo/config b/io-engine-bench/.cargo/config new file mode 100644 index 000000000..d56081d83 --- /dev/null +++ b/io-engine-bench/.cargo/config @@ -0,0 +1,7 @@ +# we need elevated privileges to run mayastor related tests +# cargo will ask you nicely to type your password +[target.x86_64-unknown-linux-gnu] +runner = ".cargo/runner.sh" + +[target.aarch64-unknown-linux-gnu] +runner = ".cargo/runner.sh" diff --git a/io-engine-bench/.cargo/runner.sh b/io-engine-bench/.cargo/runner.sh new file mode 100755 index 000000000..c8d5639e6 --- /dev/null +++ b/io-engine-bench/.cargo/runner.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +# Grab the arguments passed to the runner. +ARGS="${@}" + +if [[ $EUID -ne 0 ]]; then + MAYBE_SUDO='sudo -E' +else + MAYBE_SUDO='' +fi + +TARGET_CRITERION="$SRCDIR/target/criterion" +GIT_CRITERION="$SRCDIR/io-engine-bench/results/criterion" + +if [ -d "$GIT_CRITERION" ]; then + mv "$GIT_CRITERION" "$TARGET_CRITERION" +fi + +# Elevate to sudo so we can set some capabilities via `capsh`, then execute the args with the required capabilities: +# +# * Set `cap_setpcap` to be able to set [ambient capabilities](https://lwn.net/Articles/636533/) which can be inherited +# by children. +# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice` as they are required by the io-engine. +${MAYBE_SUDO} capsh \ + --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice+iep" \ + --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice \ + -- -c "${ARGS}" + +if [ -d "$TARGET_CRITERION" ]; then + ${MAYBE_SUDO} chown -R $USER "$TARGET_CRITERION" &>/dev/null | true + mv "$TARGET_CRITERION" "$GIT_CRITERION" +fi diff --git a/io-engine-bench/Cargo.toml b/io-engine-bench/Cargo.toml new file mode 100644 index 000000000..7f424be0a --- /dev/null +++ b/io-engine-bench/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "io-engine-bench" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +tokio = { version = "1.12.0", features = [ "full" ] } +chrono = "0.4.19" +env_logger = "0.9.0" +futures = "0.3.16" +once_cell = "1.8.0" +tonic = "0.5.2" +tracing = "0.1.26" +tracing-core = "0.1.19" +tracing-futures = "0.2.5" +tracing-subscriber = "0.2.20" +url = "2.2.2" +crossbeam = "0.8.1" +uuid = { version = "0.8.2", features = ["v4"] } +run_script = "0.8.0" +rpc = { path = "../rpc" } +io-engine = { path = "../io-engine" } +composer = { path = "../composer" } +spdk-rs = { path = "../spdk-rs" } +criterion = { version = "0.3.5", features = [ "async_tokio" ] } + +[[bench]] +name = "nexus" +path = "src/nexus.rs" +harness = false diff --git a/io-engine-bench/README.md b/io-engine-bench/README.md new file mode 100644 index 000000000..fe1f34db9 --- /dev/null +++ b/io-engine-bench/README.md @@ -0,0 +1,22 @@ +# Benchmarks + +This is where we keep benchmarks to various io-engine operations, example: how long does it take to create a nexus. + +## Prerequisites +It's recommended to run the benchmarks from this folder because: +1. we have a cargo runner to run it as root (required by the in-binary benchmark) + +2. we have code in the cargo runner to chown the results to $USER and copy them back and forth +from the `io-engine-bench/results/criterion` location and the cargo target location. + +## Examples +### To run the benchmark in release: +> RUST_LOG=disable cargo bench -p io-engine-bench + +### To run the benchmark in debug: +> RUST_LOG=disable $RUST_NIGHTLY_PATH/bin/cargo bench -p io-engine-bench --profile=dev + +## Results +The criterion results are placed in the cargo target folder. You may direct your browser to this location to inspect the results. + +For "persistence", if the path `io-engine-bench/results/` exists, the benchmarks are moved to that location. diff --git a/io-engine-bench/build.rs b/io-engine-bench/build.rs new file mode 100644 index 000000000..ba1b5a9a2 --- /dev/null +++ b/io-engine-bench/build.rs @@ -0,0 +1,7 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap(); + let spdk_rpath = + format!("{}/target/{}", std::env::var("SRCDIR").unwrap(), profile); + println!("cargo:rustc-link-search=native={}", spdk_rpath); + println!("cargo:rustc-link-arg=-Wl,-rpath={}", spdk_rpath); +} diff --git a/io-engine-bench/src/common b/io-engine-bench/src/common new file mode 120000 index 000000000..d5165507a --- /dev/null +++ b/io-engine-bench/src/common @@ -0,0 +1 @@ +../../io-engine/tests/common \ No newline at end of file diff --git a/io-engine-bench/src/lib.rs b/io-engine-bench/src/lib.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/io-engine-bench/src/lib.rs @@ -0,0 +1 @@ + diff --git a/io-engine-bench/src/nexus.rs b/io-engine-bench/src/nexus.rs new file mode 100644 index 000000000..5f1380e6c --- /dev/null +++ b/io-engine-bench/src/nexus.rs @@ -0,0 +1,216 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use io_engine::{ + bdev::nexus::nexus_create, + core::MayastorCliArgs, + grpc::v1::nexus::nexus_destroy, +}; +use rpc::mayastor::{BdevShareRequest, BdevUri, Null}; +use std::sync::Arc; + +#[allow(unused)] +mod common; +use common::compose::MayastorTest; +use composer::{Binary, Builder, ComposeTest}; + +/// Infer the build type from the `OUT_DIR` and `SRCDIR`. +fn build_type() -> String { + let out_dir = env!("OUT_DIR"); + let src_dir = env!("SRCDIR"); + let prefix = format!("{}/target/", src_dir); + let target = out_dir.replace(&prefix, ""); + let splits = target.split('/').take(1).collect::>(); + let build = splits.first().expect("build type not found"); + assert!(!build.is_empty()); + build.to_string() +} + +/// Create a new compose test cluster. +async fn new_compose() -> Arc { + let binary = Binary::from_target(&build_type(), "io-engine"); + + let builder = Builder::new() + .name("cargo-bench") + .network("10.1.0.0/16") + .add_container_bin("io-engine-1", binary.clone()) + .add_container_bin("io-engine-2", binary.clone()) + .add_container_bin("io-engine-3", binary.clone()) + .add_container_bin("io-engine-4", binary) + .with_clean(true) + .build() + .await + .unwrap(); + Arc::new(builder) +} +/// Create a new in-binary environment. +fn new_environment<'a>() -> Arc> { + Arc::new(MayastorTest::new(MayastorCliArgs::default())) +} + +/// Get remote nvmf targets to use as nexus children. +async fn get_children(compose: Arc) -> &'static Vec { + static STATIC_TARGETS: tokio::sync::OnceCell> = + tokio::sync::OnceCell::const_new(); + + STATIC_TARGETS + .get_or_init(|| async move { + // get the handles if needed, to invoke methods to the containers + let mut hdls = compose.grpc_handles().await.unwrap(); + let mut children = Vec::with_capacity(hdls.len()); + + let disk_index = 0; + // create and share a bdev on each container + for h in &mut hdls { + h.bdev.list(Null {}).await.unwrap(); + h.bdev + .create(BdevUri { + uri: format!("malloc:///disk{}?size_mb=20", disk_index), + }) + .await + .unwrap(); + h.bdev + .share(BdevShareRequest { + name: format!("disk{}", disk_index), + proto: "nvmf".into(), + }) + .await + .unwrap(); + + // create a nexus with the remote replica as its child + let child_uri = format!( + "nvmf://{}:8420/nqn.2019-05.io.openebs:disk{}", + h.endpoint.ip(), + disk_index + ); + children.push(child_uri); + } + children + }) + .await +} + +/// Created Nexus that is destroyed on drop. +struct DirectNexus(Arc>, String); +impl Drop for DirectNexus { + fn drop(&mut self) { + let name = self.1.clone(); + let io_engine = self.0.clone(); + std::thread::spawn(|| { + tokio::runtime::Runtime::new() + .unwrap() + .block_on(async move { + io_engine + .spawn(async move { + nexus_destroy(name.as_str()).await.unwrap(); + }) + .await; + }); + }) + .join() + .unwrap(); + } +} +/// Create a new nexus in-binary and return it as droppable to be destroyed. +async fn nexus_create_direct( + ms_environment: &Arc>, + compose: &Arc, + nr_children: usize, +) -> DirectNexus { + let uuid = uuid::Uuid::new_v4(); + let nexus_name = format!("nexus-{}", uuid); + let name = nexus_name.clone(); + let uuid_str = uuid.to_string(); + + let children = get_children(compose.clone()) + .await + .iter() + .take(nr_children) + .cloned(); + + let name = ms_environment + .spawn(async move { + nexus_create( + &name, + 10 * 1024 * 1024, + Some(uuid_str.as_str()), + &children.collect::>(), + ) + .await + .unwrap(); + uuid_str + }) + .await; + DirectNexus(ms_environment.clone(), name) +} + +/// Created Grpc Nexus that is destroyed on drop. +struct GrpcNexus(Arc, rpc::mayastor::Nexus); +impl Drop for GrpcNexus { + fn drop(&mut self) { + let uuid = self.1.uuid.clone(); + let compose = self.0.clone(); + std::thread::spawn(|| { + tokio::runtime::Runtime::new() + .unwrap() + .block_on(async move { + let mut hdls = compose.grpc_handles().await.unwrap(); + let nexus_hdl = &mut hdls.last_mut().unwrap(); + nexus_hdl + .mayastor + .destroy_nexus(rpc::mayastor::DestroyNexusRequest { + uuid, + }) + .await + .unwrap(); + }); + }) + .join() + .unwrap() + } +} +/// Create a new nexus via grpc and return it as droppable to be destroyed. +async fn nexus_create_grpc( + compose: &Arc, + nr_children: usize, +) -> GrpcNexus { + let children = get_children(compose.clone()) + .await + .iter() + .take(nr_children) + .cloned(); + let mut hdls = compose.grpc_handles().await.unwrap(); + + let nexus_hdl = &mut hdls.last_mut().unwrap(); + let nexus = nexus_hdl + .mayastor + .create_nexus(rpc::mayastor::CreateNexusRequest { + uuid: uuid::Uuid::new_v4().to_string(), + size: 10 * 1024 * 1024, + children: children.collect::>(), + }) + .await + .unwrap(); + GrpcNexus(compose.clone(), nexus.into_inner()) +} + +fn criterion_benchmark(c: &mut Criterion) { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let compose = runtime.block_on(async move { new_compose().await }); + let ms_environment = new_environment(); + + let mut group = c.benchmark_group(format!("{}/nexus/create", build_type())); + group + // Benchmark nexus create in-binary + .bench_function("direct", |b| { + b.to_async(&runtime).iter_with_large_drop(|| { + nexus_create_direct(&ms_environment, &compose, 3) + }) + }) + // Benchmark nexus create via gRPC + .bench_function("grpc", |b| { + b.to_async(&runtime) + .iter_with_large_drop(|| nexus_create_grpc(&compose, 3)) + }); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/shell.nix b/shell.nix index bc706d8a6..3c1ea3084 100644 --- a/shell.nix +++ b/shell.nix @@ -45,6 +45,7 @@ mkShell { pytest_inputs python3 utillinux + gnuplot ] ++ (if (nospdk) then [ libspdk-dev.buildInputs ] else [ libspdk-dev ]); LIBCLANG_PATH = io-engine.LIBCLANG_PATH;