diff --git a/.circleci/config.yml b/.circleci/config.yml index c68577de0..70e55dcea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,6 +77,66 @@ jobs: - run: make clean && make test CC=clang-8 WITH_TSAN=1 - run: make clean && make test CC=clang-8 WITH_UBSAN=1 + benchmark: + docker: + - image: cossacklabs/build:ubuntu-bionic + environment: + WITH_FATAL_WARNINGS: yes + steps: + - run: + name: Install native dependencies + command: | + sudo apt update + sudo apt install --yes \ + gnuplot zip + - restore_cache: + keys: + - rust + - run: + name: Install Rust toolchain (stable) + command: | + # Instructions from https://rustup.rs + curl https://sh.rustup.rs -sSf | sh -s -- -y + cat ~/.cargo/env >> $BASH_ENV + source ~/.cargo/env + cargo --version + rustc --version + - checkout + - run: + name: Pull BoringSSL submodule + command: | + git reset --hard HEAD + git submodule sync + git submodule update --init + - run: + name: Themis Core - install + command: | + make + sudo make install + - run: + name: Themis Core - prepare benchmarks + command: | + cd benches/themis + cargo bench --no-run + # TODO: if building a pull request, compare base with updates + - run: + name: Themis Core - run benchmarks - Secure Cell (master key) + command: | + cd benches/themis + cargo bench -- 'Secure Cell .* master key/4 KB' + - run: + name: Pack benchmark report + command: | + cd benches/themis/target + zip -r ../report.zip criterion + - store_artifacts: + path: benches/themis/report.zip + - save_cache: + key: rust + paths: + - ~/.cargo + - ~/.rustup + x86_64: docker: - image: cossacklabs/android-build:2019.01 @@ -466,6 +526,7 @@ workflows: tests: jobs: - analyze + - benchmark - android - x86_64 - jsthemis @@ -486,6 +547,7 @@ workflows: - stable jobs: - analyze + - benchmark - android - x86_64 - jsthemis diff --git a/CHANGELOG.md b/CHANGELOG.md index aabfb14a2..f06c7486e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,10 @@ _Code:_ - New class `SymmetricKey` can be used to generate symmetric keys for Secure Cell ([#561](https://github.com/cossacklabs/themis/pull/561)). +_Infrastructure:_ + +- Automated benchmarking harness is now tracking Themis performance. See [`benches`](https://github.com/cossacklabs/themis/tree/master/benches/) ([#580](https://github.com/cossacklabs/themis/pull/580)). + ## [0.12.0](https://github.com/cossacklabs/themis/releases/tag/0.12.0), September 27th 2019 **TL;DR:** diff --git a/Cargo.toml b/Cargo.toml index 58a32a6aa..c6cfcdbeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "benches/themis", "src/wrappers/themis/rust", "src/wrappers/themis/rust/libthemis-sys", "src/wrappers/themis/rust/libthemis-src", diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 000000000..db2755490 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,11 @@ +# ⏱ Themis benchmarks + +Here you can find benchmarks used to measure overhead and performance of Themis and its wrappers. + +## πŸ‘Ύ Themis Core + +These are the main benchmarks for Themis. +They establish a baseline for perfomance: +high-level language wrappers generally add overhead to Core. + +See [`themis`](themis) directory for details. diff --git a/benches/themis/.gitignore b/benches/themis/.gitignore new file mode 100644 index 000000000..1477eba30 --- /dev/null +++ b/benches/themis/.gitignore @@ -0,0 +1,2 @@ +# Criterion output +target/ diff --git a/benches/themis/Cargo.toml b/benches/themis/Cargo.toml new file mode 100644 index 000000000..9ecf5f446 --- /dev/null +++ b/benches/themis/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "themis-core-bench" +version = "0.0.0" +edition = "2018" +publish = false + +[dependencies] +themis = { version = "0.12", path = "../../src/wrappers/themis/rust" } +libthemis-sys = { version = "0.12", path = "../../src/wrappers/themis/rust/libthemis-sys" } + +[dev-dependencies] +criterion = "0.3" + +[[bench]] +name = "secure_cell_seal_master_key" +harness = false diff --git a/benches/themis/README.md b/benches/themis/README.md new file mode 100644 index 000000000..3b72fb18a --- /dev/null +++ b/benches/themis/README.md @@ -0,0 +1,134 @@ +# ⏱ Themis Core benchmarks + +## Quickstart + +Themis Core benchmarks are written using [**Criterion.rs**](https://bheisler.github.io/criterion.rs/book/criterion_rs.html) statistical benchmarking tool. + +You will need Rust toolchain installed to run benchmarks. +[Visit rustup.rs](https://rustup.rs/) to install Rust. + + 1. πŸ“¦ **Install Themis Core** + + Benchmarks use Themis library from the system by default. + + + Normally, this should be enough to install Themis: + + ```bash + make install + ``` + + If it doesn’t work (or this is your first time building Themis) + then you might need to [review the documentation](https://docs.cossacklabs.com/pages/documentation-themis/#building-and-installing). + + If it still doesn’t work, please [file an issue](https://github.com/cossacklabs/themis/issues/new?labels=bug,installation,core&template=bug_report.md&title=). + + 2. βš™οΈ **Change directory from repository root** + + ```bash + cd benches/themis + ``` + + It’s not required but you would have to type less. + + 3. ⏳ **Build dependencies** + + ```bash + cargo bench --no-run + ``` + + Criterion.rs has quite a few dependencies so be patient, + you need to do this only once. + + 4. πŸš€ **Run some benchmarks** + + ```bash + cargo bench -- "Secure Cell .* Seal, master key/64 KB" + ``` + + [See FAQ](#faq) for more information on how and what you can run. + + 5. πŸ“Š **Analyze result report** + + ```bash + open target/criterion/report/index.html + ``` + + Done! πŸŽ‰ + +## Coverage + +### Secure Cell + +| | Master keys | Passphrases | +| ----------------- | ------------- | ------------- | +| Seal | βœ… complete | πŸ›  WIP | +| Token Protect | πŸ’­ soon | πŸ›  WIP | +| Context Imprint | πŸ’­ soon | βž– N/A | + +### Secure Message + +| | ECDSA | RSA | +| ----------------- | ------------- | ------------- | +| Encrypt / Decrypt | πŸ’­ soon | πŸ’­ soon | +| Sign / Verify | πŸ’­ soon | πŸ’­ soon | + +### Secure Session + +πŸ’­ soon + +### Secure Comparator + +πŸ’­ soon + + + +## FAQ + +First of all, it’s a good idea to familiarize yourself with +[Criterion.rs User Guide](https://bheisler.github.io/criterion.rs/book/criterion_rs.html), +especially sections on +[command-line options](https://bheisler.github.io/criterion.rs/book/user_guide/command_line_options.html), +[output format](https://bheisler.github.io/criterion.rs/book/user_guide/command_line_output.html), +and [interpreting results](https://bheisler.github.io/criterion.rs/book/analysis.html). + +**Q:** What benchmarks are available? + +```bash +cargo bench -- --list +``` + +**Q:** How do I run one of them? + +```bash +cargo bench -- 'one of them' # filter by regular expression +``` + +**Q:** How do I see if my optimizations have an effect? + +```bash +git checkout feature +cargo bench -- --save-baseline feature-unoptimized + +git checkout optimizations +# Work on performance +# ... + +# Compare against the baseline version +cargo bench -- --baseline feature-unoptimized +``` + +Don’t forget to _reinstall_ Themis Core library every time you make a change in it and want to measure it. + + +**Q:** Benchmarking takes ages, what can I do? + +```bash +cargo bench -- --sample-size 20 # cannot be lower than 10 +``` diff --git a/benches/themis/benches/secure_cell_seal_master_key.rs b/benches/themis/benches/secure_cell_seal_master_key.rs new file mode 100644 index 000000000..2e6a4df90 --- /dev/null +++ b/benches/themis/benches/secure_cell_seal_master_key.rs @@ -0,0 +1,158 @@ +// Copyright 2020 Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; + +use libthemis_sys::{ + themis_secure_cell_decrypt_seal, themis_secure_cell_encrypt_seal, THEMIS_BUFFER_TOO_SMALL, + THEMIS_SUCCESS, +}; +use themis::{keys::SymmetricKey, secure_cell::SecureCell}; + +const CONTEXT: &[u8] = b"Themis Core benchmark"; +const MIN_AUTH_TOKEN: usize = 128; + +const KB: usize = 1024; +const MB: usize = 1024 * KB; +#[allow(clippy::identity_op)] +const MESSAGE_SIZES: &[usize] = &[ + 16, // UUID + 32, // SymmetricKey + 64, // cache line (and close to EcdsaPrivateKey) + 256, // RsaPrivateKey + 4 * KB, // memory page + 16 * KB, + 64 * KB, + 1 * MB, // L2 cache + 2 * MB, + 4 * MB, +]; + +pub fn encryption(c: &mut Criterion) { + let master_key = SymmetricKey::new(); + + let mut group = c.benchmark_group("Secure Cell encryption - Seal, master key"); + for message_size in MESSAGE_SIZES { + group.throughput(Throughput::Bytes(*message_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(pretty(*message_size)), + message_size, + |b, &size| { + let message = vec![0; size]; + + let mut encrypted = vec![0; size + MIN_AUTH_TOKEN]; + b.iter(|| { + let mut encrypted_size = 0; + let res = unsafe { + themis_secure_cell_encrypt_seal( + master_key.as_ref().as_ptr(), + master_key.as_ref().len(), + CONTEXT.as_ptr(), + CONTEXT.len(), + message.as_ptr(), + message.len(), + std::ptr::null_mut(), + &mut encrypted_size, + ) + }; + assert_eq!(res, THEMIS_BUFFER_TOO_SMALL as i32); + assert!(encrypted_size <= encrypted.len()); + + let res = unsafe { + themis_secure_cell_encrypt_seal( + master_key.as_ref().as_ptr(), + master_key.as_ref().len(), + CONTEXT.as_ptr(), + CONTEXT.len(), + message.as_ptr(), + message.len(), + encrypted.as_mut_ptr(), + &mut encrypted_size, + ) + }; + assert_eq!(res, THEMIS_SUCCESS as i32); + }); + }, + ); + } + group.finish(); +} + +pub fn decryption(c: &mut Criterion) { + let master_key = SymmetricKey::new(); + + let mut group = c.benchmark_group("Secure Cell decryption - Seal, master key"); + for message_size in MESSAGE_SIZES { + group.throughput(Throughput::Bytes(*message_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(pretty(*message_size)), + message_size, + |b, &size| { + let message = vec![0; size]; + let encrypted = SecureCell::with_key(&master_key) + .expect("invalid key") + .seal() + .encrypt_with_context(message, CONTEXT) + .expect("failed encryption"); + + let mut decrypted = vec![0; size]; + b.iter(|| { + let mut decrypted_size = 0; + let res = unsafe { + themis_secure_cell_decrypt_seal( + master_key.as_ref().as_ptr(), + master_key.as_ref().len(), + CONTEXT.as_ptr(), + CONTEXT.len(), + encrypted.as_ptr(), + encrypted.len(), + std::ptr::null_mut(), + &mut decrypted_size, + ) + }; + assert_eq!(res, THEMIS_BUFFER_TOO_SMALL as i32); + assert!(decrypted_size <= decrypted.len()); + + let res = unsafe { + themis_secure_cell_decrypt_seal( + master_key.as_ref().as_ptr(), + master_key.as_ref().len(), + CONTEXT.as_ptr(), + CONTEXT.len(), + encrypted.as_ptr(), + encrypted.len(), + decrypted.as_mut_ptr(), + &mut decrypted_size, + ) + }; + assert_eq!(res, THEMIS_SUCCESS as i32); + }); + }, + ); + } + group.finish(); +} + +fn pretty(size: usize) -> String { + if size >= MB { + format!("{} MB", size / MB) + } else if size >= KB { + format!("{} KB", size / KB) + } else { + format!("{}", size) + } +} + +criterion_group!(benches, encryption, decryption); +criterion_main!(benches);