From eb094d64bcae0464035fd14b30b0c5ea2661bbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Sun, 13 Oct 2019 14:29:37 -0700 Subject: [PATCH] Add simple fuzzing Add infrastructure to automatically run fuzzers in CI, and implement a simple fuzzing test based on triggering all (most) public APIs in a randimized way. As far as I was able to try it catches the previous unsoundness issues in a matter of seconds. This can be tried by changing the `path = "../"` dependency to `version = "=0.6.3"` etc. and running the fuzzer manually. (Note: You'll need to tweak the `Cargo.lock` to allow downloading the yanked versions). Related to #124 --- .gitignore | 2 +- .travis.yml | 34 ++++- fuzz/Cargo.lock | 192 +++++++++++++++++++++++ fuzz/Cargo.toml | 26 ++++ fuzz/README.md | 12 ++ fuzz/fuzz_targets/smallvec_ops.rs | 243 ++++++++++++++++++++++++++++++ fuzz/travis-fuzz.sh | 19 +++ 7 files changed, 521 insertions(+), 7 deletions(-) create mode 100644 fuzz/Cargo.lock create mode 100644 fuzz/Cargo.toml create mode 100644 fuzz/README.md create mode 100644 fuzz/fuzz_targets/smallvec_ops.rs create mode 100755 fuzz/travis-fuzz.sh diff --git a/.gitignore b/.gitignore index a9d37c5..665fc20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ target -Cargo.lock +./Cargo.lock diff --git a/.travis.yml b/.travis.yml index 2e54a81..4454c72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,24 @@ language: rust -rust: - - 1.36.0 - - nightly - - beta - - stable +addons: + apt: + update: true + packages: + - binutils-dev + - libunwind8-dev + - libcurl4-openssl-dev + - libelf-dev + - libdw-dev + - cmake + - gcc + - libiberty-dev +matrix: + include: + - rust: 1.36.0 + - rust: nightly + - rust: beta + env: DO_FUZZ=true + - rust: stable + env: DO_FUZZ=true script: | cargo build --verbose && cargo test --verbose && @@ -12,4 +27,11 @@ script: | ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --features union) && ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --all-features) && ([ $TRAVIS_RUST_VERSION != nightly ] || cargo bench --verbose bench) && - ([ $TRAVIS_RUST_VERSION != nightly ] || bash ./scripts/run_miri.sh) + ([ $TRAVIS_RUST_VERSION != nightly ] || bash ./scripts/run_miri.sh) && + if [ "$DO_FUZZ" = true ] + then + ( + cd fuzz + ./travis-fuzz.sh + ) + fi diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 0000000..719e1f7 --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,192 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "afl" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arbitrary" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "atty" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "honggfuzz" +version = "0.5.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.6.10" + +[[package]] +name = "smallvec-fuzz" +version = "0.1.0" +dependencies = [ + "afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "honggfuzz 0.5.45 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-width" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "xdg" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2b15f7ed343b7446090d0de7789a640c22b163fb200d81cef958fcb9856692" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" +"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" +"checksum bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a606a02debe2813760609f57a64a2ffd27d9fdf5b2f133eaca0b248dd92cdd2" +"checksum cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "4fc9a35e1f4290eb9e5fc54ba6cf40671ed2a2514c3eeb2b2a908dda2ea5a1be" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum honggfuzz 0.5.45 (registry+https://github.com/rust-lang/crates.io-index)" = "24c27b4aa3049d6d10d8e33d52c9d03ca9aec18f8a449b246f8c4a5b0c10fb34" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" +"checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..cb94d21 --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "smallvec-fuzz" +version = "0.1.0" +authors = ["Dawid Ciężarkiewicz "] +edition = "2018" +publish = false + +[package.metadata] +cargo-fuzz = true + +[features] +afl_fuzz = ["afl"] +honggfuzz_fuzz = ["honggfuzz"] + + +[dependencies] +honggfuzz = { version = "0.5.45", optional = true } +afl = { version = "0.4", optional = true } +smallvec = { path = ".." } + +[workspace] +members = ["."] + +[[bin]] +name = "smallvec_ops" +path = "fuzz_targets/smallvec_ops.rs" diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 0000000..3230570 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,12 @@ +# Fuzzer for smallvec + +Based on fuzzing in [rust-bitcoin](https://github.com/rust-bitcoin/rust-bitcoin/tree/c8ac25219a09bf9d017f1b05abe3e746e2136f73/fuzz) + +## Running manually with afl + +``` +cargo afl build --release --bin smallvec_ops --features afl && cargo afl fuzz -i in -o out target/release/smallvec_ops +``` + +# Useful links: +* https://rust-fuzz.github.io/book/afl.html diff --git a/fuzz/fuzz_targets/smallvec_ops.rs b/fuzz/fuzz_targets/smallvec_ops.rs new file mode 100644 index 0000000..b5c932e --- /dev/null +++ b/fuzz/fuzz_targets/smallvec_ops.rs @@ -0,0 +1,243 @@ +//! Simple fuzzer testing all available `SmallVec` operations +use smallvec::SmallVec; + +// There's no point growing too much, so try not to grow +// over this size. +const CAP_GROWTH: usize = 256; + +macro_rules! next_usize { + ($b:ident) => { + $b.next().unwrap_or(0) as usize + }; +} + +macro_rules! next_u8 { + ($b:ident) => { + $b.next().unwrap_or(0) + }; +} + +fn black_box_iter(i: impl Iterator) { + // print to work as a black_box + print!("{}", i.fold(0u8, |acc, e| acc.wrapping_add(e))); +} + +fn black_box_slice(s: &[u8]) { + black_box_iter(s.iter().copied()) +} + +fn black_box_mut_slice(s: &mut [u8]) { + s.iter_mut().map(|e| *e = e.wrapping_add(1)).count(); + black_box_iter((s as &[u8]).iter().copied()) +} + +fn do_test>(data: &[u8]) -> SmallVec { + let mut v = SmallVec::::new(); + + let mut bytes = data.iter().copied(); + + while let Some(op) = bytes.next() { + match op % 27 { + 0 => { + v = SmallVec::new(); + } + 1 => { + v = SmallVec::with_capacity(next_usize!(bytes)); + } + 2 => { + v = SmallVec::from_vec(v.to_vec()); + } + 3 => { + black_box_iter(v.drain(..)); + } + 4 => { + if v.len() < CAP_GROWTH { + v.push(next_u8!(bytes)) + } + } + 5 => { + v.pop(); + } + 6 => v.grow(next_usize!(bytes) + v.len()), + 7 => { + if v.len() < CAP_GROWTH { + v.reserve(next_usize!(bytes)) + } + } + 8 => { + if v.len() < CAP_GROWTH { + v.reserve_exact(next_usize!(bytes)) + } + } + 9 => v.shrink_to_fit(), + 10 => v.truncate(next_usize!(bytes)), + 11 => black_box_slice(v.as_slice()), + 12 => black_box_mut_slice(v.as_mut_slice()), + 13 => { + if !v.is_empty() { + v.swap_remove(next_usize!(bytes) % v.len()); + } + } + 14 => { + v.clear(); + } + 15 => { + if !v.is_empty() { + v.remove(next_usize!(bytes) % v.len()); + } + } + 16 => { + let insert_pos = next_usize!(bytes) % (v.len() + 1); + v.insert(insert_pos, next_u8!(bytes)); + } + 17 => { + let insert_pos = next_usize!(bytes) % (v.len() + 1); + let how_many = next_usize!(bytes); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + v.insert_many(insert_pos, (0..how_many).map(|_| bytes.next().unwrap())); + })); + + if result.is_err() { + assert!(bytes.next().is_none()); + } + } + 18 => { + v = SmallVec::from_vec(v.into_vec()); + } + + 19 => { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + v.retain(|e| { + let alt_e = bytes.next().unwrap(); + let retain = *e >= alt_e; + *e = e.wrapping_add(alt_e); + retain + }); + })); + + if result.is_err() { + assert!(bytes.next().is_none()); + } + } + 20 => { + v.dedup(); + } + + 21 => { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + v.dedup_by(|a, b| { + let substitute = bytes.next().unwrap(); + let dedup = a == b; + *a = a.wrapping_add(substitute); + *b = b.wrapping_add(substitute); + dedup + }); + })); + + if result.is_err() { + assert!(bytes.next().is_none()); + } + } + 22 => { + v = SmallVec::from_slice(data); + } + + 23 => { + if v.len() < CAP_GROWTH { + v.extend_from_slice(data) + } + } + + 24 => { + if v.len() < CAP_GROWTH { + let insert_pos = next_usize!(bytes) % (v.len() + 1); + v.insert_from_slice(insert_pos, data); + } + } + + 25 => { + if v.len() < CAP_GROWTH { + v.resize(next_usize!(bytes), next_u8!(bytes)); + } + } + 26 => { + v = SmallVec::from_elem(next_u8!(bytes), next_usize!(bytes)); + } + _ => panic!("booo"), + } + } + v +} + +fn do_test_all(data: &[u8]) { + do_test::<[u8; 0]>(data); + do_test::<[u8; 1]>(data); + do_test::<[u8; 2]>(data); + do_test::<[u8; 7]>(data); + do_test::<[u8; 8]>(data); +} + +#[cfg(feature = "afl")] +#[macro_use] +extern crate afl; +#[cfg(feature = "afl")] +fn main() { + fuzz!(|data| { + // Remove the panic hook so we can actually catch panic + // See https://github.com/rust-fuzz/afl.rs/issues/150 + std::panic::set_hook(Box::new(|_| {})); + do_test_all(data); + }); +} + +#[cfg(feature = "honggfuzz")] +#[macro_use] +extern crate honggfuzz; +#[cfg(feature = "honggfuzz")] +fn main() { + loop { + fuzz!(|data| { + // Remove the panic hook so we can actually catch panic + // See https://github.com/rust-fuzz/afl.rs/issues/150 + std::panic::set_hook(Box::new(|_| {})); + do_test_all(data); + }); + } +} + +#[cfg(test)] +mod tests { + fn extend_vec_from_hex(hex: &str, out: &mut Vec) { + let mut b = 0; + for (idx, c) in hex.as_bytes().iter().enumerate() { + b <<= 4; + match *c { + b'A'..=b'F' => b |= c - b'A' + 10, + b'a'..=b'f' => b |= c - b'a' + 10, + b'0'..=b'9' => b |= c - b'0', + b'\n' => {} + b' ' => {} + _ => panic!("Bad hex"), + } + if (idx & 1) == 1 { + out.push(b); + b = 0; + } + } + } + + #[test] + fn duplicate_crash() { + let mut a = Vec::new(); + // paste the output of `xxd -p ` here and run `cargo test` + extend_vec_from_hex( + r#" + 646e21f9f910f90200f9d9f9c7030000def9000010646e2af9f910f90264 + 6e21f9f910f90200f9d9f9c7030000def90000106400f9f9d9f9c7030000 + def90000106400f9d9f9e7f1000000d9f9e7f1000000f9 + "#, + &mut a, + ); + super::do_test_all(&a); + } +} diff --git a/fuzz/travis-fuzz.sh b/fuzz/travis-fuzz.sh new file mode 100755 index 0000000..731e52f --- /dev/null +++ b/fuzz/travis-fuzz.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e +cargo install --force honggfuzz +for TARGET in fuzz_targets/*; do + FILENAME=$(basename $TARGET) + FILE="${FILENAME%.*}" + if [ -d hfuzz_input/$FILE ]; then + HFUZZ_INPUT_ARGS="-f hfuzz_input/$FILE/input" + fi + HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE + + if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then + cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT + for CASE in hfuzz_workspace/$FILE/SIG*; do + cat $CASE | xxd -p + done + exit 1 + fi +done