From db7e3235f0538aa432551c57dd2a67814d3b26a1 Mon Sep 17 00:00:00 2001 From: Samuel Moelius Date: Sun, 25 Dec 2022 13:38:07 +0000 Subject: [PATCH] Preliminary support for `libfuzzer` --- .github/workflows/ci.yml | 53 ++- README.md | 90 ++--- cargo-test-fuzz/Cargo.toml | 8 +- cargo-test-fuzz/patches/solana.patch | 74 ++-- .../src/fuzzer/impls/aflplusplus.rs | 192 ++++++++++ cargo-test-fuzz/src/fuzzer/impls/libfuzzer.rs | 189 ++++++++++ cargo-test-fuzz/src/fuzzer/impls/mod.rs | 2 + cargo-test-fuzz/src/fuzzer/mod.rs | 30 ++ cargo-test-fuzz/src/lib.rs | 339 +++++++----------- cargo-test-fuzz/src/transition.rs | 116 +++--- cargo-test-fuzz/tests/auto_concretize.rs | 6 +- cargo-test-fuzz/tests/auto_generate.rs | 25 +- cargo-test-fuzz/tests/breadcrumbs.rs | 87 +++++ cargo-test-fuzz/tests/build.rs | 15 +- cargo-test-fuzz/tests/concretizations.rs | 9 +- cargo-test-fuzz/tests/consolidate.rs | 39 +- cargo-test-fuzz/tests/display.rs | 9 +- cargo-test-fuzz/tests/fuzz.rs | 38 +- cargo-test-fuzz/tests/fuzz_generic.rs | 36 +- cargo-test-fuzz/tests/replay.rs | 44 ++- cargo-test-fuzz/tests/test_log.rs | 24 +- cargo-test-fuzz/tests/third_party.rs | 40 ++- cargo-test-fuzz/third_party.json | 4 +- clippy.toml | 3 + docs/crates.dot | 6 +- docs/libfuzzer_notes.md | 30 ++ examples/Cargo.toml | 8 +- examples/tests/qwerty_stepped.rs | 17 + internal/Cargo.toml | 7 +- internal/src/dirs.rs | 45 ++- internal/src/fuzzer.rs | 42 +++ internal/src/lib.rs | 3 + macro/Cargo.toml | 6 +- macro/build.rs | 7 + macro/src/auto_concretize.rs | 2 +- macro/src/fuzzer.rs | 145 ++++++++ macro/src/lib.rs | 155 ++++---- macro/src/mod_utils.rs | 6 +- runtime/Cargo.toml | 9 +- runtime/build.rs | 7 + runtime/src/lib.rs | 19 +- runtime/src/libfuzzer.rs | 67 ++++ test-fuzz/Cargo.toml | 14 +- test-fuzz/build.rs | 7 + test-fuzz/src/lib.rs | 2 +- test-fuzz/tests/auto_generate.rs | 5 +- test-fuzz/tests/conversion.rs | 1 + test-fuzz/tests/default.rs | 5 +- test-fuzz/tests/github.rs | 69 ++++ test-fuzz/tests/in_production.rs | 1 + test-fuzz/tests/link.rs | 9 +- test-fuzz/tests/rename.rs | 1 + test-fuzz/tests/serde_format.rs | 5 +- test-fuzz/tests/test_fuzz_log.rs | 5 +- test-fuzz/tests/versions.rs | 1 + testing/Cargo.toml | 2 +- testing/src/command_ext.rs | 14 + testing/src/lib.rs | 3 + 58 files changed, 1613 insertions(+), 584 deletions(-) create mode 100644 cargo-test-fuzz/src/fuzzer/impls/aflplusplus.rs create mode 100644 cargo-test-fuzz/src/fuzzer/impls/libfuzzer.rs create mode 100644 cargo-test-fuzz/src/fuzzer/impls/mod.rs create mode 100644 cargo-test-fuzz/src/fuzzer/mod.rs create mode 100644 cargo-test-fuzz/tests/breadcrumbs.rs create mode 100644 clippy.toml create mode 100644 docs/libfuzzer_notes.md create mode 100644 examples/tests/qwerty_stepped.rs create mode 100644 internal/src/fuzzer.rs create mode 100644 macro/src/fuzzer.rs create mode 100644 runtime/src/libfuzzer.rs create mode 100644 test-fuzz/tests/github.rs create mode 100644 testing/src/command_ext.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 951d6b3c..241efb79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,12 +97,47 @@ jobs: cargo clean && cargo +nightly udeps --features=test-fuzz/auto_concretize --all-targets test: - runs-on: ubuntu-latest - strategy: matrix: - serde_format: [bincode, cbor, cbor4ii] - toolchain: [stable, nightly] + include: + - fuzzer: aflplusplus + serde_format: bincode + environment: ubuntu-latest + toolchain: stable + - fuzzer: aflplusplus + serde_format: cbor + environment: ubuntu-latest + toolchain: nightly + - fuzzer: aflplusplus + serde_format: cbor4ii + environment: macos-latest + toolchain: stable + - fuzzer: aflplusplus-persistent + serde_format: bincode + environment: macos-latest + toolchain: nightly + - fuzzer: aflplusplus-persistent + serde_format: cbor + environment: ubuntu-latest + toolchain: stable + - fuzzer: aflplusplus-persistent + serde_format: cbor4ii + environment: ubuntu-latest + toolchain: nightly + - fuzzer: libfuzzer + serde_format: bincode + environment: macos-latest + toolchain: stable + - fuzzer: libfuzzer + serde_format: cbor + environment: macos-latest + toolchain: nightly + - fuzzer: libfuzzer + serde_format: cbor4ii + environment: ubuntu-latest + toolchain: stable + + runs-on: ${{ matrix.environment }} env: CARGO_TERM_COLOR: always @@ -114,6 +149,7 @@ jobs: run: rustup default ${{ matrix.toolchain }} - name: Install llvm + if: ${{ matrix.environment == 'ubuntu-latest' }} run: sudo apt-get install llvm # smoelius: The Substrate tests require `protoc`. @@ -137,11 +173,11 @@ jobs: AUTO_CONCRETIZE= IGNORED= SHUFFLE= - if [[ ${{ matrix.toolchain }} = nightly ]]; then + if [[ ${{ matrix.toolchain }} = 'nightly' ]]; then AUTO_CONCRETIZE='--features=test-fuzz/auto_concretize' SHUFFLE='-Z unstable-options --shuffle --test-threads=1' fi - if [[ ${{ github.event_name }} = schedule ]]; then + if [[ ${{ github.event_name }} = 'schedule' ]]; then IGNORED='--ignored' fi cargo test --features=test-fuzz/serde_${{ matrix.serde_format }} "$AUTO_CONCRETIZE" -- --nocapture $IGNORED $SHUFFLE @@ -149,6 +185,7 @@ jobs: AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 RUST_BACKTRACE: 1 RUST_LOG: warn + TEST_FUZZ_FUZZER: ${{ matrix.fuzzer }} test-uninstalled-cargo-afl: runs-on: ubuntu-latest @@ -164,7 +201,7 @@ jobs: run: | OUTPUT="$(cargo run -p cargo-test-fuzz -- test-fuzz -p test-fuzz-examples --no-run 2>&1 1>/dev/null || true)" echo "$OUTPUT" - echo "$OUTPUT" | grep '^Error: Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`.$' + echo "$OUTPUT" | grep 'Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`.' test-incompatible-cargo-afl: runs-on: ubuntu-latest @@ -172,6 +209,7 @@ jobs: env: CARGO_TERM_COLOR: always RUSTUP_TOOLCHAIN: nightly + TEST_FUZZ_FUZZER: aflplusplus-persistent steps: - uses: actions/checkout@v3 @@ -194,6 +232,7 @@ jobs: env: CARGO_TERM_COLOR: always RUSTUP_TOOLCHAIN: nightly + TEST_FUZZ_FUZZER: aflplusplus-persistent steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index db8a017a..2116056b 100644 --- a/README.md +++ b/README.md @@ -248,50 +248,52 @@ The `cargo test-fuzz` command is used to interact with fuzz targets, and to mani #### Options ``` - --backtrace Display backtraces - --consolidate Move one target's crashes, hangs, and work queue to its corpus; to - consolidate all targets, use --consolidate-all - --display Display concretizations, corpus, crashes, `impl` concretizations, - hangs, or work queue. By default, corpus uses an uninstrumented fuzz - target; the others use an instrumented fuzz target. To display the - corpus with instrumentation, use --display corpus-instrumented. - [possible values: concretizations, corpus, corpus-instrumented, - crashes, hangs, impl-concretizations, queue] - --exact Target name is an exact name rather than a substring - --exit-code Exit with 0 if the time limit was reached, 1 for other programmatic - aborts, and 2 if an error occurred; implies --no-ui, does not imply - --run-until-crash or -- -V - --features Space or comma separated list of features to activate - --list List fuzz targets - --manifest-path Path to Cargo.toml - --no-default-features Do not activate the `default` feature - --no-instrumentation Compile without instrumentation (for testing build process) - --no-run Compile, but don't fuzz - --no-ui Disable user interface - -p, --package Package containing fuzz target - --persistent Enable persistent mode fuzzing - --pretty-print Pretty-print debug output when displaying/replaying - --replay Replay corpus, crashes, hangs, or work queue. By default, corpus uses - an uninstrumented fuzz target; the others use an instrumented fuzz - target. To replay the corpus with instrumentation, use --replay - corpus-instrumented. [possible values: concretizations, corpus, - corpus-instrumented, crashes, hangs, impl-concretizations, queue] - --reset Clear fuzzing data for one target, but leave corpus intact; to reset - all targets, use --reset-all - --resume Resume target's last fuzzing session - --run-until-crash Stop fuzzing once a crash is found - --test Integration test containing fuzz target - --timeout Number of milliseconds to consider a hang when fuzzing or replaying - (equivalent to -- -t when fuzzing) - --verbose Show build output when displaying/replaying - -h, --help Print help - -V, --version Print version - -To fuzz at most of time, use: - - cargo test-fuzz ... -- -V - -Try `cargo afl fuzz --help` to see additional fuzzer options. + --backtrace Display backtraces + --consolidate Move one target's crashes, hangs, and work queue to its corpus; to + consolidate all targets, use --consolidate-all + --display Display concretizations, corpus, crashes, `impl` concretizations, + hangs, or work queue. By default, corpus uses an uninstrumented + fuzz target; the others use an instrumented fuzz target. To + display the corpus with instrumentation, use --display + corpus-instrumented. [possible values: concretizations, corpus, + corpus-instrumented, crashes, hangs, impl-concretizations, queue] + --exact Target name is an exact name rather than a substring + --exit-code Exit with 0 if the time limit was reached, 1 for other + programmatic aborts, and 2 if an error occurred; implies --no-ui, + does not imply --run-until-crash or --max-total-time + --features Space or comma separated list of features to activate + --fuzzer Fuzz using [possible values: aflplusplus, + aflplusplus-persistent, libfuzzer] + --list List fuzz targets + --manifest-path Path to Cargo.toml + --max-total-time Fuzz at most of time (equivalent to -- -V for + aflplusplus, and -- --max_total_time for libfuzzer) + --no-default-features Do not activate the `default` feature + --no-instrumentation Compile without instrumentation (for testing build process) + --no-run Compile, but don't fuzz + --no-ui Disable user interface + -p, --package Package containing fuzz target + --pretty-print Pretty-print debug output when displaying/replaying + --replay Replay corpus, crashes, hangs, or work queue. By default, corpus + uses an uninstrumented fuzz target; the others use an instrumented + fuzz target. To replay the corpus with instrumentation, use + --replay corpus-instrumented. [possible values: concretizations, + corpus, corpus-instrumented, crashes, hangs, impl-concretizations, + queue] + --reset Clear fuzzing data for one target, but leave corpus intact; to + reset all targets, use --reset-all + --resume Resume target's last fuzzing session + --run-until-crash Stop fuzzing once a crash is found + --test Integration test containing fuzz target + --timeout Number of milliseconds to consider a hang when fuzzing or + replaying (equivalent to -- -t when fuzzing with + aflplusplus, and -- -timeout when fuzzing with + libfuzzer) + --verbose Show build output when displaying/replaying + -h, --help Print help + -V, --version Print version + +Try `cargo afl fuzz --help` to see additional AFLplusplus options. ``` ### Convenience functions and macros diff --git a/cargo-test-fuzz/Cargo.toml b/cargo-test-fuzz/Cargo.toml index f743dd65..4178596a 100644 --- a/cargo-test-fuzz/Cargo.toml +++ b/cargo-test-fuzz/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "cargo-test-fuzz" version = "3.0.5" -edition = "2018" +edition = "2021" description = "cargo-test-fuzz" @@ -17,18 +17,22 @@ path = "src/bin/cargo_test_fuzz.rs" doctest = false [dependencies] -anyhow = "1.0" +anyhow = { version = "1.0", features = ["backtrace"] } bitflags = "2.1" +cargo-fuzz = { git = "https://github.com/trail-of-forks/cargo-fuzz", features = ["no-manifest-check"] } cargo_metadata = "0.15" clap = { version = "4.2", features = ["cargo", "derive", "wrap_help"] } env_logger = "0.10" +fs_extra = "1.3" heck = "0.4" lazy_static = "1.4" log = "0.4" +once_cell = "1.16" paste = "1.0" remain = "0.2" semver = "1.0" serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" strum_macros = "0.24" subprocess = "0.2" diff --git a/cargo-test-fuzz/patches/solana.patch b/cargo-test-fuzz/patches/solana.patch index a4744dbd..d9b6bd2a 100644 --- a/cargo-test-fuzz/patches/solana.patch +++ b/cargo-test-fuzz/patches/solana.patch @@ -21,43 +21,41 @@ index 7f00f43..38eacf7 100644 pub struct ComputeBudget { /// Number of compute units that a transaction or individual instruction is diff --git a/program-runtime/src/invoke_context.rs b/program-runtime/src/invoke_context.rs -index 9a12416..decd9ca 100644 +index cc842dd..ca63985 100644 --- a/program-runtime/src/invoke_context.rs +++ b/program-runtime/src/invoke_context.rs -@@ -123,12 +123,29 @@ impl fmt::Display for AllocErr { +@@ -118,4 +118,5 @@ impl fmt::Display for AllocErr { } -+struct DummyAllocator; -+ -+impl Alloc for DummyAllocator { -+ fn alloc(&mut self, _layout: Layout) -> Result { -+ std::process::exit(0); -+ } -+ fn dealloc(&mut self, _addr: u64, _layout: Layout) { -+ std::process::exit(0); -+ } -+} -+ -+fn dummy_allocator() -> Rc> { -+ Rc::new(RefCell::new(DummyAllocator)) -+} -+ +#[derive(Clone, serde::Deserialize, serde::Serialize)] - struct SyscallContext { - check_aligned: bool, - check_size: bool, - orig_account_lengths: Vec, -+ #[serde(skip, default = "dummy_allocator")] - allocator: Rc>, + pub struct BpfAllocator { + len: u64, +@@ -146,4 +147,5 @@ impl BpfAllocator { } --#[derive(Default)] -+#[derive(Default, Clone, serde::Deserialize, serde::Serialize)] - pub struct TraceLogStackFrame { - pub trace_log: Vec<[u64; 12]>, -@@ -136,8 +153,35 @@ pub struct TraceLogStackFrame { ++#[derive(Clone, serde::Deserialize, serde::Serialize)] + pub struct SyscallContext { + pub allocator: BpfAllocator, +@@ -152,9 +154,54 @@ pub struct SyscallContext { } ++pub fn serialize_ref(x: &&T, serializer: S) -> Result ++where ++ S: serde::Serializer, ++ T: serde::Serialize, ++{ ++ ::serialize(*x, serializer) ++} ++ ++pub fn deserialize_ref<'de, D, T>(deserializer: D) -> Result<&'static T, D::Error> ++where ++ D: serde::Deserializer<'de>, ++ T: serde::de::DeserializeOwned + std::fmt::Debug, ++{ ++ let x = ::deserialize(deserializer)?; ++ Ok(Box::leak(Box::new(x))) ++} ++ +pub fn serialize_ref_mut(x: &&mut T, serializer: S) -> Result +where + S: serde::Serializer, @@ -90,8 +88,16 @@ index 9a12416..decd9ca 100644 pre_accounts: Vec, + #[serde(skip, default = "default_builtin_programs")] builtin_programs: &'a [BuiltinProgram], ++ #[serde(serialize_with = "serialize_ref", deserialize_with = "deserialize_ref")] sysvar_cache: &'a SysvarCache, -@@ -157,4 +201,22 @@ pub struct InvokeContext<'a> { + log_collector: Option>>, +@@ -163,4 +210,5 @@ pub struct InvokeContext<'a> { + compute_meter: RefCell, + accounts_data_meter: AccountsDataMeter, ++ #[serde(skip)] + pub tx_executor_cache: Rc>, + pub feature_set: Arc, +@@ -171,4 +219,20 @@ pub struct InvokeContext<'a> { } +impl<'a> Clone for InvokeContext<'a> { @@ -99,8 +105,6 @@ index 9a12416..decd9ca 100644 + Self { + transaction_context: Box::leak(Box::new(self.transaction_context.clone())), + pre_accounts: self.pre_accounts.clone(), -+ sysvar_cache: self.sysvar_cache.clone(), -+ trace_log_stack: self.trace_log_stack.clone(), + log_collector: self.log_collector.clone(), + compute_meter: self.compute_meter.clone(), + tx_executor_cache: self.tx_executor_cache.clone(), @@ -177,10 +181,10 @@ index 79de085..39f0384 100644 [dev-dependencies] memoffset = "0.8" diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs -index 65546a1..6c64602 100644 +index db06f2c..5e5bab4 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs -@@ -423,6 +423,7 @@ fn create_memory_mapping<'a, 'b, C: ContextObject>( +@@ -438,6 +438,7 @@ fn create_memory_mapping<'a, 'b, C: ContextObject>( } -pub fn process_instruction( @@ -209,10 +213,10 @@ index 7e8368d..56d903d 100644 [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = { workspace = true } diff --git a/sdk/src/feature_set.rs b/sdk/src/feature_set.rs -index 5437320..3342b47 100644 +index 7a85b12..339d328 100644 --- a/sdk/src/feature_set.rs +++ b/sdk/src/feature_set.rs -@@ -846,5 +846,5 @@ lazy_static! { +@@ -851,5 +851,5 @@ lazy_static! { /// `FeatureSet` holds the set of currently active/inactive runtime features -#[derive(AbiExample, Debug, Clone, Eq, PartialEq)] diff --git a/cargo-test-fuzz/src/fuzzer/impls/aflplusplus.rs b/cargo-test-fuzz/src/fuzzer/impls/aflplusplus.rs new file mode 100644 index 00000000..7ec32f99 --- /dev/null +++ b/cargo-test-fuzz/src/fuzzer/impls/aflplusplus.rs @@ -0,0 +1,192 @@ +use super::super::Interface; +use crate::{ + check_dependency_version, exec_from_command, Executable, TestFuzz, BASE_ENVS, ENTRY_SUFFIX, + NANOS_PER_MILLI, +}; +use anyhow::{anyhow, bail, ensure, Context, Result}; +use internal::Fuzzer; +use log::debug; +use once_cell::sync::OnceCell; +use semver::Version; +use std::{ + io::BufRead, + path::Path, + process::{exit, Command}, +}; +use subprocess::Redirection; + +struct Impl { + persistent: bool, +} + +static IMPL_PERSISTENT: Impl = Impl { persistent: true }; +static IMPL: Impl = Impl { persistent: false }; + +pub(crate) fn instantiate(persistent: bool) -> &'static dyn Interface { + if persistent { + &IMPL_PERSISTENT + } else { + &IMPL + } +} + +impl Interface for Impl { + fn to_enum(&self) -> internal::Fuzzer { + if self.persistent { + Fuzzer::AflplusplusPersistent + } else { + Fuzzer::Aflplusplus + } + } + + fn dependency_name(&self) -> Option<&'static str> { + if self.persistent { + Some("afl") + } else { + None + } + } + + fn check_dependency_version(&self, executable_name: &str, version: &Version) -> Result<()> { + check_dependency_version( + executable_name, + "afl", + Some(version), + "cargo-afl", + cached_cargo_afl_version(), + "afl", + ) + } + + fn build(&self, manifest_path: Option<&Path>) -> Result { + let mut command = Self::make_cargo_command(manifest_path, "test"); + command.args(["--no-run"]); + Ok(command) + } + + fn fuzz( + &self, + opts: &TestFuzz, + executable: &Executable, + target: &str, + input_dir: &Path, + output_dir: &Path, + ) -> Result<()> { + let mut command = Command::new("cargo"); + + command.envs(BASE_ENVS.iter().copied()); + if opts.no_ui { + command.env("AFL_NO_UI", "1"); + } + if opts.run_until_crash { + command.env("AFL_BENCH_UNTIL_CRASH", "1"); + } + + command.args([ + "afl", + "fuzz", + "-i", + &input_dir.to_string_lossy(), + "-o", + &output_dir.to_string_lossy(), + "-D", + "-M", + "default", + ]); + if let Some(max_total_time) = opts.max_total_time { + command.args(["-V".to_owned(), max_total_time.to_string()]); + } + if let Some(timeout) = opts.timeout { + command.args(["-t".to_owned(), (timeout * NANOS_PER_MILLI).to_string()]); + } + command.args(opts.zzargs.clone()); + command.args([ + "--", + &executable.path.to_string_lossy(), + "--exact", + &(target.to_owned() + ENTRY_SUFFIX), + ]); + + #[allow(clippy::if_not_else)] + if !opts.exit_code { + debug!("{:?}", command); + let status = command + .status() + .with_context(|| format!("Could not get status of `{command:?}`"))?; + + ensure!(status.success(), "Command failed: {:?}", command); + } else { + let exec = exec_from_command(&command).stdout(Redirection::Pipe); + debug!("{:?}", exec); + let mut popen = exec.clone().popen()?; + let stdout = popen + .stdout + .take() + .ok_or_else(|| anyhow!("Could not get output of `{:?}`", exec))?; + let mut time_limit_was_reached = false; + let mut testing_aborted_programmatically = false; + for line in std::io::BufReader::new(stdout).lines() { + let line = line.with_context(|| format!("Could not get output of `{exec:?}`"))?; + if line.contains("Time limit was reached") { + time_limit_was_reached = true; + } + // smoelius: Work around "pizza mode" bug. + if line.contains("+++ Testing aborted programmatically +++") + || line.contains("+++ Baking aborted programmatically +++") + { + testing_aborted_programmatically = true; + } + println!("{line}"); + } + let status = popen + .wait() + .with_context(|| format!("`wait` failed for `{popen:?}`"))?; + + if !testing_aborted_programmatically || !status.success() { + bail!("Command failed: {:?}", exec); + } + + if !time_limit_was_reached { + exit(1); + } + } + + Ok(()) + } +} + +impl Impl { + fn make_cargo_command(manifest_path: Option<&Path>, subcommand: &str) -> Command { + // smoelius: Ensure `cargo-afl` is installed. + let _ = cached_cargo_afl_version(); + + let mut command = Command::new("cargo"); + command.args(["afl", subcommand]); + if let Some(path) = manifest_path { + command.args(["--manifest-path", &path.to_string_lossy()]); + } + command + } +} + +fn cached_cargo_afl_version() -> &'static Version { + #[allow(clippy::unwrap_used)] + CARGO_AFL_VERSION.get_or_init(|| cargo_afl_version().unwrap()) +} + +static CARGO_AFL_VERSION: OnceCell = OnceCell::new(); + +fn cargo_afl_version() -> Result { + let mut command = Command::new("cargo"); + command.args(["afl", "--version"]); + let output = command + .output() + .with_context(|| format!("Could not get output of `{command:?}`"))?; + let stdout = String::from_utf8_lossy(&output.stdout); + let version = stdout.strip_prefix("cargo-afl ").ok_or_else(|| { + anyhow!( + "Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`." + ) + })?; + Version::parse(version.trim_end()).map_err(Into::into) +} diff --git a/cargo-test-fuzz/src/fuzzer/impls/libfuzzer.rs b/cargo-test-fuzz/src/fuzzer/impls/libfuzzer.rs new file mode 100644 index 00000000..363586b1 --- /dev/null +++ b/cargo-test-fuzz/src/fuzzer/impls/libfuzzer.rs @@ -0,0 +1,189 @@ +use super::super::Interface; +use crate::{Executable, TestFuzz, BASE_ENVS, ENTRY_SUFFIX}; +use anyhow::{bail, Context, Result}; +use cargo_fuzz::{ + options::{default_opts, BuildOptions, Sanitizer}, + project::FuzzProject, +}; +use internal::Fuzzer; +use log::debug; +use semver::Version; +use std::{ + fs::{create_dir_all, remove_dir_all}, + path::Path, + process::{exit, Command}, + time::Duration, +}; + +const OUTPUT_GRACE: u64 = 25; // minutes + +// smoelius: libfuzzer's default is 20 minutes, which is way too long. 1000 milliseconds is +// AFLplusplus's default timeout: +// https://github.com/AFLplusplus/AFLplusplus/blob/0c0a6c3bfabf0facaed33fae1aa5ad54a6a11b32/include/config.h#L127-L130 +const DEFAULT_TIMEOUT: u64 = 1000; // milliseconds + +// smoelius: +// https://github.com/rust-fuzz/libfuzzer/blob/03a00b2c2ab7b82838536883e0b66eae59ab130d/libfuzzer/FuzzerOptions.h#L23-L26 +// `INTERRUPT_EXIT_CODE` is currently unused. +const TIMEOUT_EXIT_CODE: i32 = 70; +const OOM_EXIT_CODE: i32 = 71; +#[allow(dead_code)] +const INTERRUPT_EXIT_CODE: i32 = 72; +const ERROR_EXIT_CODE: i32 = 77; + +struct Impl; + +static IMPL: Impl = Impl; + +pub(crate) fn instantiate() -> &'static dyn Interface { + &IMPL +} + +impl Interface for Impl { + fn to_enum(&self) -> internal::Fuzzer { + Fuzzer::Libfuzzer + } + + fn dependency_name(&self) -> Option<&'static str> { + Some("libfuzzer-sys") + } + + fn check_dependency_version(&self, _executable_name: &str, _version: &Version) -> Result<()> { + Ok(()) + } + + fn build(&self, manifest_path: Option<&Path>) -> Result { + let project = Self::make_project(manifest_path)?; + let build = BuildOptions { + dev: true, + release: false, + sanitizer: Sanitizer::None, + ..default_opts() + }; + let mut command = project.cargo("build", &build)?; + extend_env(&mut command, "RUSTFLAGS", " -C debuginfo=2"); + Ok(command) + } + + fn fuzz( + &self, + opts: &TestFuzz, + executable: &Executable, + target: &str, + input_dir: &Path, + output_dir: &Path, + ) -> Result<()> { + if !opts.resume && output_dir.exists() { + if !within_grace_period(output_dir)? { + bail!("Found at-risk data in {output_dir:?}"); + } + remove_dir_all(output_dir) + .with_context(|| format!("Could not remove {output_dir:?}"))?; + } + + let output_queue = output_dir.join("queue"); + let output_artifacts = output_dir.join("artifacts"); + + create_dir_all(&output_queue).unwrap_or_default(); + create_dir_all(&output_artifacts).unwrap_or_default(); + + fs_extra::dir::copy( + input_dir, + &output_queue, + &fs_extra::dir::CopyOptions { + content_only: true, + overwrite: true, + ..Default::default() + }, + )?; + + let mut args = vec![ + output_queue.to_string_lossy().to_string(), + format!("-artifact_prefix={}/", output_artifacts.to_string_lossy()), + "-detect_leaks=0".to_owned(), + // smoelius: Without `-reduce_inputs`, Libfuzzer fails to find hangs in + // `parse_duration`. I don't know why. + // "-reduce_inputs=0".to_owned(), + ]; + if let Some(max_total_time) = opts.max_total_time { + args.push(format!("-max_total_time={max_total_time}")); + } + let timeout = opts.timeout.unwrap_or(DEFAULT_TIMEOUT); + args.push(format!("-timeout={}", (timeout + 999) / 1000)); + args.extend(opts.zzargs.iter().cloned()); + let args_json = serde_json::to_string(&args)?; + + let mut command = Command::new(&executable.path); + + command.envs(BASE_ENVS.iter().copied()); + + command.env("TEST_FUZZ_LIBFUZZER_ARGS", args_json); + + command.args([ + "--exact", + &(target.to_owned() + ENTRY_SUFFIX), + "--nocapture", + "--test-threads=1", + ]); + + debug!("{:?}", command.get_envs()); + debug!("{:?}", command); + + let status = command + .status() + .with_context(|| format!("Could not get status of `{command:?}`"))?; + + if !status.success() { + let code = status + .code() + .with_context(|| format!("Could not get exit code of `{command:?}`"))?; + if opts.exit_code { + exit( + if [TIMEOUT_EXIT_CODE, OOM_EXIT_CODE, ERROR_EXIT_CODE].contains(&code) { + 1 + } else { + 2 + }, + ); + } + exit(code); + } + + Ok(()) + } +} + +impl Impl { + fn make_project(manifest_path: Option<&Path>) -> Result { + let manifest_dir = manifest_path + .and_then(Path::parent) + .or_else(|| Some(Path::new("."))); + FuzzProject::new(manifest_dir.map(Path::to_path_buf)) + } +} + +fn within_grace_period(path: &Path) -> Result { + let metadata = path + .metadata() + .with_context(|| format!("Could not get {path:?} metadata"))?; + let created = metadata + .created() + .with_context(|| format!("Could not get {path:?} creation time"))?; + let modified = metadata + .modified() + .with_context(|| format!("Could not get {path:?} modification time"))?; + let elapsed = modified.duration_since(created)?; + Ok(elapsed < Duration::from_secs(OUTPUT_GRACE * 60)) +} + +fn extend_env(command: &mut Command, key: &str, val: &str) { + let Some(mut val_next) = command + .get_envs() + .find_map(|(key_curr, val_curr)| if key == key_curr { val_curr } else { None }) + .map(ToOwned::to_owned) + else { + return; + }; + val_next.push(val); + command.env(key, val_next); +} diff --git a/cargo-test-fuzz/src/fuzzer/impls/mod.rs b/cargo-test-fuzz/src/fuzzer/impls/mod.rs new file mode 100644 index 00000000..b8482fdf --- /dev/null +++ b/cargo-test-fuzz/src/fuzzer/impls/mod.rs @@ -0,0 +1,2 @@ +pub mod aflplusplus; +pub mod libfuzzer; diff --git a/cargo-test-fuzz/src/fuzzer/mod.rs b/cargo-test-fuzz/src/fuzzer/mod.rs new file mode 100644 index 00000000..ce6ec97c --- /dev/null +++ b/cargo-test-fuzz/src/fuzzer/mod.rs @@ -0,0 +1,30 @@ +use super::{Executable, TestFuzz}; +use anyhow::Result; +use internal::Fuzzer; +use semver::Version; +use std::{path::Path, process::Command}; + +mod impls; + +pub(super) fn instantiate(fuzzer: Fuzzer) -> &'static dyn Interface { + match fuzzer { + Fuzzer::Aflplusplus => impls::aflplusplus::instantiate(false), + Fuzzer::AflplusplusPersistent => impls::aflplusplus::instantiate(true), + Fuzzer::Libfuzzer => impls::libfuzzer::instantiate(), + } +} + +pub(super) trait Interface { + fn to_enum(&self) -> Fuzzer; + fn dependency_name(&self) -> Option<&'static str>; + fn check_dependency_version(&self, executable_name: &str, version: &Version) -> Result<()>; + fn build(&self, manifest_path: Option<&Path>) -> Result; + fn fuzz( + &self, + opts: &TestFuzz, + executable: &Executable, + target: &str, + input_dir: &Path, + output_dir: &Path, + ) -> Result<()>; +} diff --git a/cargo-test-fuzz/src/lib.rs b/cargo-test-fuzz/src/lib.rs index 07e2661d..761095f4 100644 --- a/cargo-test-fuzz/src/lib.rs +++ b/cargo-test-fuzz/src/lib.rs @@ -11,31 +11,32 @@ use cargo_metadata::{ }; use clap::{crate_version, ValueEnum}; use heck::ToKebabCase; -use internal::dirs::{ - concretizations_directory_from_target, corpus_directory_from_target, - crashes_directory_from_target, hangs_directory_from_target, - impl_concretizations_directory_from_target, output_directory_from_target, - queue_directory_from_target, target_directory, +use internal::{ + dirs::{ + concretizations_directory_from_target, corpus_directory_from_target, + crashes_directory_from_target, hangs_directory_from_target, + impl_concretizations_directory_from_target, output_directory_from_target, + queue_directory_from_target, target_directory, + }, + fuzzer, Fuzzer, }; -use lazy_static::lazy_static; use log::debug; -use semver::Version; -use semver::VersionReq; +use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use std::{ ffi::OsStr, fmt::{Debug, Formatter}, fs::{create_dir_all, read, read_dir, remove_dir_all, File}, io::{BufRead, Read}, - iter, path::{Path, PathBuf}, process::{exit, Command}, - sync::Mutex, time::Duration, }; use strum_macros::Display; use subprocess::{CommunicateError, Exec, ExitStatus, NullFile, Redirection}; +mod fuzzer; + mod transition; #[allow(deprecated)] pub use transition::cargo_test_fuzz; @@ -80,14 +81,15 @@ struct TestFuzz { exact: bool, exit_code: bool, features: Vec, + fuzzer: Option, list: bool, manifest_path: Option, + max_total_time: Option, no_default_features: bool, no_instrumentation: bool, no_run: bool, no_ui: bool, package: Option, - persistent: bool, pretty_print: bool, replay: Option, reset: bool, @@ -106,7 +108,7 @@ struct Executable { path: PathBuf, name: String, test_fuzz_version: Option, - afl_version: Option, + fuzzer_version: Option, } impl Debug for Executable { @@ -120,8 +122,8 @@ impl Debug for Executable { .as_ref() .map(ToString::to_string) .unwrap_or_default(); - let afl_version = self - .afl_version + let fuzzer_version = self + .fuzzer_version .as_ref() .map(ToString::to_string) .unwrap_or_default(); @@ -129,7 +131,7 @@ impl Debug for Executable { .field("path", &self.path) .field("name", &self.name) .field("test_fuzz_version", &test_fuzz_version) - .field("afl_version", &afl_version) + .field("fuzzer_version", &fuzzer_version) .finish() } } @@ -152,6 +154,9 @@ fn run(opts: TestFuzz) -> Result<()> { opts }; + let fuzzer_env = fuzzer()?; + let fuzzer = fuzzer::instantiate(opts.fuzzer.unwrap_or(fuzzer_env)); + if let Some(object) = opts.replay { ensure!( !matches!( @@ -163,13 +168,11 @@ fn run(opts: TestFuzz) -> Result<()> { ); } - cache_cargo_afl_version()?; - let display = opts.display.is_some(); let replay = opts.replay.is_some(); - let executables = build(&opts, display || replay)?; + let executables = build(&opts, fuzzer, display || replay)?; let mut executable_targets = executable_targets(&executables)?; @@ -177,7 +180,7 @@ fn run(opts: TestFuzz) -> Result<()> { executable_targets = filter_executable_targets(&opts, pat, &executable_targets); } - check_test_fuzz_and_afl_versions(&executable_targets)?; + check_test_fuzz_and_fuzzer_versions(fuzzer, &executable_targets)?; if opts.list { println!("{executable_targets:#?}"); @@ -190,28 +193,28 @@ fn run(opts: TestFuzz) -> Result<()> { if opts.consolidate_all || opts.reset_all { if opts.consolidate_all { - consolidate(&opts, &executable_targets)?; + consolidate(&opts, fuzzer, &executable_targets)?; } - return reset(&opts, &executable_targets); + return reset(&opts, fuzzer, &executable_targets); } let (executable, target) = executable_target(&opts, &executable_targets)?; if opts.consolidate || opts.reset { if opts.consolidate { - consolidate(&opts, &executable_targets)?; + consolidate(&opts, fuzzer, &executable_targets)?; } - return reset(&opts, &executable_targets); + return reset(&opts, fuzzer, &executable_targets); } let (flags, dir) = None .or_else(|| { opts.display - .map(|object| flags_and_dir(object, &executable.name, &target)) + .map(|object| flags_and_dir(fuzzer, object, &executable.name, &target)) }) .or_else(|| { opts.replay - .map(|object| flags_and_dir(object, &executable.name, &target)) + .map(|object| flags_and_dir(fuzzer, object, &executable.name, &target)) }) .unwrap_or((Flags::empty(), PathBuf::default())); @@ -224,7 +227,7 @@ fn run(opts: TestFuzz) -> Result<()> { return Ok(()); } - fuzz(&opts, &executable, &target).map_err(|error| { + fuzz(&opts, fuzzer, &executable, &target).map_err(|error| { if opts.exit_code { eprintln!("{error:?}"); exit(2); @@ -233,49 +236,47 @@ fn run(opts: TestFuzz) -> Result<()> { }) } -fn build(opts: &TestFuzz, quiet: bool) -> Result> { - let metadata = metadata(opts)?; +fn build(opts: &TestFuzz, fuzzer: &dyn fuzzer::Interface, quiet: bool) -> Result> { + let metadata = metadata(opts, fuzzer)?; - let mut args = vec![]; - if !opts.no_instrumentation { - args.extend_from_slice(&["afl"]); - } - args.extend_from_slice(&["test", "--frozen", "--offline", "--no-run"]); + let mut command = if opts.no_instrumentation { + let mut command = Command::new("cargo"); + command.args(["test", "--no-run"]); + if let Some(path) = &opts.manifest_path { + command.args(["--manifest-path", path]); + } + command + } else { + fuzzer.build(opts.manifest_path.as_deref().map(Path::new))? + }; + + // command.args(&["--frozen", "--offline"]); if opts.no_default_features { - args.extend_from_slice(&["--no-default-features"]); + command.args(["--no-default-features"]); } for features in &opts.features { - args.extend_from_slice(&["--features", features]); + command.args(["--features", features]); } - let target_dir = target_directory(true); - let target_dir_str = target_dir.to_string_lossy(); if !opts.no_instrumentation { - args.extend_from_slice(&["--target-dir", &target_dir_str]); - } - if let Some(path) = &opts.manifest_path { - args.extend_from_slice(&["--manifest-path", path]); + let target_dir = target_directory(Some(fuzzer.to_enum())); + let target_dir_str = target_dir.to_string_lossy(); + command.args(["--target-dir", &target_dir_str]); } if let Some(package) = &opts.package { - args.extend_from_slice(&["--package", package]); - } - if opts.persistent { - args.extend_from_slice(&["--features", "test-fuzz/__persistent"]); + command.args(["--package", package]); } + let fuzzer_as_feature = "test-fuzz/".to_owned() + &fuzzer.to_enum().as_feature(); + command.args(["--features", &fuzzer_as_feature]); if let Some(name) = &opts.test { - args.extend_from_slice(&["--test", name]); + command.args(["--test", name]); } // smoelius: Suppress "Warning: AFL++ tools will need to set AFL_MAP_SIZE..." Setting // `AFL_QUIET=1` doesn't work here, so pipe standard error to /dev/null. // smoelius: Suppressing all of standard error is too extreme. For now, suppress only when // displaying/replaying. - let mut exec = Exec::cmd("cargo") - .args( - &args - .iter() - .chain(iter::once(&"--message-format=json")) - .collect::>(), - ) + let mut exec = exec_from_command(&command) + .arg("--message-format=json") .stdout(Redirection::Pipe); if quiet && !opts.verbose { exec = exec.stderr(NullFile); @@ -306,7 +307,7 @@ fn build(opts: &TestFuzz, quiet: bool) -> Result> { // smoelius: If the command failed, re-execute it without --message-format=json. This is easier // than trying to capture and colorize `CompilerMessage`s like Cargo does. if !status.success() { - let mut popen = Exec::cmd("cargo").args(&args).popen()?; + let mut popen = exec_from_command(&command).popen()?; let status = popen .wait() .with_context(|| format!("`wait` failed for `{popen:?}`"))?; @@ -329,13 +330,13 @@ fn build(opts: &TestFuzz, quiet: bool) -> Result> { .. } = artifact { - let (test_fuzz_version, afl_version) = - test_fuzz_and_afl_versions(&metadata, &package_id)?; + let (test_fuzz_version, fuzzer_version) = + test_fuzz_and_fuzzer_versions(fuzzer, &metadata, &package_id)?; Ok(Some(Executable { path: executable.into(), name: build_target.name, test_fuzz_version, - afl_version, + fuzzer_version, })) } else { Ok(None) @@ -345,13 +346,13 @@ fn build(opts: &TestFuzz, quiet: bool) -> Result> { Ok(executables.into_iter().flatten().collect()) } -fn metadata(opts: &TestFuzz) -> Result { +fn metadata(opts: &TestFuzz, fuzzer: &dyn fuzzer::Interface) -> Result { let mut command = MetadataCommand::new(); if opts.no_default_features { command.features(CargoOpt::NoDefaultFeatures); } let mut features = opts.features.clone(); - features.push("test-fuzz/__persistent".to_owned()); + features.push("test-fuzz/".to_owned() + &fuzzer.to_enum().as_feature()); command.features(CargoOpt::SomeFeatures(features)); if let Some(path) = &opts.manifest_path { command.manifest_path(path); @@ -359,23 +360,42 @@ fn metadata(opts: &TestFuzz) -> Result { command.exec().map_err(Into::into) } -fn test_fuzz_and_afl_versions( +fn exec_from_command(command: &Command) -> Exec { + let mut exec = Exec::cmd(command.get_program()).args(&command.get_args().collect::>()); + for (key, val) in command.get_envs() { + if let Some(val) = val { + exec = exec.env(key, val); + } else { + exec = exec.env_remove(key); + } + } + if let Some(path) = command.get_current_dir() { + exec = exec.cwd(path); + } + exec +} + +fn test_fuzz_and_fuzzer_versions( + fuzzer: &dyn fuzzer::Interface, metadata: &Metadata, package_id: &PackageId, ) -> Result<(Option, Option)> { let test_fuzz = package_dependency(metadata, package_id, "test-fuzz")?; - let afl = test_fuzz + let fuzzer = test_fuzz .as_ref() - .map(|package_id| package_dependency(metadata, package_id, "afl")) + .zip(fuzzer.dependency_name()) + .map(|(package_id, dependency_name)| { + package_dependency(metadata, package_id, dependency_name) + }) .transpose()?; let test_fuzz_version = test_fuzz .map(|package_id| package_version(metadata, &package_id)) .transpose()?; - let afl_version = afl + let fuzzer_version = fuzzer .flatten() .map(|package_id| package_version(metadata, &package_id)) .transpose()?; - Ok((test_fuzz_version, afl_version)) + Ok((test_fuzz_version, fuzzer_version)) } fn package_dependency( @@ -540,7 +560,8 @@ fn match_message(opts: &TestFuzz) -> String { }) } -fn check_test_fuzz_and_afl_versions( +fn check_test_fuzz_and_fuzzer_versions( + fuzzer: &dyn fuzzer::Interface, executable_targets: &[(Executable, Vec)], ) -> Result<()> { let cargo_test_fuzz_version = Version::parse(crate_version!())?; @@ -553,52 +574,13 @@ fn check_test_fuzz_and_afl_versions( &cargo_test_fuzz_version, "cargo-test-fuzz", )?; - #[allow(clippy::expect_used)] - check_dependency_version( - &executable.name, - "afl", - executable.afl_version.as_ref(), - "cargo-afl", - CARGO_AFL_VERSION - .lock() - .expect("Could not lock `CARGO_AFL_VERSION`") - .as_ref() - .expect("Could not determine `cargo-afl` version"), - "afl", - )?; + if let Some(fuzzer_version) = executable.fuzzer_version.as_ref() { + fuzzer.check_dependency_version(&executable.name, fuzzer_version)?; + } } Ok(()) } -#[allow(clippy::significant_drop_tightening)] -fn cache_cargo_afl_version() -> Result<()> { - let cargo_afl_version = cargo_afl_version()?; - let mut lock = CARGO_AFL_VERSION - .lock() - .map_err(|error| anyhow!(error.to_string()))?; - *lock = Some(cargo_afl_version); - Ok(()) -} - -lazy_static! { - static ref CARGO_AFL_VERSION: Mutex> = Mutex::new(None); -} - -fn cargo_afl_version() -> Result { - let mut command = Command::new("cargo"); - command.args(["afl", "--version"]); - let output = command - .output() - .with_context(|| format!("Could not get output of `{command:?}`"))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let version = stdout.strip_prefix("cargo-afl ").ok_or_else(|| { - anyhow!( - "Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`." - ) - })?; - Version::parse(version.trim_end()).map_err(Into::into) -} - fn check_dependency_version( name: &str, dependency: &str, @@ -635,7 +617,11 @@ fn as_version_req(version: &Version) -> VersionReq { VersionReq::parse(&version.to_string()).expect("Could not parse version as version request") } -fn consolidate(opts: &TestFuzz, executable_targets: &[(Executable, Vec)]) -> Result<()> { +fn consolidate( + opts: &TestFuzz, + fuzzer: &dyn fuzzer::Interface, + executable_targets: &[(Executable, Vec)], +) -> Result<()> { assert!(opts.consolidate_all || executable_targets.len() == 1); for (executable, targets) in executable_targets { @@ -643,9 +629,10 @@ fn consolidate(opts: &TestFuzz, executable_targets: &[(Executable, Vec)] for target in targets { let corpus_dir = corpus_directory_from_target(&executable.name, target); - let crashes_dir = crashes_directory_from_target(&executable.name, target); - let hangs_dir = hangs_directory_from_target(&executable.name, target); - let queue_dir = queue_directory_from_target(&executable.name, target); + let crashes_dir = + crashes_directory_from_target(fuzzer.to_enum(), &executable.name, target); + let hangs_dir = hangs_directory_from_target(fuzzer.to_enum(), &executable.name, target); + let queue_dir = queue_directory_from_target(fuzzer.to_enum(), &executable.name, target); for dir in &[crashes_dir, hangs_dir, queue_dir] { for entry in read_dir(dir) @@ -681,14 +668,19 @@ fn consolidate(opts: &TestFuzz, executable_targets: &[(Executable, Vec)] Ok(()) } -fn reset(opts: &TestFuzz, executable_targets: &[(Executable, Vec)]) -> Result<()> { +fn reset( + opts: &TestFuzz, + fuzzer: &dyn fuzzer::Interface, + executable_targets: &[(Executable, Vec)], +) -> Result<()> { assert!(opts.reset_all || executable_targets.len() == 1); for (executable, targets) in executable_targets { assert!(opts.reset_all || targets.len() == 1); for target in targets { - let output_dir = output_directory_from_target(&executable.name, target); + let output_dir = + output_directory_from_target(fuzzer.to_enum(), &executable.name, target); if !output_dir.exists() { continue; } @@ -704,15 +696,29 @@ fn reset(opts: &TestFuzz, executable_targets: &[(Executable, Vec)]) -> R Ok(()) } -fn flags_and_dir(object: Object, krate: &str, target: &str) -> (Flags, PathBuf) { +fn flags_and_dir( + fuzzer: &dyn fuzzer::Interface, + object: Object, + krate: &str, + target: &str, +) -> (Flags, PathBuf) { match object { Object::Corpus | Object::CorpusInstrumented => ( Flags::REQUIRES_CARGO_TEST, corpus_directory_from_target(krate, target), ), - Object::Crashes => (Flags::empty(), crashes_directory_from_target(krate, target)), - Object::Hangs => (Flags::empty(), hangs_directory_from_target(krate, target)), - Object::Queue => (Flags::empty(), queue_directory_from_target(krate, target)), + Object::Crashes => ( + Flags::empty(), + crashes_directory_from_target(fuzzer.to_enum(), krate, target), + ), + Object::Hangs => ( + Flags::empty(), + hangs_directory_from_target(fuzzer.to_enum(), krate, target), + ), + Object::Queue => ( + Flags::empty(), + queue_directory_from_target(fuzzer.to_enum(), krate, target), + ), Object::ImplConcretizations => ( Flags::REQUIRES_CARGO_TEST | Flags::RAW, impl_concretizations_directory_from_target(krate, target), @@ -895,7 +901,12 @@ fn for_each_entry( } #[allow(clippy::too_many_lines)] -fn fuzz(opts: &TestFuzz, executable: &Executable, target: &str) -> Result<()> { +fn fuzz( + opts: &TestFuzz, + fuzzer: &dyn fuzzer::Interface, + executable: &Executable, + target: &str, +) -> Result<()> { let input_dir = if opts.resume { "-".to_owned() } else { @@ -917,98 +928,10 @@ fn fuzz(opts: &TestFuzz, executable: &Executable, target: &str) -> Result<()> { corpus_dir.to_string_lossy().into_owned() }; - let output_dir = output_directory_from_target(&executable.name, target); + let output_dir = output_directory_from_target(fuzzer.to_enum(), &executable.name, target); create_dir_all(&output_dir).unwrap_or_default(); - let mut envs = BASE_ENVS.to_vec(); - if opts.no_ui { - envs.push(("AFL_NO_UI", "1")); - } - if opts.run_until_crash { - envs.push(("AFL_BENCH_UNTIL_CRASH", "1")); - } - - let mut args = vec![]; - args.extend( - vec![ - "afl", - "fuzz", - "-i", - &input_dir, - "-o", - &output_dir.to_string_lossy(), - "-D", - "-M", - "default", - ] - .into_iter() - .map(String::from), - ); - if let Some(timeout) = opts.timeout { - args.extend(["-t".to_owned(), format!("{}", timeout * NANOS_PER_MILLI)]); - } - args.extend(opts.zzargs.clone()); - args.extend( - vec![ - "--", - &executable.path.to_string_lossy(), - "--exact", - &(target.to_owned() + ENTRY_SUFFIX), - ] - .into_iter() - .map(String::from), - ); - - #[allow(clippy::if_not_else)] - if !opts.exit_code { - let mut command = Command::new("cargo"); - command.envs(envs).args(args); - debug!("{:?}", command); - let status = command - .status() - .with_context(|| format!("Could not get status of `{command:?}`"))?; - - ensure!(status.success(), "Command failed: {:?}", command); - } else { - let exec = Exec::cmd("cargo") - .env_extend(&envs) - .args(&args) - .stdout(Redirection::Pipe); - debug!("{:?}", exec); - let mut popen = exec.clone().popen()?; - let stdout = popen - .stdout - .take() - .ok_or_else(|| anyhow!("Could not get output of `{:?}`", exec))?; - let mut time_limit_was_reached = false; - let mut testing_aborted_programmatically = false; - for line in std::io::BufReader::new(stdout).lines() { - let line = line.with_context(|| format!("Could not get output of `{exec:?}`"))?; - if line.contains("Time limit was reached") { - time_limit_was_reached = true; - } - // smoelius: Work around "pizza mode" bug. - if line.contains("+++ Testing aborted programmatically +++") - || line.contains("+++ Baking aborted programmatically +++") - { - testing_aborted_programmatically = true; - } - println!("{line}"); - } - let status = popen - .wait() - .with_context(|| format!("`wait` failed for `{popen:?}`"))?; - - if !testing_aborted_programmatically || !status.success() { - bail!("Command failed: {:?}", exec); - } - - if !time_limit_was_reached { - exit(1); - } - } - - Ok(()) + fuzzer.fuzz(opts, executable, target, Path::new(&input_dir), &output_dir) } fn auto_generate_corpus(executable: &Executable, target: &str) -> Result<()> { diff --git a/cargo-test-fuzz/src/transition.rs b/cargo-test-fuzz/src/transition.rs index a7d78f64..86a1f838 100644 --- a/cargo-test-fuzz/src/transition.rs +++ b/cargo-test-fuzz/src/transition.rs @@ -1,4 +1,4 @@ -use super::Object; +use super::{Fuzzer, Object}; use anyhow::Result; use clap::{crate_version, ArgAction, Parser}; use heck::ToKebabCase; @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use std::{env, ffi::OsStr}; #[derive(Debug, Parser)] -#[clap(bin_name = "cargo")] +#[command(bin_name = "cargo")] struct Opts { #[clap(subcommand)] subcmd: SubCommand, @@ -21,25 +21,22 @@ enum SubCommand { // smoelius: Wherever possible, try to reuse cargo test and libtest option names. #[allow(clippy::struct_excessive_bools)] #[derive(Clone, Debug, Deserialize, Parser, Serialize)] -#[clap(version = crate_version!(), after_help = "To fuzz at most of time, use: - - cargo test-fuzz ... -- -V - -Try `cargo afl fuzz --help` to see additional fuzzer options. +#[command(version = crate_version!(), after_help = "\ +Try `cargo afl fuzz --help` to see additional AFLplusplus options. ")] #[remain::sorted] struct TestFuzzWithDeprecations { - #[clap(long, help = "Display backtraces")] + #[arg(long, help = "Display backtraces")] backtrace: bool, - #[clap( + #[arg( long, help = "Move one target's crashes, hangs, and work queue to its corpus; to consolidate \ all targets, use --consolidate-all" )] consolidate: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] consolidate_all: bool, - #[clap( + #[arg( long, value_name = "OBJECT", help = "Display concretizations, corpus, crashes, `impl` concretizations, hangs, or work \ @@ -48,56 +45,66 @@ struct TestFuzzWithDeprecations { corpus-instrumented." )] display: Option, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_concretizations: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_corpus: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_corpus_instrumented: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_crashes: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_hangs: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_impl_concretizations: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] display_queue: bool, - #[clap(long, help = "Target name is an exact name rather than a substring")] + #[arg(long, help = "Target name is an exact name rather than a substring")] exact: bool, - #[clap( + #[arg( long, help = "Exit with 0 if the time limit was reached, 1 for other programmatic aborts, and 2 \ - if an error occurred; implies --no-ui, does not imply --run-until-crash or -- -V " + if an error occurred; implies --no-ui, does not imply --run-until-crash or \ + --max-total-time " )] exit_code: bool, - #[clap( + #[arg( long, action = ArgAction::Append, help = "Space or comma separated list of features to activate" )] features: Vec, - #[clap(long, help = "List fuzz targets")] + #[arg(long, help = "Fuzz using ")] + fuzzer: Option, + #[arg(long, help = "List fuzz targets")] list: bool, - #[clap(long, value_name = "PATH", help = "Path to Cargo.toml")] + #[arg(long, value_name = "PATH", help = "Path to Cargo.toml")] manifest_path: Option, - #[clap(long, help = "Do not activate the `default` feature")] + #[arg( + long, + value_name = "SECONDS", + help = "Fuzz at most of time (equivalent to -- -V for aflplusplus, and \ + -- --max_total_time for libfuzzer)" + )] + max_total_time: Option, + #[arg(long, help = "Do not activate the `default` feature")] no_default_features: bool, - #[clap( + #[arg( long, help = "Compile without instrumentation (for testing build process)" )] no_instrumentation: bool, - #[clap(long, help = "Compile, but don't fuzz")] + #[arg(long, help = "Compile, but don't fuzz")] no_run: bool, - #[clap(long, help = "Disable user interface")] + #[arg(long, help = "Disable user interface")] no_ui: bool, - #[clap(short, long, help = "Package containing fuzz target")] + #[arg(short, long, help = "Package containing fuzz target")] package: Option, - #[clap(long, help = "Enable persistent mode fuzzing")] + #[arg(long, hide = true)] persistent: bool, - #[clap(long, help = "Pretty-print debug output when displaying/replaying")] + #[arg(long, help = "Pretty-print debug output when displaying/replaying")] pretty_print: bool, - #[clap( + #[arg( long, value_name = "OBJECT", help = "Replay corpus, crashes, hangs, or work queue. By default, corpus uses an \ @@ -105,50 +112,51 @@ struct TestFuzzWithDeprecations { corpus with instrumentation, use --replay corpus-instrumented." )] replay: Option, - #[clap(long, hide = true)] + #[arg(long, hide = true)] replay_corpus: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] replay_corpus_instrumented: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] replay_crashes: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] replay_hangs: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] replay_queue: bool, - #[clap( + #[arg( long, help = "Clear fuzzing data for one target, but leave corpus intact; to reset all \ targets, use --reset-all" )] reset: bool, - #[clap(long, hide = true)] + #[arg(long, hide = true)] reset_all: bool, - #[clap(long, help = "Resume target's last fuzzing session")] + #[arg(long, help = "Resume target's last fuzzing session")] resume: bool, - #[clap(long, help = "Stop fuzzing once a crash is found")] + #[arg(long, help = "Stop fuzzing once a crash is found")] run_until_crash: bool, - #[clap(long, value_name = "TARGETNAME", hide = true)] + #[arg(long, value_name = "TARGETNAME", hide = true)] target: Option, - #[clap( + #[arg( long, value_name = "NAME", help = "Integration test containing fuzz target" )] test: Option, - #[clap( + #[arg( long, help = "Number of milliseconds to consider a hang when fuzzing or replaying (equivalent \ - to -- -t when fuzzing)" + to -- -t when fuzzing with aflplusplus, and -- -timeout when \ + fuzzing with libfuzzer)" )] timeout: Option, - #[clap(long, help = "Show build output when displaying/replaying")] + #[arg(long, help = "Show build output when displaying/replaying")] verbose: bool, - #[clap( + #[arg( value_name = "TARGETNAME", help = "String that fuzz target's name must contain" )] ztarget: Option, - #[clap(last = true, name = "ARGS", help = "Arguments for the fuzzer")] + #[arg(last = true, value_name = "ARGS", help = "Arguments for the fuzzer")] zzargs: Vec, } @@ -169,14 +177,16 @@ impl From for super::TestFuzz { exact, exit_code, features, + fuzzer, list, manifest_path, + max_total_time, no_default_features, no_instrumentation, no_run, no_ui, package, - persistent, + persistent: _, pretty_print, replay, replay_corpus: _, @@ -203,14 +213,15 @@ impl From for super::TestFuzz { exact, exit_code, features, + fuzzer, list, manifest_path, + max_total_time, no_default_features, no_instrumentation, no_run, no_ui, package, - persistent, pretty_print, replay, reset, @@ -262,6 +273,11 @@ pub fn cargo_test_fuzz>(args: &[T]) -> Result<()> { process_deprecated_action_object!(opts, replay, hangs); process_deprecated_action_object!(opts, replay, queue); + if opts.persistent { + eprintln!("`--persistent` is deprecated. Use `--fuzzer aflplusplus-persistent`."); + opts.fuzzer = Some(Fuzzer::AflplusplusPersistent); + } + if let Some(target_name) = opts.target.take() { eprintln!("`--target ` is deprecated. Use just ``."); if opts.ztarget.is_none() { diff --git a/cargo-test-fuzz/tests/auto_concretize.rs b/cargo-test-fuzz/tests/auto_concretize.rs index d7f0a6ee..a86cecf0 100644 --- a/cargo-test-fuzz/tests/auto_concretize.rs +++ b/cargo-test-fuzz/tests/auto_concretize.rs @@ -4,7 +4,7 @@ use internal::{ }; use std::fs::remove_dir_all; use test_log::test; -use testing::examples; +use testing::{examples, CommandExt}; #[test] fn auto_concretize() { @@ -37,7 +37,7 @@ fn test() { examples::test("auto_concretize_0", &test) .unwrap() - .assert() + .logged_assert() .success(); } } @@ -53,7 +53,7 @@ fn test_fuzz() { examples::test_fuzz("auto_concretize_0", &target) .unwrap() .args(["--no-run"]) - .assert() + .logged_assert() .success(); } } diff --git a/cargo-test-fuzz/tests/auto_generate.rs b/cargo-test-fuzz/tests/auto_generate.rs index cb92cfa6..2af222e1 100644 --- a/cargo-test-fuzz/tests/auto_generate.rs +++ b/cargo-test-fuzz/tests/auto_generate.rs @@ -1,11 +1,11 @@ use anyhow::ensure; -use internal::dirs::corpus_directory_from_target; +use internal::{dirs::corpus_directory_from_target, fuzzer, Fuzzer}; use predicates::prelude::*; use std::fs::{read_dir, remove_dir_all}; use test_log::test; -use testing::{examples, retry}; +use testing::{examples, retry, CommandExt}; -const TIMEOUT: &str = "60"; +const MAX_TOTAL_TIME: &str = "60"; // smoelius: It would be nice if these first two tests could distinguish how many "auto_generate" // tests get run (0 vs. 1). But right now, I can't think of an easy way to do this. @@ -40,7 +40,15 @@ fn auto_generate_empty() { )] #[test] fn auto_generate_nonempty() { - auto_generate("assert", "target", true, "Auto-generated", 1); + // smoelius: Libfuzzer exits with 77 when a crash is found. + let fuzzer = fuzzer().unwrap(); + auto_generate( + "assert", + "target", + fuzzer != Fuzzer::Libfuzzer, + "Auto-generated", + 1, + ); } fn auto_generate(krate: &str, target: &str, success: bool, pattern: &str, n: usize) { @@ -56,8 +64,13 @@ fn auto_generate(krate: &str, target: &str, success: bool, pattern: &str, n: usi retry(3, || { let assert = examples::test_fuzz(krate, target) .unwrap() - .args(["--no-ui", "--run-until-crash", "--", "-V", TIMEOUT]) - .assert(); + .args([ + "--no-ui", + "--run-until-crash", + "--max-total-time", + MAX_TOTAL_TIME, + ]) + .logged_assert(); let assert = if success { assert.success() diff --git a/cargo-test-fuzz/tests/breadcrumbs.rs b/cargo-test-fuzz/tests/breadcrumbs.rs new file mode 100644 index 00000000..77dfc4dc --- /dev/null +++ b/cargo-test-fuzz/tests/breadcrumbs.rs @@ -0,0 +1,87 @@ +use internal::{dirs::corpus_directory_from_target, fuzzer, Fuzzer}; +use predicates::prelude::*; +use std::{ + fs::{read_dir, remove_dir_all}, + io::{stderr, Write}, +}; +use test_log::test; +use testing::{examples, CommandExt}; + +const MAX_TOTAL_TIME: &str = "60"; + +#[test] +fn breadcrumbs(krate: &str, target: &str, fuzz_args: &[&str], pattern: &str) { + const QWERTY: &str = "qwerty"; + + if fuzzer().unwrap() == Fuzzer::Libfuzzer { + #[allow(clippy::explicit_write)] + writeln!(stderr(), "Skipping `breadcrumbs` test for libfuzzer").unwrap(); + return; + } + + let corpus = corpus_directory_from_target("qwerty_stepped", "target"); + + remove_dir_all(&corpus).unwrap_or_default(); + + examples::test("qwerty_stepped", "test") + .unwrap() + .logged_assert() + .success(); + + assert_eq!(read_dir(&corpus).unwrap().count(), 1); + + for i in 1..=QWERTY.len() { + let mut fuzz_flags = Vec::new(); + for j in i..QWERTY.len() { + fuzz_flags.push(format!("--features=__qwerty_{}", &QWERTY[j..=j])); + } + + // smoelius: A previous iteration could have found more than just one "qwerty" letter. + if replay(&fuzz_flags) { + continue; + } + + examples::test_fuzz("qwerty_stepped", "target") + .unwrap() + .args([ + "--exit-code", + "--run-until-crash", + "--max-total-time", + MAX_TOTAL_TIME, + ]) + .args(&fuzz_flags) + .logged_assert() + .code(1); + + examples::test_fuzz("qwerty_stepped", "target") + .unwrap() + .arg("--consolidate") + .logged_assert() + .success(); + + assert!(replay(&fuzz_flags)); + } + + examples::test_fuzz("qwerty_stepped", "target") + .unwrap() + .args(["--display=corpus"]) + .logged_assert() + .success() + .stdout(predicate::str::contains(r#""qwerty""#)); +} + +fn replay(fuzz_flags: &[String]) -> bool { + let assert = examples::test_fuzz("qwerty_stepped", "target") + .unwrap() + .args(["--replay=corpus"]) + .args(fuzz_flags) + .logged_assert(); + + let assert = assert.success(); + + assert + .try_stdout(predicate::str::contains( + "thread 'target_fuzz::entry' panicked", + )) + .is_ok() +} diff --git a/cargo-test-fuzz/tests/build.rs b/cargo-test-fuzz/tests/build.rs index a98b5612..584d58ba 100644 --- a/cargo-test-fuzz/tests/build.rs +++ b/cargo-test-fuzz/tests/build.rs @@ -1,12 +1,12 @@ use test_log::test; -use testing::examples; +use testing::{examples, CommandExt}; #[test] fn build_no_instrumentation() { examples::test_fuzz_all() .unwrap() .args(["--no-run", "--no-instrumentation"]) - .assert() + .logged_assert() .success(); } @@ -15,15 +15,6 @@ fn build() { examples::test_fuzz_all() .unwrap() .args(["--no-run"]) - .assert() - .success(); -} - -#[test] -fn build_pesistent() { - examples::test_fuzz_all() - .unwrap() - .args(["--no-run", "--persistent"]) - .assert() + .logged_assert() .success(); } diff --git a/cargo-test-fuzz/tests/concretizations.rs b/cargo-test-fuzz/tests/concretizations.rs index 989278eb..13ca464e 100644 --- a/cargo-test-fuzz/tests/concretizations.rs +++ b/cargo-test-fuzz/tests/concretizations.rs @@ -3,7 +3,7 @@ use internal::dirs::{ }; use std::fs::remove_dir_all; use test_log::test; -use testing::examples; +use testing::{examples, CommandExt}; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -72,7 +72,10 @@ fn test(krate: &str, test: &str, target: &str, impl_expected: &[&str], expected: )] remove_dir_all(concretizations).unwrap_or_default(); - examples::test(krate, test).unwrap().assert().success(); + examples::test(krate, test) + .unwrap() + .logged_assert() + .success(); for (option, expected) in &[ ("--display=impl-concretizations", impl_expected), @@ -81,7 +84,7 @@ fn test(krate: &str, test: &str, target: &str, impl_expected: &[&str], expected: let assert = &examples::test_fuzz(krate, target) .unwrap() .args([option]) - .assert() + .logged_assert() .success(); let mut actual = std::str::from_utf8(&assert.get_output().stdout) diff --git a/cargo-test-fuzz/tests/consolidate.rs b/cargo-test-fuzz/tests/consolidate.rs index 53e7cdf2..d4af8e60 100644 --- a/cargo-test-fuzz/tests/consolidate.rs +++ b/cargo-test-fuzz/tests/consolidate.rs @@ -1,13 +1,13 @@ use anyhow::ensure; -use internal::dirs::corpus_directory_from_target; +use internal::{dirs::corpus_directory_from_target, fuzzer, Fuzzer}; use predicates::prelude::*; use std::fs::{read_dir, remove_dir_all}; use test_log::test; -use testing::{examples, retry}; +use testing::{examples, retry, CommandExt}; -const CRASH_TIMEOUT: &str = "60"; +const CRASH_MAX_TOTAL_TIME: &str = "60"; -const HANG_TIMEOUT: &str = "120"; +const HANG_MAX_TOTAL_TIME: &str = "120"; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -18,7 +18,12 @@ fn consolidate_crashes() { consolidate( "assert", "target", - &["--run-until-crash", "--", "-V", CRASH_TIMEOUT], + &[ + "--run-until-crash", + "--max-total-time", + CRASH_MAX_TOTAL_TIME, + ], + 1, "Args { x: true }", ); } @@ -29,15 +34,20 @@ fn consolidate_crashes() { )] #[test] fn consolidate_hangs() { + let fuzzer = fuzzer().unwrap(); consolidate( "parse_duration", "parse", - &["--persistent", "--", "-V", HANG_TIMEOUT], + &["--max-total-time", HANG_MAX_TOTAL_TIME], + match fuzzer { + Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent => 0, + Fuzzer::Libfuzzer => 1, + }, "", ); } -fn consolidate(krate: &str, target: &str, fuzz_args: &[&str], pattern: &str) { +fn consolidate(krate: &str, target: &str, fuzz_args: &[&str], code: i32, pattern: &str) { let corpus = corpus_directory_from_target(krate, target); // smoelius: `corpus` is distinct for all tests. So there is no race here. @@ -47,24 +57,27 @@ fn consolidate(krate: &str, target: &str, fuzz_args: &[&str], pattern: &str) { )] remove_dir_all(&corpus).unwrap_or_default(); - examples::test(krate, "test").unwrap().assert().success(); + examples::test(krate, "test") + .unwrap() + .logged_assert() + .success(); assert_eq!(read_dir(&corpus).unwrap().count(), 1); retry(3, || { - let mut args = vec!["--no-ui"]; + let mut args = vec!["--exit-code", "--no-ui"]; args.extend_from_slice(fuzz_args); examples::test_fuzz(krate, target) .unwrap() .args(args) - .assert() - .success(); + .logged_assert() + .try_code(code)?; examples::test_fuzz(krate, target) .unwrap() .args(["--consolidate"]) - .assert() + .logged_assert() .success(); ensure!(read_dir(&corpus).unwrap().count() > 1); @@ -72,7 +85,7 @@ fn consolidate(krate: &str, target: &str, fuzz_args: &[&str], pattern: &str) { examples::test_fuzz(krate, target) .unwrap() .args(["--display=corpus"]) - .assert() + .logged_assert() .success() .try_stdout(predicate::str::contains(pattern)) .map_err(Into::into) diff --git a/cargo-test-fuzz/tests/display.rs b/cargo-test-fuzz/tests/display.rs index a32f6ded..7048a383 100644 --- a/cargo-test-fuzz/tests/display.rs +++ b/cargo-test-fuzz/tests/display.rs @@ -1,6 +1,6 @@ use predicates::prelude::*; use test_log::test; -use testing::examples; +use testing::{examples, CommandExt}; #[test] fn display_qwerty() { @@ -30,12 +30,15 @@ fn display_debug_hang() { } fn display(krate: &str, test: &str, target: &str, stdout: &str, stderr: &str) { - examples::test(krate, test).unwrap().assert().success(); + examples::test(krate, test) + .unwrap() + .logged_assert() + .success(); examples::test_fuzz(krate, target) .unwrap() .args(["--display=corpus"]) - .assert() + .logged_assert() .success() .stdout(predicate::str::contains(stdout)) .stderr(predicate::str::contains(stderr)); diff --git a/cargo-test-fuzz/tests/fuzz.rs b/cargo-test-fuzz/tests/fuzz.rs index 1efeea5d..953a92f3 100644 --- a/cargo-test-fuzz/tests/fuzz.rs +++ b/cargo-test-fuzz/tests/fuzz.rs @@ -1,10 +1,13 @@ -use internal::dirs::corpus_directory_from_target; +use internal::{dirs::corpus_directory_from_target, fuzzer, Fuzzer}; use predicates::prelude::*; -use std::fs::remove_dir_all; +use std::{ + fs::remove_dir_all, + io::{stderr, Write}, +}; use test_log::test; -use testing::{examples, retry}; +use testing::{examples, retry, CommandExt}; -const TIMEOUT: &str = "60"; +const MAX_TOTAL_TIME: &str = "60"; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -12,7 +15,7 @@ const TIMEOUT: &str = "60"; )] #[test] fn fuzz_assert() { - fuzz("assert", false); + fuzz("assert"); } #[cfg_attr( @@ -21,10 +24,16 @@ fn fuzz_assert() { )] #[test] fn fuzz_qwerty() { - fuzz("qwerty", true); + if fuzzer().unwrap() == Fuzzer::Libfuzzer { + #[allow(clippy::explicit_write)] + writeln!(stderr(), "Skipping `fuzz_qwerty` test for libfuzzer").unwrap(); + return; + } + + fuzz("qwerty"); } -fn fuzz(krate: &str, persistent: bool) { +fn fuzz(krate: &str) { let corpus = corpus_directory_from_target(krate, "target"); // smoelius: `corpus` is distinct for all tests. So there is no race here. @@ -34,18 +43,21 @@ fn fuzz(krate: &str, persistent: bool) { )] remove_dir_all(corpus).unwrap_or_default(); - examples::test(krate, "test").unwrap().assert().success(); + examples::test(krate, "test") + .unwrap() + .logged_assert() + .success(); retry(3, || { let mut command = examples::test_fuzz(krate, "target").unwrap(); let mut args = vec!["--exit-code", "--run-until-crash"]; - if persistent { - args.push("--persistent"); - } - args.extend_from_slice(&["--", "-V", TIMEOUT]); + args.extend_from_slice(&["--max-total-time", MAX_TOTAL_TIME]); - command.args(&args).assert().try_code(predicate::eq(1)) + command + .args(&args) + .logged_assert() + .try_code(predicate::eq(1)) }) .unwrap(); } diff --git a/cargo-test-fuzz/tests/fuzz_generic.rs b/cargo-test-fuzz/tests/fuzz_generic.rs index 047ae8e7..dcd43ca2 100644 --- a/cargo-test-fuzz/tests/fuzz_generic.rs +++ b/cargo-test-fuzz/tests/fuzz_generic.rs @@ -1,11 +1,11 @@ -use internal::{dirs::corpus_directory_from_target, serde_format}; +use internal::{dirs::corpus_directory_from_target, fuzzer, serde_format, Fuzzer}; use lazy_static::lazy_static; use predicates::prelude::*; use std::{fs::remove_dir_all, sync::Mutex}; use test_log::test; -use testing::{examples, retry}; +use testing::{examples, retry, CommandExt}; -const TIMEOUT: &str = "60"; +const MAX_TOTAL_TIME: &str = "60"; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -15,11 +15,17 @@ const TIMEOUT: &str = "60"; fn fuzz_foo_qwerty() { // smoelius: When `bincode` is enabled, `cargo-afl` fails because "the program crashed with one // of the test cases provided." - if serde_format().to_string() == "Bincode" { - fuzz("test_foo_qwerty", 2); - } else { - fuzz("test_foo_qwerty", 1); - }; + let fuzzer = fuzzer().unwrap(); + fuzz( + "test_foo_qwerty", + if matches!(fuzzer, Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent) + && serde_format().to_string() == "Bincode" + { + 2 + } else { + 1 + }, + ); } #[cfg_attr( @@ -48,13 +54,21 @@ fn fuzz(test: &str, code: i32) { )] remove_dir_all(corpus).unwrap_or_default(); - examples::test("generic", test).unwrap().assert().success(); + examples::test("generic", test) + .unwrap() + .logged_assert() + .success(); retry(3, || { examples::test_fuzz("generic", "target") .unwrap() - .args(["--exit-code", "--run-until-crash", "--", "-V", TIMEOUT]) - .assert() + .args([ + "--exit-code", + "--run-until-crash", + "--max-total-time", + MAX_TOTAL_TIME, + ]) + .logged_assert() .try_code(predicate::eq(code)) }) .unwrap(); diff --git a/cargo-test-fuzz/tests/replay.rs b/cargo-test-fuzz/tests/replay.rs index ce685154..f59d1b71 100644 --- a/cargo-test-fuzz/tests/replay.rs +++ b/cargo-test-fuzz/tests/replay.rs @@ -1,17 +1,17 @@ // smoelius: `rlimit` does not work on macOS. #![cfg(not(target_os = "macos"))] -use internal::dirs::corpus_directory_from_target; +use internal::{dirs::corpus_directory_from_target, fuzzer, Fuzzer}; use predicates::prelude::*; use rlimit::Resource; use std::fs::remove_dir_all; use test_log::test; -use testing::{examples, retry}; +use testing::{examples, retry, CommandExt}; // smoelius: MEMORY_LIMIT must be large enough for the build process to complete. const MEMORY_LIMIT: u64 = 1024 * 1024 * 1024; -const TIMEOUT: &str = "240"; +const MAX_TOTAL_TIME: &str = "240"; #[derive(Clone, Copy)] enum Object { @@ -25,15 +25,17 @@ enum Object { )] #[test] fn replay_crashes() { + let fuzzer = fuzzer().unwrap(); + let memory_limit = (MEMORY_LIMIT / 1024).to_string(); + let mut args = vec!["--run-until-crash"]; + if matches!(fuzzer, Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent) { + args.extend_from_slice(&["--", "-m", &memory_limit]); + } replay( "alloc", "target", - &[ - "--run-until-crash", - "--", - "-m", - &format!("{}", MEMORY_LIMIT / 1024), - ], + &args, + 1, Object::Crashes, r"(?m)\bmemory allocation of \d{10,} bytes failed$|\bcapacity overflow\b", ); @@ -46,16 +48,21 @@ fn replay_crashes() { #[allow(clippy::trivial_regex)] #[test] fn replay_hangs() { + let fuzzer = fuzzer().unwrap(); replay( "parse_duration", "parse", - &["--persistent", "--", "-V", TIMEOUT], + &["--max-total-time", MAX_TOTAL_TIME], + match fuzzer { + Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent => 0, + Fuzzer::Libfuzzer => 1, + }, Object::Hangs, r"(?m)\bTimeout$", ); } -fn replay(krate: &str, target: &str, fuzz_args: &[&str], object: Object, re: &str) { +fn replay(krate: &str, target: &str, fuzz_args: &[&str], code: i32, object: Object, re: &str) { let corpus = corpus_directory_from_target(krate, target); // smoelius: `corpus` is distinct for all tests. So there is no race here. @@ -65,23 +72,26 @@ fn replay(krate: &str, target: &str, fuzz_args: &[&str], object: Object, re: &st )] remove_dir_all(corpus).unwrap_or_default(); - examples::test(krate, "test").unwrap().assert().success(); + examples::test(krate, "test") + .unwrap() + .logged_assert() + .success(); examples::test_fuzz(krate, target) .unwrap() .args(["--reset"]) - .assert() + .logged_assert() .success(); retry(3, || { - let mut args = vec!["--no-ui"]; + let mut args = vec!["--exit-code", "--no-ui"]; args.extend_from_slice(fuzz_args); examples::test_fuzz(krate, target) .unwrap() .args(args) - .assert() - .success(); + .logged_assert() + .try_code(code)?; // smoelius: The memory limit must be set to replay the crashes, but not the hangs. Resource::DATA.set(MEMORY_LIMIT, MEMORY_LIMIT).unwrap(); @@ -93,7 +103,7 @@ fn replay(krate: &str, target: &str, fuzz_args: &[&str], object: Object, re: &st Object::Crashes => "--replay=crashes", Object::Hangs => "--replay=hangs", }]) - .assert() + .logged_assert() .success() .try_stdout(predicate::str::is_match(re).unwrap()) }) diff --git a/cargo-test-fuzz/tests/test_log.rs b/cargo-test-fuzz/tests/test_log.rs index 2861dded..cdd8dcb9 100644 --- a/cargo-test-fuzz/tests/test_log.rs +++ b/cargo-test-fuzz/tests/test_log.rs @@ -7,14 +7,20 @@ use test_log::test; #[test] fn all_tests_use_test_log() { - let tests = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"); - - for entry in read_dir(tests).unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - let file = File::open(path).unwrap(); - assert!(BufReader::new(file) - .lines() - .any(|line| { line.unwrap() == "use test_log::test;" })); + for tests in [ + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"), + Path::new(env!("CARGO_MANIFEST_DIR")).join("../test-fuzz/tests"), + ] { + for entry in read_dir(tests).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let file = File::open(&path).unwrap(); + assert!( + BufReader::new(file) + .lines() + .any(|line| { line.unwrap() == "use test_log::test;" }), + "failed for {path:?}" + ); + } } } diff --git a/cargo-test-fuzz/tests/third_party.rs b/cargo-test-fuzz/tests/third_party.rs index 7983a5a4..c14527e0 100644 --- a/cargo-test-fuzz/tests/third_party.rs +++ b/cargo-test-fuzz/tests/third_party.rs @@ -1,6 +1,7 @@ use assert_cmd::Command; use bitflags::bitflags; use cargo_metadata::MetadataCommand; +use internal::{fuzzer, Fuzzer}; use lazy_static::lazy_static; use option_set::option_set; use predicates::prelude::*; @@ -14,6 +15,7 @@ use std::{ }; use tempfile::tempdir_in; use test_log::test; +use testing::CommandExt; option_set! { #[derive(Copy, Clone, Eq, PartialEq)] @@ -21,7 +23,8 @@ option_set! { const EXPENSIVE = 1 << 0; const SKIP = 1 << 1; const SKIP_NIGHTLY = 1 << 2; - const REQUIRES_ISOLATION = 1 << 3; + const SKIP_LIBFUZZER = 1 << 3; + const REQUIRES_ISOLATION = 1 << 4; } } @@ -51,6 +54,7 @@ mod cheap_tests { #[test] fn test() { let version_meta = version_meta().unwrap(); + let fuzzer = fuzzer().unwrap(); for test in TESTS.iter() { run_test( module_path!(), @@ -58,7 +62,8 @@ mod cheap_tests { test.flags.contains(Flags::EXPENSIVE) || test.flags.contains(Flags::SKIP) || (test.flags.contains(Flags::SKIP_NIGHTLY) - && version_meta.channel == Channel::Nightly), + && version_meta.channel == Channel::Nightly) + || (test.flags.contains(Flags::SKIP_LIBFUZZER) && fuzzer == Fuzzer::Libfuzzer), ); } } @@ -70,13 +75,15 @@ mod all_tests { #[ignore] fn test() { let version_meta = version_meta().unwrap(); + let fuzzer = fuzzer().unwrap(); for test in TESTS.iter() { run_test( module_path!(), test, test.flags.contains(Flags::SKIP) || (test.flags.contains(Flags::SKIP_NIGHTLY) - && version_meta.channel == Channel::Nightly), + && version_meta.channel == Channel::Nightly) + || (test.flags.contains(Flags::SKIP_LIBFUZZER) && fuzzer == Fuzzer::Libfuzzer), ); } } @@ -103,7 +110,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { Command::new("git") .current_dir(&tempdir) .args(["clone", &test.url, "."]) - .assert() + .logged_assert() .success(); #[allow(unknown_lints)] @@ -117,7 +124,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { Command::new("git") .current_dir(&tempdir) .args(["apply", &patch.to_string_lossy()]) - .assert() + .logged_assert() .success(); let subdir = tempdir.path().join(&test.subdir); @@ -141,7 +148,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { Command::new("cargo") .current_dir(&subdir) .args(["update"]) - .assert() + .logged_assert() .success(); // smoelius: The `libp2p-swarm-derive` issue appears to have been resolved. @@ -151,8 +158,11 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { command .current_dir(&subdir) .args(["update", "-p", "libp2p-swarm-derive"]); - if command.assert().try_success().is_ok() { - command.args(["--precise", "0.30.1"]).assert().success(); + if command.logged_assert().try_success().is_ok() { + command + .args(["--precise", "0.30.1"]) + .logged_assert() + .success(); } } @@ -165,7 +175,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { Command::new("cargo") .current_dir(&subdir) .args(["test", "--package", &test.package, "--", "--nocapture"]) - .assert() + .logged_assert() .success(); for target in &test.targets { @@ -179,7 +189,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { "--display=corpus", target, ]) - .assert() + .logged_assert() .success() .stdout(predicate::str::is_match(r#"(?m)^[[:xdigit:]]{40}:"#).unwrap()); @@ -193,7 +203,7 @@ fn run_test(module_path: &str, test: &Test, no_run: bool) { "--replay=corpus", target, ]) - .assert() + .logged_assert() .success() .stdout( predicate::str::is_match(r#"(?m)^[[:xdigit:]]{40}: Ret\((Ok|Err)\(.*\)\)$"#) @@ -212,7 +222,7 @@ fn check_test_fuzz_dependency(subdir: &Path, test_package: &str) { .packages .iter() .find(|package| package.name == test_package) - .unwrap_or_else(|| panic!("Could not find package `{}`", test_package)); + .unwrap_or_else(|| panic!("Could not find package `{test_package}`",)); let dep = package .dependencies .iter() @@ -231,7 +241,7 @@ fn patches_are_current() { Command::new("git") .current_dir(&tempdir) .args(["clone", "--depth=1", &test.url, "."]) - .assert() + .logged_assert() .success(); let patch_path = Path::new(env!("CARGO_MANIFEST_DIR")) @@ -243,7 +253,7 @@ fn patches_are_current() { .current_dir(&tempdir) .args(["apply"]) .write_stdin(patch.as_bytes()) - .assert() + .logged_assert() .success(); // smoelius: The following checks are *not* redundant. They can fail even if the patch @@ -252,7 +262,7 @@ fn patches_are_current() { let assert = Command::new("git") .current_dir(&tempdir) .args(["diff", &format!("--unified={LINES_OF_CONTEXT}")]) - .assert() + .logged_assert() .success(); let diff = String::from_utf8_lossy(&assert.get_output().stdout); diff --git a/cargo-test-fuzz/third_party.json b/cargo-test-fuzz/third_party.json index f42325b4..3bb7fd30 100644 --- a/cargo-test-fuzz/third_party.json +++ b/cargo-test-fuzz/third_party.json @@ -1,6 +1,6 @@ [ { - "flags": [], + "flags": ["SKIP_LIBFUZZER"], "url": "https://github.com/CosmWasm/cw-plus", "patch": "cw-plus.patch", "subdir": ".", @@ -16,7 +16,7 @@ "targets": ["import"] }, { - "flags": ["REQUIRES_ISOLATION"], + "flags": ["SKIP_LIBFUZZER", "REQUIRES_ISOLATION"], "url": "https://github.com/solana-labs/example-helloworld", "patch": "example-helloworld.patch", "subdir": "src/program-rust", diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..e201f94c --- /dev/null +++ b/clippy.toml @@ -0,0 +1,3 @@ +disallowed-methods = [ + { path = "assert_cmd::cmd::Command::assert", reason = "use `test_fuzz_testing::CommandExt::logged_assert`" }, +] diff --git a/docs/crates.dot b/docs/crates.dot index 078da079..b6ca3100 100644 --- a/docs/crates.dot +++ b/docs/crates.dot @@ -1,13 +1,13 @@ digraph { "cargo-test-fuzz" -> "internal" "cargo-test-fuzz" -> "test-fuzz" - "cargo-test-fuzz" -> "testing" + "cargo-test-fuzz" -> "testing\n(dev)" "macro" -> "internal" "macro-generated-code" -> "test-fuzz" "runtime" -> "internal" "test-fuzz" -> "internal" "test-fuzz" -> "macro" "test-fuzz" -> "runtime" - "test-fuzz" -> "testing" - "testing" -> "internal" + "test-fuzz" -> "testing\n(dev)" + "testing\n(dev)" -> "internal" } diff --git a/docs/libfuzzer_notes.md b/docs/libfuzzer_notes.md new file mode 100644 index 00000000..bf30427d --- /dev/null +++ b/docs/libfuzzer_notes.md @@ -0,0 +1,30 @@ +# Libfuzzer integration + +## `libtest` calls `libfuzzer`: why? + +`libfuzzer` and `libtest` both want to provide a `main` function. Which one should we use? + +In each case, we get a kind of "callback" from the `main` function. This gives us an opportunity to call the other `main` function. So the question becomes: should `libfuzzer` call `libtest`, or should `libtest` call `libfuzzer`? I.e., + +``` +libfuzzer --> libtest +``` + +or + +``` +libtest --> libfuzzer +``` + +We have chosen the latter (i.e., `libtest` calls `libfuzzer`) because doing the former (i.e., `libfuzzer` calls `libtest`) would be difficult: + +1. `test-fuzz` relies heavily on `libtest`'s command line option parsing. If `libfuzzer` called `libtest`, we would need to find an alternative way to pass options to `libtest`. Moreover, this alternative way would ony apply when `libfuzzer` was selected as the fuzzer, so it would need to be turned on and off. In short: it would be a hassle. +2. The code that `libfuzzer`'s `main` function calls is called repeatedly. If `libfuzzer`'s main function called `libtest`'s main function, the calls would happen repeatedly. + +## Implementation + +### `LIBFUZZER_FUZZ_TARGET` + +`libfuzzer` expects there to be just one fuzz target (`rust_fuzzer_test_input`) which `libfuzzer` calls from its `main` function. But `test-fuzz` allows a binary to contain multiple fuzz targets. So we need a way to tell `libfuzzer` which one to call. + +Our solution is to use a global atomic pointer, `LIBFUZZER_FUZZ_TARGET`. This pointer is set from within the macro generated code after `libtest` is called, but before control is handed to `libfuzzer`. Our `rust_fuzzer_test_input` fuzz target reads that pointer and (unsafely) calls the pointed-to function. diff --git a/examples/Cargo.toml b/examples/Cargo.toml index f69f52af..6f96cb74 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz-examples" version = "3.0.5" -edition = "2018" +edition = "2021" publish = false [[bin]] @@ -21,3 +21,9 @@ serde_json = "1.0" __auto_concretize = ["test-fuzz/auto_concretize"] __bar_fuzz = [] __inapplicable_conversion = [] +__qwerty_q = [] +__qwerty_w = [] +__qwerty_e = [] +__qwerty_r = [] +__qwerty_t = [] +__qwerty_y = [] diff --git a/examples/tests/qwerty_stepped.rs b/examples/tests/qwerty_stepped.rs new file mode 100644 index 00000000..a02497f8 --- /dev/null +++ b/examples/tests/qwerty_stepped.rs @@ -0,0 +1,17 @@ +#[test_fuzz::test_fuzz] +fn target(data: &str) { + assert!( + !(data.len() == 6 + && (cfg!(feature = "__qwerty_q") || data.as_bytes()[0] == b'q') + && (cfg!(feature = "__qwerty_w") || data.as_bytes()[1] == b'w') + && (cfg!(feature = "__qwerty_e") || data.as_bytes()[2] == b'e') + && (cfg!(feature = "__qwerty_r") || data.as_bytes()[3] == b'r') + && (cfg!(feature = "__qwerty_t") || data.as_bytes()[4] == b't') + && (cfg!(feature = "__qwerty_y") || data.as_bytes()[5] == b'y')) + ); +} + +#[test] +fn test() { + target("asdfgh"); +} diff --git a/internal/Cargo.toml b/internal/Cargo.toml index 434a2739..a6bc5bdd 100644 --- a/internal/Cargo.toml +++ b/internal/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz-internal" version = "3.0.5" -edition = "2018" +edition = "2021" description = "test-fuzz-internal" @@ -10,10 +10,15 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/trailofbits/test-fuzz" [dependencies] +anyhow = "1.0" cargo_metadata = "0.15" +clap = { version = "4.1", features = ["derive"] } +heck = "0.4" proc-macro2 = "1.0" quote = "1.0" +remain = "0.2" serde = "1.0" +strum = "0.24" strum_macros = "0.24" [features] diff --git a/internal/src/dirs.rs b/internal/src/dirs.rs index 09667976..a6d759a7 100644 --- a/internal/src/dirs.rs +++ b/internal/src/dirs.rs @@ -1,4 +1,6 @@ +use super::Fuzzer; use cargo_metadata::MetadataCommand; +use heck::ToSnakeCase; use std::{any::type_name, env, path::PathBuf}; #[must_use] @@ -32,54 +34,63 @@ pub fn corpus_directory_from_target(krate: &str, target: &str) -> PathBuf { } #[must_use] -pub fn crashes_directory_from_target(krate: &str, target: &str) -> PathBuf { - output_directory_from_target(krate, target).join("default/crashes") +pub fn crashes_directory_from_target(fuzzer: Fuzzer, krate: &str, target: &str) -> PathBuf { + output_directory_from_target(fuzzer, krate, target).join(match fuzzer { + Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent => "default/crashes", + Fuzzer::Libfuzzer => "artifacts", + }) } #[must_use] -pub fn hangs_directory_from_target(krate: &str, target: &str) -> PathBuf { - output_directory_from_target(krate, target).join("default/hangs") +pub fn hangs_directory_from_target(fuzzer: Fuzzer, krate: &str, target: &str) -> PathBuf { + output_directory_from_target(fuzzer, krate, target).join(match fuzzer { + Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent => "default/hangs", + Fuzzer::Libfuzzer => "artifacts", + }) } #[must_use] -pub fn queue_directory_from_target(krate: &str, target: &str) -> PathBuf { - output_directory_from_target(krate, target).join("default/queue") +pub fn queue_directory_from_target(fuzzer: Fuzzer, krate: &str, target: &str) -> PathBuf { + output_directory_from_target(fuzzer, krate, target).join(match fuzzer { + Fuzzer::Aflplusplus | Fuzzer::AflplusplusPersistent => "default/queue", + Fuzzer::Libfuzzer => "queue", + }) } #[must_use] -pub fn output_directory_from_target(krate: &str, target: &str) -> PathBuf { - output_directory().join(path_from_target(krate, target)) +pub fn output_directory_from_target(fuzzer: Fuzzer, krate: &str, target: &str) -> PathBuf { + output_directory(fuzzer).join(path_from_target(krate, target)) } #[must_use] fn impl_concretizations_directory() -> PathBuf { - target_directory(false).join("impl_concretizations") + target_directory(None).join("impl_concretizations") } #[must_use] fn concretizations_directory() -> PathBuf { - target_directory(false).join("concretizations") + target_directory(None).join("concretizations") } #[must_use] fn corpus_directory() -> PathBuf { - target_directory(false).join("corpus") + target_directory(None).join("corpus") } #[must_use] -fn output_directory() -> PathBuf { - target_directory(true).join("output") +fn output_directory(fuzzer: Fuzzer) -> PathBuf { + target_directory(Some(fuzzer)).join("output") } #[must_use] -pub fn target_directory(instrumented: bool) -> PathBuf { +pub fn target_directory(fuzzer: Option) -> PathBuf { let mut command = MetadataCommand::new(); if let Ok(path) = env::var("TEST_FUZZ_MANIFEST_PATH") { command.manifest_path(path); } let mut target_dir = command.no_deps().exec().unwrap().target_directory; - if instrumented { - target_dir = target_dir.join("afl"); + if let Some(fuzzer) = fuzzer { + target_dir = target_dir.join(fuzzer.to_string().to_snake_case()); } target_dir.into() } @@ -89,7 +100,7 @@ fn path_from_args_type() -> String { let type_name = type_name::(); let n = type_name .find("_fuzz") - .unwrap_or_else(|| panic!("unexpected type name: `{}`", type_name)); + .unwrap_or_else(|| panic!("unexpected type name: `{type_name}`")); type_name[..n].to_owned() } diff --git a/internal/src/fuzzer.rs b/internal/src/fuzzer.rs new file mode 100644 index 00000000..aea00274 --- /dev/null +++ b/internal/src/fuzzer.rs @@ -0,0 +1,42 @@ +use anyhow::{anyhow, Result}; +use clap::ValueEnum; +use heck::ToSnakeCase; +use serde::{Deserialize, Serialize}; +use std::env::var; +use strum_macros::{Display, EnumIter}; + +// smoelius: The user selects the fuzzer via an environment variable (`TEST_FUZZ_FUZZER`) or a +// command line option. Hence, cargo-test-fuzz needs to know all available fuzzers. This is unlike +// the Serde format, which the user selects via a Cargo feature. +#[derive( + Clone, Copy, Debug, Display, Deserialize, EnumIter, PartialEq, Eq, Serialize, ValueEnum, +)] +#[remain::sorted] +pub enum Fuzzer { + Aflplusplus, + AflplusplusPersistent, + Libfuzzer, +} + +const DEFAULT_FUZZER: Fuzzer = Fuzzer::Aflplusplus; + +pub fn fuzzer() -> Result { + match var("TEST_FUZZ_FUZZER") { + Ok(value) => { + let fuzzer = Fuzzer::from_str(&value, false).map_err(|error| { + anyhow!( + "`TEST_FUZZ_FUZZER` is set to {value:?}, which could not be parsed: {error}" + ) + })?; + Ok(fuzzer) + } + Err(_) => Ok(DEFAULT_FUZZER), + } +} + +impl Fuzzer { + #[must_use] + pub fn as_feature(self) -> String { + String::from("__fuzzer_") + &self.to_string().to_snake_case() + } +} diff --git a/internal/src/lib.rs b/internal/src/lib.rs index dd36dc66..1aa85164 100644 --- a/internal/src/lib.rs +++ b/internal/src/lib.rs @@ -3,5 +3,8 @@ pub use auto_concretize::enabled as auto_concretize_enabled; pub mod dirs; +mod fuzzer; +pub use fuzzer::*; + mod serde_format; pub use serde_format::*; diff --git a/macro/Cargo.toml b/macro/Cargo.toml index c7ef1e0b..4feb2b05 100644 --- a/macro/Cargo.toml +++ b/macro/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz-macro" version = "3.0.5" -edition = "2018" +edition = "2021" description = "test-fuzz-macro" @@ -27,7 +27,9 @@ internal = { path = "../internal", package = "test-fuzz-internal", version = "=3 [features] __auto_concretize = [] -__persistent = [] +__fuzzer_aflplusplus = [] +__fuzzer_aflplusplus_persistent = [] +__fuzzer_libfuzzer = [] __serde_bincode = ["internal/__serde_bincode"] __serde_cbor = ["internal/__serde_cbor"] __serde_cbor4ii = ["internal/__serde_cbor4ii"] diff --git a/macro/build.rs b/macro/build.rs index f67c8e01..b92de738 100644 --- a/macro/build.rs +++ b/macro/build.rs @@ -1,4 +1,11 @@ fn main() { + #[cfg(not(any( + feature = "__fuzzer_aflplusplus", + feature = "__fuzzer_aflplusplus_persistent", + feature = "__fuzzer_libfuzzer", + )))] + println!("cargo:rustc-cfg=fuzzer_default"); + #[cfg(not(any( feature = "__serde_bincode", feature = "__serde_cbor", diff --git a/macro/src/auto_concretize.rs b/macro/src/auto_concretize.rs index 9a5cd97c..85f9a6f2 100644 --- a/macro/src/auto_concretize.rs +++ b/macro/src/auto_concretize.rs @@ -112,7 +112,7 @@ mod functions { || Err(Kind::None), |path| { Ok(read_to_string(&path) - .unwrap_or_else(|_| panic!("`read_to_string` failed for `{:?}`", path))) + .unwrap_or_else(|_| panic!("`read_to_string` failed for `{path:?}`"))) }, ) } diff --git a/macro/src/fuzzer.rs b/macro/src/fuzzer.rs new file mode 100644 index 00000000..a110cfcd --- /dev/null +++ b/macro/src/fuzzer.rs @@ -0,0 +1,145 @@ +use super::Components; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; + +// smoelius: Do not set the panic hook when replaying. Leave cargo test's +// panic hook in place. +#[cfg(any(fuzzer_default, feature = "__fuzzer_aflplusplus"))] +pub(crate) fn args_entry_stmts(components: &Components) -> TokenStream2 { + let Components { + set_panic_hook, + take_panic_hook, + input_args, + output_args, + args_ret_ty, + output_ret, + call_in_environment, + .. + } = components; + + let call_in_environment_with_deserialized_arguments = quote! { + let ret: Option< #args_ret_ty > = args.map(|mut args| + #call_in_environment + ); + }; + + quote! { + if test_fuzz::runtime::display_enabled() + || test_fuzz::runtime::replay_enabled() + { + #input_args + if test_fuzz::runtime::display_enabled() { + #output_args + } + if test_fuzz::runtime::replay_enabled() { + #call_in_environment_with_deserialized_arguments + #output_ret + } + } else { + #set_panic_hook + #input_args + #call_in_environment_with_deserialized_arguments + #take_panic_hook + } + } +} + +#[cfg(feature = "__fuzzer_aflplusplus_persistent")] +pub(crate) fn args_entry_stmts(components: &Components) -> TokenStream2 { + let Components { + combined_concretization, + set_panic_hook, + take_panic_hook, + args_ret_ty, + call_in_environment, + .. + } = components; + + quote! { + if test_fuzz::runtime::display_enabled() + || test_fuzz::runtime::replay_enabled() + { + panic!("Displaying/replaying with `aflplusplus-persistent` is not supported."); + } else { + #set_panic_hook + + test_fuzz::afl::fuzz!(|data: &[u8]| { + let mut args = UsingReader::<_>::read_args #combined_concretization (data); + let ret: Option< #args_ret_ty > = args.map(|mut args| + #call_in_environment + ); + }); + + #take_panic_hook + } + } +} + +#[cfg(feature = "__fuzzer_libfuzzer")] +pub(crate) fn args_entry_stmts(components: &Components) -> TokenStream2 { + let Components { + combined_concretization, + output_args, + args_ret_ty, + output_ret, + call_in_environment, + .. + } = components; + + quote! { + static RET: std::sync::Mutex> = std::sync::Mutex::new(None); + + fn libfuzzer_fuzz_target(data: &[u8]) { + let mut args = UsingReader::<_>::read_args #combined_concretization (data); + if let Some(ret) = args.map(|mut args| + #call_in_environment + ) { + let mut lock = RET.lock().unwrap(); + *lock = Some(ret); + } + } + + let libfuzzer_fuzz_target_ptr = libfuzzer_fuzz_target as fn(&[u8]); + test_fuzz::runtime::libfuzzer::LIBFUZZER_FUZZ_TARGET.store( + unsafe { std::mem::transmute::<_, *mut ()>(libfuzzer_fuzz_target_ptr) }, + std::sync::atomic::Ordering::SeqCst + ); + + let mut code = 0; + + if test_fuzz::runtime::display_enabled() + || test_fuzz::runtime::replay_enabled() + { + let data = { + use std::io::Read; + let mut data = Vec::new(); + std::io::stdin().read_to_end(&mut data).expect("Could not read from `stdin`"); + data + }; + + if test_fuzz::runtime::display_enabled() { + let args = UsingReader::<_>::read_args #combined_concretization (data.as_slice()); + #output_args + } + + // smoelius: If replaying with instrumentation, call `rust_fuzzer_test_input` directly. + if test_fuzz::runtime::replay_enabled() { + extern "C" { + #[allow(improper_ctypes)] + fn rust_fuzzer_test_input(input: &[u8]) -> i32; + } + code = unsafe { rust_fuzzer_test_input(&data) }; + let mut lock = RET.lock().unwrap(); + let ret: Option< #args_ret_ty > = lock.take(); + #output_ret + } + } else { + // smoelius: Don't set the panic hook when running under libfuzzer. + code = test_fuzz::runtime::libfuzzer::libfuzzer_main(); + } + + if code != 0 { + std::process::exit(code); + } + } +} diff --git a/macro/src/lib.rs b/macro/src/lib.rs index 6d7865e4..bf27bd96 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -27,6 +27,9 @@ use unzip_n::unzip_n; mod auto_concretize; +mod fuzzer; +use fuzzer::args_entry_stmts; + #[cfg(feature = "__auto_concretize")] mod mod_utils; @@ -39,6 +42,18 @@ mod type_utils; type Conversions = BTreeMap; +#[allow(dead_code)] +struct Components<'a> { + combined_concretization: &'a Option, + set_panic_hook: &'a TokenStream2, + take_panic_hook: &'a TokenStream2, + input_args: &'a TokenStream2, + output_args: &'a TokenStream2, + args_ret_ty: &'a Type, + output_ret: &'a TokenStream2, + call_in_environment: &'a Expr, +} + lazy_static! { static ref CARGO_CRATE_NAME: String = var("CARGO_CRATE_NAME").expect("Could not get `CARGO_CRATE_NAME`"); @@ -121,8 +136,10 @@ fn map_impl_item( } } -// smoelius: This function is slightly misnamed. The mapped item could actually be an associated +// smoelius: `map_method` is slightly misnamed. The mapped item could actually be an associated // function. I am keeping this name to be consistent with `ImplItem::Method`. +// smoelius: `syn` 2.0 renamed `ImplItem::Method` to `ImplItem::Fn`. We should rename this function +// when we upgrade. fn map_method( generics: &Generics, trait_path: &Option, @@ -505,28 +522,26 @@ fn map_method_or_fn( } } }; - let input_args = { - #[cfg(feature = "__persistent")] - quote! {} - #[cfg(not(feature = "__persistent"))] + let (set_panic_hook, take_panic_hook) = ( quote! { - let mut args = UsingReader::<_>::read_args #combined_concretization (std::io::stdin()); - } - }; - let output_args = { - #[cfg(feature = "__persistent")] - quote! {} - #[cfg(not(feature = "__persistent"))] + std::panic::set_hook(std::boxed::Box::new(|_| std::process::abort())); + }, quote! { - args.as_ref().map(|x| { - if test_fuzz::runtime::pretty_print_enabled() { - eprint!("{:#?}", x); - } else { - eprint!("{:?}", x); - }; - }); - eprintln!(); - } + let _ = std::panic::take_hook(); + }, + ); + let input_args = quote! { + let mut args = UsingReader::<_>::read_args #combined_concretization (std::io::stdin()); + }; + let output_args = quote! { + args.as_ref().map(|x| { + if test_fuzz::runtime::pretty_print_enabled() { + eprint!("{:#?}", x); + } else { + eprint!("{:?}", x); + }; + }); + eprintln!(); }; let args_ret_ty: Type = parse_quote! { ::RetTy @@ -562,53 +577,38 @@ fn map_method_or_fn( } else { call }; - let call_in_environment_with_deserialized_arguments = { - #[cfg(feature = "__persistent")] - quote! { - test_fuzz::afl::fuzz!(|data: &[u8]| { - let mut args = UsingReader::<_>::read_args #combined_concretization (data); - let ret: Option< #args_ret_ty > = args.map(|mut args| - #call_in_environment - ); - }); - } - #[cfg(not(feature = "__persistent"))] - quote! { - let ret: Option< #args_ret_ty > = args.map(|mut args| - #call_in_environment - ); - } - }; - let output_ret = { - #[cfg(feature = "__persistent")] - quote! { - // smoelius: Suppress unused variable warning. - let _: Option< #args_ret_ty > = None; - } - #[cfg(not(feature = "__persistent"))] - quote! { - struct Ret( #args_ret_ty ); - impl std::fmt::Debug for Ret { - fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use test_fuzz::runtime::TryDebugFallback; - let mut debug_tuple = fmt.debug_tuple("Ret"); - test_fuzz::runtime::TryDebug(&self.0).apply(&mut |value| { - debug_tuple.field(value); - }); - debug_tuple.finish() - } + let output_ret = quote! { + struct Ret( #args_ret_ty ); + impl std::fmt::Debug for Ret { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use test_fuzz::runtime::TryDebugFallback; + let mut debug_tuple = fmt.debug_tuple("Ret"); + test_fuzz::runtime::TryDebug(&self.0).apply(&mut |value| { + debug_tuple.field(value); + }); + debug_tuple.finish() } - let ret = ret.map(Ret); - ret.map(|x| { - if test_fuzz::runtime::pretty_print_enabled() { - eprint!("{:#?}", x); - } else { - eprint!("{:?}", x); - }; - }); - eprintln!(); } + let ret = ret.map(Ret); + ret.map(|x| { + if test_fuzz::runtime::pretty_print_enabled() { + eprint!("{:#?}", x); + } else { + eprint!("{:?}", x); + }; + }); + eprintln!(); }; + let args_entry_stmts = args_entry_stmts(&Components { + combined_concretization: &combined_concretization, + set_panic_hook: &set_panic_hook, + take_panic_hook: &take_panic_hook, + input_args: &input_args, + output_args: &output_args, + output_ret: &output_ret, + args_ret_ty: &args_ret_ty, + call_in_environment: &call_in_environment, + }); let mod_items = if opts.only_concretizations { quote! {} } else { @@ -688,26 +688,8 @@ fn map_method_or_fn( } fn entry() { - // smoelius: Do not set the panic hook when replaying. Leave cargo test's - // panic hook in place. if test_fuzz::runtime::test_fuzz_enabled() { - if test_fuzz::runtime::display_enabled() - || test_fuzz::runtime::replay_enabled() - { - #input_args - if test_fuzz::runtime::display_enabled() { - #output_args - } - if test_fuzz::runtime::replay_enabled() { - #call_in_environment_with_deserialized_arguments - #output_ret - } - } else { - std::panic::set_hook(std::boxed::Box::new(|_| std::process::abort())); - #input_args - #call_in_environment_with_deserialized_arguments - let _ = std::panic::take_hook(); - } + #args_entry_stmts } } } @@ -1034,7 +1016,10 @@ fn args_as_turbofish(args: &Punctuated) -> TokenS } // smoelius: The current strategy for combining auto-generated values is a kind of "round robin." -// The strategy ensures that each auto-generated value gets into at least one `Arg` value. +// The strategy ensures that each auto-generated value gets into at least one `Args` value. +// smoelius: One problem with the current approach is that it increments `Args` fields in lockstep. +// So for any two fields with the same number of values, if value x appears alongside value y, then +// whenever x appears, it appears alongside y (and vice versa). fn args_from_autos(autos: &[Expr]) -> Expr { let lens: Vec = (0..autos.len()) .map(|i| { diff --git a/macro/src/mod_utils.rs b/macro/src/mod_utils.rs index 7b372f9d..838ca356 100644 --- a/macro/src/mod_utils.rs +++ b/macro/src/mod_utils.rs @@ -41,10 +41,10 @@ fn contains(left: Span, right: Span) -> bool { pub fn module_path(span: Span) -> Vec { let source = span.source_file(); let path = source.path(); - let contents = read_to_string(&path) - .unwrap_or_else(|_| panic!("`read_to_string` failed for `{:?}`", path)); + let contents = + read_to_string(&path).unwrap_or_else(|_| panic!("`read_to_string` failed for `{path:?}`")); let file: File = - parse_str(&contents).unwrap_or_else(|_| panic!("Could not parse `{:?}` contents", source)); + parse_str(&contents).unwrap_or_else(|_| panic!("Could not parse `{source:?}` contents")); let mut visitor = ModVisitor { target: span, stack: Vec::new(), diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 285bd4b2..fe71c5aa 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz-runtime" version = "3.0.5" -edition = "2018" +edition = "2021" description = "test-fuzz-runtime" @@ -18,14 +18,21 @@ repository = "https://github.com/trailofbits/test-fuzz" bincode = "1.3" cbor4ii = { version = "0.3", features = ["serde1", "use_std"], optional = true } hex = "0.4" +libc = { version = "0.2", optional = true } +libfuzzer-sys = { version = "0.4", optional = true } +nix = { version = "0.26", optional = true } num-traits = "0.2" serde = { version = "1.0", features = ["derive"] } serde_cbor = { version = "0.11", optional = true } +serde_json = { version = "1.0", optional = true } sha-1 = "0.10" internal = { path = "../internal", package = "test-fuzz-internal", version = "=3.0.5" } [features] +__fuzzer_aflplusplus = [] +__fuzzer_aflplusplus_persistent = [] +__fuzzer_libfuzzer = ["libc", "libfuzzer-sys", "nix", "serde_json"] __serde_bincode = [] __serde_cbor = ["serde_cbor"] __serde_cbor4ii = ["cbor4ii"] diff --git a/runtime/build.rs b/runtime/build.rs index f67c8e01..b92de738 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,4 +1,11 @@ fn main() { + #[cfg(not(any( + feature = "__fuzzer_aflplusplus", + feature = "__fuzzer_aflplusplus_persistent", + feature = "__fuzzer_libfuzzer", + )))] + println!("cargo:rustc-cfg=fuzzer_default"); + #[cfg(not(any( feature = "__serde_bincode", feature = "__serde_cbor", diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 436e9206..45141ebe 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -21,8 +21,25 @@ pub use num_traits; pub mod traits; +#[cfg(feature = "__fuzzer_libfuzzer")] +pub mod libfuzzer; + #[cfg(any(serde_default, feature = "__serde_bincode"))] -const BYTE_LIMIT: u64 = 1024 * 1024 * 1024; +const BYTE_LIMIT: u64 = { + #[cfg(any( + fuzzer_default, + feature = "__fuzzer_aflplusplus", + feature = "__fuzzer_aflplusplus_persistent" + ))] + { + 1024 * 1024 * 1024 + } + // smoelius: With a large byte limit, runs often timeout under libfuzzer. + #[cfg(feature = "__fuzzer_libfuzzer")] + { + 32 * 1024 + } +}; // smoelius: TryDebug, etc. use Nikolai Vazquez's trick from `impls`. // https://github.com/nvzqz/impls#how-it-works diff --git a/runtime/src/libfuzzer.rs b/runtime/src/libfuzzer.rs new file mode 100644 index 00000000..345b49c9 --- /dev/null +++ b/runtime/src/libfuzzer.rs @@ -0,0 +1,67 @@ +use libc::{c_char, c_int}; +use libfuzzer_sys::fuzz_target; +use nix::{ + sys::wait::{waitpid, WaitStatus}, + unistd::{fork, ForkResult}, +}; +use serde_json; +use std::{ + ffi::CString, + sync::atomic::{AtomicPtr, Ordering}, +}; + +extern "C" { + #[allow(improper_ctypes)] + fn LLVMFuzzerRunDriver( + argc: *const c_int, + argv: *const *const *const c_char, + callback: fn(data: *const u8, size: usize) -> i32, + ) -> i32; +} + +pub fn libfuzzer_main() -> i32 { + // Running libfuzzer in a test interferes with its timeout handling. libtest runs each test in + // its own thread. But libfuzzer expects `SIGALRM` signals to be handled by a specific thread: + // https://github.com/rust-fuzz/libfuzzer/blob/03a00b2c2ab7b82838536883e0b66eae59ab130d/libfuzzer/FuzzerLoop.cpp#L280 + // To work around this, run libfuzzer in a new child process. Thanks to @maxammann for this very + // good idea. + if let ForkResult::Parent { child } = unsafe { fork() }.unwrap() { + match waitpid(child, None) { + Ok(WaitStatus::Exited(pid, code)) => { + assert_eq!(child, pid); + return code; + } + result => panic!("Could not wait for {child}: {result:?}"), + } + } + + let args_json = + std::env::var("TEST_FUZZ_LIBFUZZER_ARGS").expect("`TEST_FUZZ_LIBFUZZER_ARGS` is not set"); + let args = serde_json::from_str::>(&args_json).expect(&format!( + "Could not deserialize `TEST_FUZZ_LIBFUZZER_ARGS`: {:#?}", + args_json + )); + + let argv0 = std::env::args() + .next() + .and_then(|arg| CString::new(arg).ok()) + .unwrap(); + let argv = std::iter::once(&argv0) + .chain(args.iter()) + .map(|arg| arg.as_ptr()) + .collect::>(); + + let argc: c_int = argv.len() as c_int; + let argv: *const *const c_char = &argv[0]; + + unsafe { LLVMFuzzerRunDriver(&argc, &argv, libfuzzer_sys::test_input_wrap) } +} + +pub static LIBFUZZER_FUZZ_TARGET: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); + +fuzz_target!(|data: &[u8]| { + let libfuzzer_fuzz_target_ptr = LIBFUZZER_FUZZ_TARGET.load(Ordering::SeqCst); + let libfuzzer_fuzz_target = + unsafe { std::mem::transmute::<_, fn(&[u8])>(libfuzzer_fuzz_target_ptr) }; + libfuzzer_fuzz_target(data); +}); diff --git a/test-fuzz/Cargo.toml b/test-fuzz/Cargo.toml index 094018eb..43d67e96 100644 --- a/test-fuzz/Cargo.toml +++ b/test-fuzz/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz" version = "3.0.5" -edition = "2018" +edition = "2021" description = "To make fuzzing Rust easy" @@ -12,6 +12,7 @@ repository = "https://github.com/trailofbits/test-fuzz" [dependencies] afl = { version = "0.12", optional = true } serde = "1.0" +serde_json = { version = "1.0", optional = true } internal = { path = "../internal", package = "test-fuzz-internal", version = "=3.0.5" } runtime = { path = "../runtime", package = "test-fuzz-runtime", version = "=3.0.5" } @@ -20,10 +21,17 @@ test-fuzz-macro = { path = "../macro", version = "=3.0.5" } [dev-dependencies] assert_cmd = "2.0" cargo_metadata = "0.15" +env_logger = "0.10" +heck = "0.4" +itertools = "0.10" lazy_static = "1.4" predicates = "3.0" regex = "1.7" semver = "1.0" +strum = "0.24" +strum_macros = "0.24" +test-log = "0.2" +yaml-rust = "0.4" testing = { path = "../testing", package = "test-fuzz-testing" } @@ -35,7 +43,9 @@ auto_concretize = ["internal/__auto_concretize", "test-fuzz-macro/__auto_concret serde_bincode = ["internal/__serde_bincode", "runtime/__serde_bincode", "test-fuzz-macro/__serde_bincode"] serde_cbor = ["internal/__serde_cbor", "runtime/__serde_cbor", "test-fuzz-macro/__serde_cbor"] serde_cbor4ii = ["internal/__serde_cbor4ii", "runtime/__serde_cbor4ii", "test-fuzz-macro/__serde_cbor4ii"] -__persistent = ["afl", "test-fuzz-macro/__persistent"] +__fuzzer_aflplusplus = ["runtime/__fuzzer_aflplusplus", "test-fuzz-macro/__fuzzer_aflplusplus"] +__fuzzer_aflplusplus_persistent = ["runtime/__fuzzer_aflplusplus_persistent", "test-fuzz-macro/__fuzzer_aflplusplus_persistent", "afl"] +__fuzzer_libfuzzer = ["runtime/__fuzzer_libfuzzer", "test-fuzz-macro/__fuzzer_libfuzzer"] [package.metadata.cargo-udeps.ignore] normal = ["afl"] diff --git a/test-fuzz/build.rs b/test-fuzz/build.rs index 6eb1f9e3..1f4f2163 100644 --- a/test-fuzz/build.rs +++ b/test-fuzz/build.rs @@ -1,4 +1,11 @@ fn main() { + #[cfg(not(any( + feature = "__fuzzer_aflplusplus", + feature = "__fuzzer_aflplusplus_persistent", + feature = "__fuzzer_libfuzzer", + )))] + println!("cargo:rustc-cfg=fuzzer_default"); + #[cfg(not(any( feature = "serde_bincode", feature = "serde_cbor", diff --git a/test-fuzz/src/lib.rs b/test-fuzz/src/lib.rs index ed618053..39a1e335 100644 --- a/test-fuzz/src/lib.rs +++ b/test-fuzz/src/lib.rs @@ -3,7 +3,7 @@ pub use test_fuzz_macro::{test_fuzz, test_fuzz_impl}; // smoelius: Re-export afl so that test-fuzz clients do not need to add it to their Cargo.toml // files. -#[cfg(feature = "__persistent")] +#[cfg(feature = "__fuzzer_aflplusplus_persistent")] pub use afl; // smoelius: Unfortunately, the same trick doesn't work for serde. diff --git a/test-fuzz/tests/auto_generate.rs b/test-fuzz/tests/auto_generate.rs index b8cc05bf..b34d1f65 100644 --- a/test-fuzz/tests/auto_generate.rs +++ b/test-fuzz/tests/auto_generate.rs @@ -1,6 +1,7 @@ use internal::dirs::corpus_directory_from_target; use std::fs::{read_dir, remove_dir_all}; -use testing::examples; +use test_log::test; +use testing::{examples, CommandExt}; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -35,7 +36,7 @@ fn test(name: &str, n: usize) { &format!("{name}::target_fuzz::auto_generate"), ) .unwrap() - .assert() + .logged_assert() .success(); assert_eq!(read_dir(corpus).map(Iterator::count).unwrap_or_default(), n); diff --git a/test-fuzz/tests/conversion.rs b/test-fuzz/tests/conversion.rs index bd7c0124..8b9ebd99 100644 --- a/test-fuzz/tests/conversion.rs +++ b/test-fuzz/tests/conversion.rs @@ -1,6 +1,7 @@ use assert_cmd::prelude::*; use predicates::prelude::*; use std::process::Command; +use test_log::test; use testing::examples::MANIFEST_PATH; #[test] diff --git a/test-fuzz/tests/default.rs b/test-fuzz/tests/default.rs index a37e227d..76a3a27e 100644 --- a/test-fuzz/tests/default.rs +++ b/test-fuzz/tests/default.rs @@ -1,6 +1,7 @@ use internal::dirs::corpus_directory_from_target; use std::fs::{read_dir, remove_dir_all}; -use testing::examples; +use test_log::test; +use testing::{examples, CommandExt}; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", @@ -32,7 +33,7 @@ fn test(name: &str, n: usize) { examples::test("default", &format!("{name}::target_fuzz::auto_generate")) .unwrap() - .assert() + .logged_assert() .success(); assert_eq!(read_dir(corpus).map(Iterator::count).unwrap_or_default(), n); diff --git a/test-fuzz/tests/github.rs b/test-fuzz/tests/github.rs new file mode 100644 index 00000000..48f75c8d --- /dev/null +++ b/test-fuzz/tests/github.rs @@ -0,0 +1,69 @@ +use heck::ToKebabCase; +use internal::Fuzzer; +use itertools::Itertools; +use std::{fs::read_to_string, path::Path}; +use strum::IntoEnumIterator; +use strum_macros::{Display, EnumIter}; +use yaml_rust::YamlLoader; + +// smoelius: We cannot use `internal::SerdeFormat` because only one of its variants will be enabled. +#[derive(Clone, Copy, Debug, Display, EnumIter)] +enum SerdeFormat { + Bincode, + Cbor, + Cbor4ii, +} + +#[derive(Clone, Copy, Debug, Display, EnumIter)] +enum Environment { + UbuntuLatest, + MacosLatest, +} + +#[derive(Clone, Copy, Debug, Display, EnumIter)] +enum Toolchain { + Stable, + Nightly, +} + +#[test] +fn matrix() { + let expecteds = Fuzzer::iter().cartesian_product(SerdeFormat::iter()).zip( + Environment::iter() + .cartesian_product(Toolchain::iter()) + .cycle(), + ); + + let ci_yml = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.github/workflows/ci.yml"); + let contents = read_to_string(ci_yml).unwrap(); + let doc = YamlLoader::load_from_str(&contents) + .ok() + .and_then(|vec| vec.into_iter().next()) + .unwrap(); + let actuals = doc["jobs"]["test"]["strategy"]["matrix"]["include"] + .as_vec() + .unwrap(); + + assert_eq!(expecteds.clone().count(), actuals.len()); + + for (expected, actual) in expecteds.zip(actuals) { + let ((expected_fuzzer, expected_serde_format), (expected_environment, expected_toolchain)) = + expected; + assert_eq!( + expected_fuzzer.to_string().to_kebab_case(), + actual["fuzzer"].as_str().unwrap() + ); + assert_eq!( + expected_serde_format.to_string().to_kebab_case(), + actual["serde_format"].as_str().unwrap() + ); + assert_eq!( + expected_environment.to_string().to_kebab_case(), + actual["environment"].as_str().unwrap() + ); + assert_eq!( + expected_toolchain.to_string().to_kebab_case(), + actual["toolchain"].as_str().unwrap() + ); + } +} diff --git a/test-fuzz/tests/in_production.rs b/test-fuzz/tests/in_production.rs index 3394767c..029b1db2 100644 --- a/test-fuzz/tests/in_production.rs +++ b/test-fuzz/tests/in_production.rs @@ -8,6 +8,7 @@ use std::{ process::Command, sync::Mutex, }; +use test_log::test; #[cfg_attr( dylint_lib = "non_thread_safe_call_in_test", diff --git a/test-fuzz/tests/link.rs b/test-fuzz/tests/link.rs index 92cd7765..e00342e3 100644 --- a/test-fuzz/tests/link.rs +++ b/test-fuzz/tests/link.rs @@ -1,7 +1,8 @@ use assert_cmd::Command; use internal::dirs::target_directory; use predicates::prelude::*; -use testing::examples::MANIFEST_PATH; +use test_log::test; +use testing::{examples::MANIFEST_PATH, CommandExt}; const SERDE_DEFAULT: &str = "bincode"; @@ -15,7 +16,7 @@ fn link() { "--features", &("test-fuzz/".to_owned() + test_fuzz::serde_format().as_feature()), ]) - .assert() + .logged_assert() .success(); let pred = predicate::str::contains(SERDE_DEFAULT); @@ -25,11 +26,11 @@ fn link() { // smoelius: https://stackoverflow.com/questions/7219845/difference-between-nm-and-objdump Command::new("nm") - .args([target_directory(false) + .args([target_directory(None) .join("debug/hello-world") .to_string_lossy() .to_string()]) - .assert() + .logged_assert() .success() .stdout(pred); } diff --git a/test-fuzz/tests/rename.rs b/test-fuzz/tests/rename.rs index 41ea5828..9da424ee 100644 --- a/test-fuzz/tests/rename.rs +++ b/test-fuzz/tests/rename.rs @@ -1,6 +1,7 @@ use assert_cmd::prelude::*; use predicates::prelude::*; use std::process::Command; +use test_log::test; use testing::examples::MANIFEST_PATH; #[test] diff --git a/test-fuzz/tests/serde_format.rs b/test-fuzz/tests/serde_format.rs index e09c91b0..f0a14016 100644 --- a/test-fuzz/tests/serde_format.rs +++ b/test-fuzz/tests/serde_format.rs @@ -3,7 +3,8 @@ use std::{ fs::{read_dir, remove_dir_all, File}, io::Read, }; -use testing::examples; +use test_log::test; +use testing::{examples, CommandExt}; #[test] fn serde_format() { @@ -13,7 +14,7 @@ fn serde_format() { examples::test("serde", "unit_variant::test") .unwrap() - .assert() + .logged_assert() .success(); for entry in read_dir(corpus).unwrap() { diff --git a/test-fuzz/tests/test_fuzz_log.rs b/test-fuzz/tests/test_fuzz_log.rs index b2d645af..282e2c11 100644 --- a/test-fuzz/tests/test_fuzz_log.rs +++ b/test-fuzz/tests/test_fuzz_log.rs @@ -1,6 +1,7 @@ use assert_cmd::Command; use predicates::prelude::*; -use testing::examples::MANIFEST_PATH; +use test_log::test; +use testing::{examples::MANIFEST_PATH, CommandExt}; // smoelius: This test will fail if run twice because the target will have already been built. #[test] @@ -15,7 +16,7 @@ fn test_fuzz_log() { "--features", &("test-fuzz/".to_owned() + test_fuzz::serde_format().as_feature()), ]) - .assert() + .logged_assert() .success() .stdout(predicate::str::is_match(r"(?m)^#\[cfg\(test\)\]\nmod parse_fuzz \{$").unwrap()); } diff --git a/test-fuzz/tests/versions.rs b/test-fuzz/tests/versions.rs index 72a404df..6bc0cabf 100644 --- a/test-fuzz/tests/versions.rs +++ b/test-fuzz/tests/versions.rs @@ -6,6 +6,7 @@ use std::{ fs::read_to_string, path::{Path, PathBuf}, }; +use test_log::test; lazy_static! { static ref METADATA: Metadata = MetadataCommand::new().no_deps().exec().unwrap(); diff --git a/testing/Cargo.toml b/testing/Cargo.toml index d7cb58a3..e13f16fc 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-fuzz-testing" version = "3.0.5" -edition = "2018" +edition = "2021" publish = false [dependencies] diff --git a/testing/src/command_ext.rs b/testing/src/command_ext.rs new file mode 100644 index 00000000..8a5a3694 --- /dev/null +++ b/testing/src/command_ext.rs @@ -0,0 +1,14 @@ +use assert_cmd::{assert::Assert, Command}; +use log::debug; + +pub trait CommandExt { + fn logged_assert(&mut self) -> Assert; +} + +impl CommandExt for Command { + fn logged_assert(&mut self) -> Assert { + debug!("{:?}", self); + #[allow(clippy::disallowed_methods)] + Self::assert(self) + } +} diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 66905603..cb9fdcf4 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -1,3 +1,6 @@ +mod command_ext; +pub use command_ext::CommandExt; + pub mod examples; mod retry;