diff --git a/Cargo.toml b/Cargo.toml index ae30bb12..9cf5aeeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,6 @@ include = ["src/*.rs", "src/frame/**/*", "src/block/**/*", "README.md", "LICENSE all-features = true rustdoc-args = ["--cfg", "docsrs"] -[[bench]] -harness = false -name = "crit_bench" -path = "benches/crit_bench.rs" - [dev-dependencies] criterion = { git = "https://github.com/PSeitz/criterion.rs/", rev = "cf60ffc"} lzzzz = "1.0.4" @@ -28,6 +23,8 @@ more-asserts = "0.3.1" snap = "1.1.0" serde_json = "1.0.91" proptest = "1.0.0" +binggan = { version = "0.8", path = "../../binggan"} +jemallocator = "0.5.4" [dev-dependencies.lz-fear] git = "https://github.com/main--/rust-lz-fear" @@ -60,10 +57,20 @@ opt-level = 3 [profile.release] codegen-units = 1 -#debug = true +debug = true opt-level = 3 panic = "unwind" +[[bench]] +harness = false +name = "binggan_bench" +path = "benches/binggan_bench.rs" + +#[[bench]] +#harness = false +#name = "crit_bench" +#path = "benches/crit_bench.rs" + # [[bench]] # harness = false # name = "quickbench" diff --git a/benches/binggan_bench.rs b/benches/binggan_bench.rs new file mode 100644 index 00000000..8ebb77b0 --- /dev/null +++ b/benches/binggan_bench.rs @@ -0,0 +1,283 @@ +#![allow(dead_code)] +extern crate criterion; + +#[allow(unused)] +use std::io::{Read, Write}; + +use binggan::black_box; +use binggan::BenchRunner; +use binggan::InputGroup; +use binggan::PeakMemAlloc; +use lz_fear::raw::compress2; +use lz_fear::raw::decompress_raw; +use lz_fear::raw::U16Table; +use lz_fear::raw::U32Table; + +const COMPRESSION1K: &[u8] = include_bytes!("compression_1k.txt"); +const COMPRESSION34K: &[u8] = include_bytes!("compression_34k.txt"); +const COMPRESSION65K: &[u8] = include_bytes!("compression_65k.txt"); +const COMPRESSION66K: &[u8] = include_bytes!("compression_66k_JSON.txt"); +const COMPRESSION10MB: &[u8] = include_bytes!("dickens.txt"); +const COMPRESSION95K_VERY_GOOD_LOGO: &[u8] = include_bytes!("../logo.jpg"); + +#[global_allocator] +//pub static GLOBAL: PeakMemAlloc = PeakMemAlloc::new(std::alloc::System); +pub static GLOBAL: PeakMemAlloc = PeakMemAlloc::new(jemallocator::Jemalloc); + +const ALL: &[&[u8]] = &[ + COMPRESSION1K as &[u8], + COMPRESSION34K as &[u8], + COMPRESSION65K as &[u8], + COMPRESSION66K as &[u8], + COMPRESSION10MB as &[u8], + COMPRESSION95K_VERY_GOOD_LOGO as &[u8], +]; + +fn main() { + #[cfg(feature = "frame")] + { + let data_sets = get_frame_datasets(); + frame_decompress(&data_sets); + frame_compress(InputGroup::new_with_inputs(data_sets)); + } + + let named_data = ALL + .iter() + .map(|data| (data.len().to_string(), data.to_vec())) + .collect(); + block_compress(InputGroup::new_with_inputs(named_data)); + block_decompress(); +} + +#[cfg(feature = "frame")] +fn frame_decompress(data_sets: &[(String, Vec)]) { + let mut runner = BenchRunner::with_name("frame_decompress"); + runner.set_alloc(&GLOBAL); + for (name, data_set) in data_sets { + let compressed_independent = lz4_cpp_frame_compress(&data_set, true).unwrap(); + let compressed_linked = lz4_cpp_frame_compress(&data_set, false).unwrap(); + let mut group = runner.new_group(); + group.set_name(name); + group.set_input_size(data_set.len()); + //runner.enable_perf(); + + group.register_with_input("lz4 flex independent", &compressed_independent, move |i| { + black_box(lz4_flex_frame_decompress(i).unwrap()); + }); + group.register_with_input("lz4 c90 independent", &compressed_independent, move |i| { + black_box(lz4_cpp_frame_decompress(i).unwrap()); + }); + group.register_with_input("lz4 flex linked", &compressed_linked, move |i| { + black_box(lz4_flex_frame_decompress(i).unwrap()); + }); + group.register_with_input("lz4 c90 linked", &compressed_linked, move |i| { + black_box(lz4_cpp_frame_decompress(i).unwrap()); + }); + + group.run(); + } +} + +#[cfg(feature = "frame")] +fn frame_compress(mut runner: InputGroup>) { + runner.set_name("frame_compress".to_string()); + runner.set_alloc(&GLOBAL); + + runner.throughput(|data| data.len()); + runner.register("lz4 flex independent", move |i| { + let mut frame_info = lz4_flex::frame::FrameInfo::new(); + frame_info.block_size = lz4_flex::frame::BlockSize::Max256KB; + frame_info.block_mode = lz4_flex::frame::BlockMode::Independent; + black_box(lz4_flex_frame_compress_with(frame_info, i).unwrap()); + }); + runner.register("lz4 c90 indep", move |i| { + black_box(lz4_cpp_frame_compress(i, true).unwrap()); + }); + runner.register("lz4 flex linked", move |i| { + let mut frame_info = lz4_flex::frame::FrameInfo::new(); + frame_info.block_size = lz4_flex::frame::BlockSize::Max256KB; + frame_info.block_mode = lz4_flex::frame::BlockMode::Linked; + black_box(lz4_flex_frame_compress_with(frame_info, i).unwrap()); + }); + runner.register("lz4 c90 linked", move |i| { + black_box(lz4_cpp_frame_compress(i, false).unwrap()); + }); + + runner.run(); +} + +fn block_compress(mut runner: InputGroup>) { + runner.set_name("block_compress".to_string()); + runner.set_alloc(&GLOBAL); // Set the peak mem allocator. This will enable peak memory reporting. + //runner.enable_perf(); + runner.throughput(|data| data.len()); + runner.register("lz4 flex", move |i| { + black_box(lz4_flex::compress(i)); + }); + runner.register("lz4 c90", move |i| { + black_box(lz4_cpp_block_compress(i).unwrap()); + }); + runner.register("snap", move |i| { + black_box(compress_snap(i)); + }); + + runner.run(); +} + +fn block_decompress() { + let mut runner = BenchRunner::with_name("block_decompress"); + runner.set_alloc(&GLOBAL); + for data_uncomp in ALL { + let comp_lz4 = lz4_cpp_block_compress(data_uncomp).unwrap(); + let bundle = (comp_lz4, data_uncomp.len()); + + let name = data_uncomp.len().to_string(); + let mut group = runner.new_group(); + group.set_name(name.clone()); + group.set_input_size(data_uncomp.len()); + + group.register_with_input("lz4 flex", &bundle, move |i| { + black_box(lz4_flex::decompress(&i.0, i.1).unwrap()); + }); + group.register_with_input("lz4 c90", &bundle, move |i| { + black_box(lz4_cpp_block_decompress(&i.0, i.1).unwrap()); + }); + + group.run(); + } +} + +fn get_frame_datasets() -> Vec<(String, Vec)> { + let paths = [ + "compression_1k.txt", + "dickens.txt", + "hdfs.json", + "reymont.pdf", + "xml_collection.xml", + ]; + paths + .iter() + .map(|path| { + let path_buf = std::path::Path::new("benches").join(path); + let mut file = std::fs::File::open(path_buf).unwrap(); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + (path.to_string(), buf) + }) + .collect() +} + +fn compress_lz4_fear(input: &[u8]) -> Vec { + let mut buf = Vec::new(); + if input.len() <= 0xFFFF { + compress2(input, 0, &mut U16Table::default(), &mut buf).unwrap(); + } else { + compress2(input, 0, &mut U32Table::default(), &mut buf).unwrap(); + } + buf +} + +fn decompress_lz4_fear(input: &[u8]) -> Vec { + let mut vec = Vec::new(); + decompress_raw(input, &[], &mut vec, std::usize::MAX).unwrap(); + vec +} + +fn compress_snap(input: &[u8]) -> Vec { + snap::raw::Encoder::new().compress_vec(input).unwrap() +} + +fn decompress_snap(input: &[u8]) -> Vec { + snap::raw::Decoder::new().decompress_vec(input).unwrap() +} + +#[cfg(feature = "frame")] +fn compress_snap_frame(input: &[u8]) -> Vec { + let mut fe = snap::write::FrameEncoder::new(Vec::new()); + fe.write_all(input).unwrap(); + fe.into_inner().unwrap() +} + +#[cfg(feature = "frame")] +fn decompress_snap_frame(input: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut fe = snap::read::FrameDecoder::new(input); + fe.read_to_end(&mut out).unwrap(); + out +} + +fn lz4_cpp_block_decompress(input: &[u8], decomp_len: usize) -> Result, lzzzz::Error> { + let mut out = vec![0u8; decomp_len]; + lzzzz::lz4::decompress(input, &mut out)?; + Ok(out) +} + +fn lz4_cpp_block_compress(input: &[u8]) -> Result, lzzzz::Error> { + let mut out = Vec::new(); + lzzzz::lz4::compress_to_vec(input, &mut out, lzzzz::lz4::ACC_LEVEL_DEFAULT).unwrap(); + Ok(out) +} + +#[cfg(feature = "frame")] +fn lz4_cpp_frame_compress(input: &[u8], independent: bool) -> Result, lzzzz::Error> { + let pref = lzzzz::lz4f::PreferencesBuilder::new() + .block_mode(if independent { + lzzzz::lz4f::BlockMode::Independent + } else { + lzzzz::lz4f::BlockMode::Linked + }) + .block_size(lzzzz::lz4f::BlockSize::Max64KB) + .build(); + let mut comp = lzzzz::lz4f::WriteCompressor::new(Vec::new(), pref).unwrap(); + comp.write_all(input).unwrap(); + let out = comp.into_inner(); + + Ok(out) +} + +#[cfg(feature = "frame")] +fn lz4_cpp_frame_decompress(mut input: &[u8]) -> Result, lzzzz::lz4f::Error> { + let mut r = lzzzz::lz4f::ReadDecompressor::new(&mut input)?; + let mut buf = Vec::new(); + r.read_to_end(&mut buf).unwrap(); + + Ok(buf) +} + +#[cfg(feature = "frame")] +pub fn lz4_flex_frame_compress_with( + frame_info: lz4_flex::frame::FrameInfo, + input: &[u8], +) -> Result, lz4_flex::frame::Error> { + let buffer = Vec::new(); + let mut enc = lz4_flex::frame::FrameEncoder::with_frame_info(frame_info, buffer); + enc.write_all(input)?; + enc.finish() +} + +#[cfg(feature = "frame")] +pub fn lz4_flex_frame_decompress(input: &[u8]) -> Result, lz4_flex::frame::Error> { + let mut de = lz4_flex::frame::FrameDecoder::new(input); + let mut out = Vec::new(); + de.read_to_end(&mut out)?; + Ok(out) +} + +//pub fn lz4_flex_master_frame_compress_with( +//frame_info: lz4_flex_master::frame::FrameInfo, +//input: &[u8], +//) -> Result, lz4_flex_master::frame::Error> { +//let buffer = Vec::new(); +//let mut enc = lz4_flex_master::frame::FrameEncoder::with_frame_info(frame_info, buffer); +//enc.write_all(input)?; +//Ok(enc.finish()?) +//} + +//pub fn lz4_flex_master_frame_decompress( +//input: &[u8], +//) -> Result, lz4_flex_master::frame::Error> { +//let mut de = lz4_flex_master::frame::FrameDecoder::new(input); +//let mut out = Vec::new(); +//de.read_to_end(&mut out)?; +//Ok(out) +//} diff --git a/src/fastcpy.rs b/src/fastcpy.rs index aec17321..8bbd480c 100644 --- a/src/fastcpy.rs +++ b/src/fastcpy.rs @@ -99,6 +99,7 @@ fn double_copy_trick(src: &[u8], dst: &mut [u8]) { #[cfg(test)] mod tests { use super::slice_copy; + use alloc::vec::Vec; use proptest::prelude::*; proptest! { #[test] diff --git a/src/fastcpy_unsafe.rs b/src/fastcpy_unsafe.rs index 06053230..fb172804 100644 --- a/src/fastcpy_unsafe.rs +++ b/src/fastcpy_unsafe.rs @@ -116,6 +116,7 @@ fn double_copy_trick(src: *const u8, dst: *mut u8, len: usize #[cfg(test)] mod tests { use super::slice_copy; + use alloc::vec::Vec; use proptest::prelude::*; proptest! { #[test]