From 11c09c093df18f02b62b41e2969ad0a02872bc54 Mon Sep 17 00:00:00 2001 From: Carlos Rolo <3799585+cjrolo@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:22:33 +0100 Subject: [PATCH 1/2] Uncompressing --- brro-compressor/src/compressor/constant.rs | 12 ++++-- brro-compressor/src/compressor/fft.rs | 11 ++++-- brro-compressor/src/compressor/mod.rs | 16 ++++++-- brro-compressor/src/compressor/noop.rs | 12 ++++-- brro-compressor/src/data.rs | 23 ++++++++++- brro-compressor/src/frame/mod.rs | 11 +++++- brro-compressor/src/main.rs | 45 +++++++++++++++------- 7 files changed, 100 insertions(+), 30 deletions(-) diff --git a/brro-compressor/src/compressor/constant.rs b/brro-compressor/src/compressor/constant.rs index 48da3e5..5a68f10 100644 --- a/brro-compressor/src/compressor/constant.rs +++ b/brro-compressor/src/compressor/constant.rs @@ -120,6 +120,12 @@ pub fn constant(data: &[f64]) -> Vec { c.to_bytes() } +pub fn constant_to_data(sample_number: usize, compressed_data: &[u8]) -> Vec { + let c = Constant::decompress(compressed_data); + let out_i64 = c.to_data(sample_number); + out_i64.iter().map(|&x| x as f64).collect() +} + #[cfg(test)] mod tests { use super::*; @@ -164,11 +170,9 @@ mod tests { #[test] fn test_to_data() { let vector1 = vec![1.0, 2.0, 1.0, 1.0, 3.0]; - // Currently we are not dealing with floats - let out_vector1: Vec = vector1.iter().map(|&x| x as i64).collect(); let frame_size = vector1.len(); let compressed_data = constant(&vector1); - let out = Constant::decompress(&compressed_data).to_data(frame_size); - assert_eq!(out_vector1, out); + let out = constant_to_data(frame_size, &compressed_data); + assert_eq!(vector1, out); } } diff --git a/brro-compressor/src/compressor/fft.rs b/brro-compressor/src/compressor/fft.rs index f522bbd..d33963c 100644 --- a/brro-compressor/src/compressor/fft.rs +++ b/brro-compressor/src/compressor/fft.rs @@ -233,6 +233,12 @@ pub fn fft(data: &[f64], max_freqs: usize, min: f64, max: f64) -> Vec { c.to_bytes() } +/// Uncompress a FFT data +pub fn fft_to_data(sample_number: usize, compressed_data: &[u8]) -> Vec { + let c = FFT::decompress(compressed_data); + c.to_data(sample_number) +} + /// Compress targeting a specific max error allowed. This is very computational intensive, /// as the FFT will be calculated over and over until the specific error threshold is achived. /// `max_freqs` is used as a starting point for the calculation @@ -268,7 +274,7 @@ mod tests { let vector1 = vec![1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 3.0, 1.0, 1.0, 5.0]; let frame_size = vector1.len(); let compressed_data = fft(&vector1, frame_size, 1.0, 5.0); - let out = FFT::decompress(&compressed_data).to_data(frame_size); + let out = fft_to_data(vector1.len(), &compressed_data); assert_eq!(vector1, out); } @@ -276,9 +282,8 @@ mod tests { fn test_to_lossy_data() { let vector1 = vec![1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 3.0, 1.0, 1.0, 5.0]; let lossy_vec = vec![1.1, 1.0, 1.1, 1.0, 2.1, 1.0, 1.1, 1.0, 3.1, 1.0, 1.1, 4.9]; - let frame_size = vector1.len(); let compressed_data = fft(&vector1, 6, 1.0, 5.0); - let out = FFT::decompress(&compressed_data).to_data(frame_size); + let out = fft_to_data(vector1.len(), &compressed_data); assert_eq!(lossy_vec, out); } diff --git a/brro-compressor/src/compressor/mod.rs b/brro-compressor/src/compressor/mod.rs index 0970e62..5b05f7b 100644 --- a/brro-compressor/src/compressor/mod.rs +++ b/brro-compressor/src/compressor/mod.rs @@ -1,8 +1,9 @@ use bincode::config::{self, Configuration}; use bincode::{Decode, Encode}; -use self::constant::constant; -use self::fft::fft; -use self::noop::noop; + +use self::constant::{constant, constant_to_data}; +use self::fft::{fft, fft_to_data}; +use self::noop::{noop, noop_to_data}; pub mod noop; pub mod constant; @@ -28,6 +29,15 @@ impl Compressor { _ => todo!(), } } + + pub fn decompress(&self, samples: usize, data: &[u8] ) -> Vec { + match self { + Compressor::Noop => noop_to_data(samples, data), + Compressor::FFT => fft_to_data(samples, data), + Compressor::Constant => constant_to_data(samples, data), + _ => todo!() + } + } } pub struct BinConfig { diff --git a/brro-compressor/src/compressor/noop.rs b/brro-compressor/src/compressor/noop.rs index 97d6b8c..f98cb69 100644 --- a/brro-compressor/src/compressor/noop.rs +++ b/brro-compressor/src/compressor/noop.rs @@ -62,6 +62,12 @@ pub fn noop(data: &[f64]) -> Vec { c.to_bytes() } +pub fn noop_to_data(sample_number: usize, compressed_data: &[u8]) -> Vec { + let c = Noop::decompress(compressed_data); + let out_i64 = c.to_data(sample_number); + out_i64.iter().map(|&x| x as f64).collect() +} + #[cfg(test)] mod tests { use super::*; @@ -86,9 +92,7 @@ mod tests { #[test] fn test_decompression() { let vector1 = vec![1.0, 2.0, 3.0, 4.0, 1.0]; - assert_eq!( - Noop::decompress(&noop(&vector1)).to_data(0), - [1, 2, 3, 4, 1] - ); + let n = noop(&vector1); + assert_eq!(noop_to_data(vector1.len(), &n),vector1); } } diff --git a/brro-compressor/src/data.rs b/brro-compressor/src/data.rs index 22f1690..8c60bc1 100644 --- a/brro-compressor/src/data.rs +++ b/brro-compressor/src/data.rs @@ -32,7 +32,6 @@ impl CompressedStream { compressor_frame.compress(chunk); compressor_frame.close(); self.data_frames.push(compressor_frame); - } /// Transforms the whole CompressedStream into bytes to be written to a file @@ -42,12 +41,21 @@ impl CompressedStream { bincode::encode_to_vec(self, config).unwrap() } - /// Gets a binary stream and generates a Compressed Stream + /// Gets a binary stream and generates a Compressed Stream, at this point, anything inside the stream is + /// still in the compressed state pub fn from_bytes(data: &[u8]) -> Self { let config = BinConfig::get(); let (compressed_stream, _) = bincode::decode_from_slice(data, config).unwrap(); compressed_stream } + + /// Decompresses all the frames and returns a vector with the data + pub fn decompress(&self) -> Vec { + self.data_frames.iter() + .flat_map(|f| f.decompress()) + .collect() + } + } #[cfg(test)] @@ -89,4 +97,15 @@ mod tests { let cs2 = CompressedStream::from_bytes(&b); assert_eq!(len, cs2.data_frames.len()); } + + #[test] + fn test_constant_decompression() { + let vector1 = vec![1.0; 1024]; + let mut cs = CompressedStream::new(); + cs.compress_chunk_with(&vector1, Compressor::Constant); + let b = cs.to_bytes(); + let cs2 = CompressedStream::from_bytes(&b); + let out = cs2.decompress(); + assert_eq!(vector1, out); + } } \ No newline at end of file diff --git a/brro-compressor/src/frame/mod.rs b/brro-compressor/src/frame/mod.rs index c379a9e..b8a9664 100644 --- a/brro-compressor/src/frame/mod.rs +++ b/brro-compressor/src/frame/mod.rs @@ -1,5 +1,6 @@ use std::mem::size_of_val; use bincode::{Decode, Encode}; +use log::debug; use crate::compressor::Compressor; /// This is the structure of a compressor frame @@ -8,7 +9,7 @@ pub struct CompressorFrame{ /// The frame size in bytes, frame_size: usize, /// The number of samples in this frame, - samples: u32, + samples: usize, /// The compressor used in the current frame compressor: Compressor, /// Output from the compressor @@ -41,6 +42,14 @@ impl CompressorFrame { pub fn compress(&mut self, data: &[f64]) { // TODO: Optimize here // self.compressor = optimizer_selection + self.samples = data.len(); self.data = self.compressor.compress(data); } + + /// Decompresses a frame and returns the resulting data array + pub fn decompress(&self) -> Vec { + debug!("Decompressing Frame. Size: {}, Samples: {}", self.frame_size, self.samples); + self.compressor.decompress(self.samples, &self.data) + } + } \ No newline at end of file diff --git a/brro-compressor/src/main.rs b/brro-compressor/src/main.rs index e643480..1b6473b 100644 --- a/brro-compressor/src/main.rs +++ b/brro-compressor/src/main.rs @@ -25,6 +25,7 @@ fn process_args(arguments: &Args) { /// Processes all files in a given directory. fn process_directory(arguments: &Args) { + // TODO: Uncompresses directories let new_name = format!( "{}-compressed", arguments.input.file_name().unwrap().to_string_lossy() @@ -48,15 +49,21 @@ fn process_directory(arguments: &Args) { /// Processes a single file. fn process_single_file(arguments: &Args) { debug!("Processing single file..."); - let (vec, tag) = reader::read_file(&arguments.input).expect("Failed to read file"); - let compressed_data = compress_data(&vec, &tag, arguments); - if let Some(filename_osstr) = arguments.input.file_name() { - if let Some(filename_str) = filename_osstr.to_str() { - // BRO extension - let new_filename_string = - writer::replace_extension(&filename_str.to_string(), "bro"); - let new_path = arguments.input.parent().unwrap().join(new_filename_string); - write_compressed_data_to_path(&compressed_data, &new_path); + if arguments.uncompress { + // TODO: Read a BRRO file, feed it to the decompressor + decompress_data(&[1,2,3]); + // TODO: Write the ouput out + } else { + let (vec, tag) = reader::read_file(&arguments.input).expect("Failed to read file"); + let compressed_data = compress_data(&vec, &tag, arguments); + if let Some(filename_osstr) = arguments.input.file_name() { + if let Some(filename_str) = filename_osstr.to_str() { + // BRO extension + let new_filename_string = + writer::replace_extension(&filename_str.to_string(), "bro"); + let new_path = arguments.input.parent().unwrap().join(new_filename_string); + write_compressed_data_to_path(&compressed_data, &new_path); + } } } } @@ -77,6 +84,13 @@ fn compress_data(vec: &Vec, tag: &MetricTag, arguments: &Args) -> Vec { } } +/// Compresses the data based on the provided tag and arguments. +fn decompress_data(compressed_data: &[u8]) -> Vec { + debug!("decompressing data!"); + let cs = CompressedStream::from_bytes(compressed_data); + cs.decompress() +} + /// Writes the compressed data to the specified path. fn write_compressed_data_to_path(compressed: &[u8], path: &Path) { let mut file = @@ -89,18 +103,23 @@ fn write_compressed_data_to_path(compressed: &[u8], path: &Path) { struct Args { /// input file input: PathBuf, - /// Processes a directory instead of a single file - #[arg(short, action)] + #[arg(long, action)] directory: bool, - /// Forces Noop compressor #[arg(long, action)] noop: bool, - /// Forces Constant compressor #[arg(long, action)] constant: bool, + /// Forces FFT compressor + //TODO: This needs to be a subcommand + #[arg(long, action)] + fft: bool, + /// Uncompresses the input file/directory + #[arg(short, action)] + uncompress: bool, + } fn main() { From e0771ea0a60e2da08e1265271d1714d17cca54ca Mon Sep 17 00:00:00 2001 From: Carlos Rolo <3799585+cjrolo@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:44:24 +0100 Subject: [PATCH 2/2] Test fix --- brro-compressor/src/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brro-compressor/src/data.rs b/brro-compressor/src/data.rs index 8c60bc1..751731f 100644 --- a/brro-compressor/src/data.rs +++ b/brro-compressor/src/data.rs @@ -84,7 +84,7 @@ mod tests { let mut cs = CompressedStream::new(); cs.compress_chunk_with(&vector1, Compressor::Constant); let b = cs.to_bytes(); - assert_eq!(b, [66, 82, 82, 79, 0, 1, 37, 0, 3, 3, 0, 2, 0]); + assert_eq!(b, [66, 82, 82, 79, 0, 1, 41, 251, 0, 4, 3, 3, 0, 2, 0]); } #[test]