src support lz4, refactor processing linear syslogs

support parsing lz4 compressed files `.lz4`

refactor processing sequential-only syslogs (compressed logs that can only
be read linearly; no binary searching)

add logs for LZ4 `.lz4`, and old LZMA `.lz`, and variations of those

add tests in compare-current-and-expected

add lz4 flamegraph in flamegraphs.sh

Fix Issue #201 with tests that had unknown panics for blockreader
processing gzip files.

Issue PSeitz/lz4_flex#159
Issue #201
Issue #128
Issue #291
Issue #283

Cargo.toml

@@ -54,6 +54,7 @@ kinded = "0.3.0"
 lazy_static = "1.4.0"
 lru = "0.12.3"
 lzma-rs = "0.3.0"
+lz4_flex = "0.11"
 memoffset = "0.9.1"
 mime_guess = "2.0.4"
 min-max = "0.1.8"

src/common.rs

@@ -573,6 +573,8 @@ pub enum FileTypeArchive {
     ///
     /// Presumed to contain one regular file; see Issue #8
     Gz,
+    /// a compressed LZIP file, e.g. `log.lzma`
+    Lz4,
     /// a file within a `.tar` archive file
     Tar,
     /// a file compressed "xz'd" file, e.g. `log.xz`
@@ -589,6 +591,7 @@ impl std::fmt::Display for FileTypeArchive {
         match self {
             FileTypeArchive::Normal => write!(f, "Normal"),
             FileTypeArchive::Gz => write!(f, "gzip"),
+            FileTypeArchive::Lz4 => write!(f, "lz4"),
             FileTypeArchive::Tar => write!(f, "tar"),
             FileTypeArchive::Xz => write!(f, "xz"),
         }
@@ -728,18 +731,22 @@ impl FileType {
         match self {
             FileType::Evtx{ archival_type: FileTypeArchive::Normal } => false,
             FileType::Evtx{ archival_type: FileTypeArchive::Gz } => true,
+            FileType::Evtx{ archival_type: FileTypeArchive::Lz4 } => true,
             FileType::Evtx{ archival_type: FileTypeArchive::Tar } => false,
             FileType::Evtx{ archival_type: FileTypeArchive::Xz } => true,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Normal, .. } => false,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Gz, .. } => true,
+            FileType::FixedStruct{ archival_type: FileTypeArchive::Lz4, .. } => true,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Tar, .. } => false,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Xz, .. } => true,
             FileType::Journal{ archival_type: FileTypeArchive::Normal } => false,
             FileType::Journal{ archival_type: FileTypeArchive::Gz } => true,
+            FileType::Journal{ archival_type: FileTypeArchive::Lz4 } => true,
             FileType::Journal{ archival_type: FileTypeArchive::Tar } => false,
             FileType::Journal{ archival_type: FileTypeArchive::Xz } => true,
             FileType::Text{ archival_type: FileTypeArchive::Normal, .. } => false,
             FileType::Text{ archival_type: FileTypeArchive::Gz, .. } => true,
+            FileType::Text{ archival_type: FileTypeArchive::Lz4, .. } => true,
             FileType::Text{ archival_type: FileTypeArchive::Tar, .. } => false,
             FileType::Text{ archival_type: FileTypeArchive::Xz, .. } => true,
             FileType::Unparsable => false,
@@ -751,18 +758,22 @@ impl FileType {
         match self {
             FileType::Evtx{ archival_type: FileTypeArchive::Normal } => false,
             FileType::Evtx{ archival_type: FileTypeArchive::Gz } => false,
+            FileType::Evtx{ archival_type: FileTypeArchive::Lz4 } => false,
             FileType::Evtx{ archival_type: FileTypeArchive::Tar } => true,
             FileType::Evtx{ archival_type: FileTypeArchive::Xz } => false,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Normal, .. } => false,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Gz, .. } => false,
+            FileType::FixedStruct{ archival_type: FileTypeArchive::Lz4, .. } => false,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Tar, .. } => true,
             FileType::FixedStruct{ archival_type: FileTypeArchive::Xz, .. } => false,
             FileType::Journal{ archival_type: FileTypeArchive::Normal } => false,
             FileType::Journal{ archival_type: FileTypeArchive::Gz } => false,
+            FileType::Journal{ archival_type: FileTypeArchive::Lz4 } => false,
             FileType::Journal{ archival_type: FileTypeArchive::Tar } => true,
             FileType::Journal{ archival_type: FileTypeArchive::Xz } => false,
             FileType::Text{ archival_type: FileTypeArchive::Normal, ..} => false,
             FileType::Text{ archival_type: FileTypeArchive::Gz, .. } => false,
+            FileType::Text{ archival_type: FileTypeArchive::Lz4, .. } => false,
             FileType::Text{ archival_type: FileTypeArchive::Tar, .. } => true,
             FileType::Text{ archival_type: FileTypeArchive::Xz, .. } => false,
             FileType::Unparsable => false,

src/printer/summary.rs

@@ -1285,8 +1285,10 @@ fn print_summary_opt_processed_summaryblockreader(
             );
         }
         FileType::FixedStruct{ archival_type: FileTypeArchive::Gz, fixedstruct_type: _ }
+        | FileType::FixedStruct{ archival_type: FileTypeArchive::Lz4, fixedstruct_type: _ }
         | FileType::FixedStruct{ archival_type: FileTypeArchive::Xz, fixedstruct_type: _ }
         | FileType::Text{ archival_type: FileTypeArchive::Gz, encoding_type: _ }
+        | FileType::Text{ archival_type: FileTypeArchive::Lz4, encoding_type: _ }
         | FileType::Text{ archival_type: FileTypeArchive::Xz, encoding_type: _ }
         => {
             eprintln!(