Added file delete detection and thread management

Achiefs · Feb 1, 2025 · f9ce202 · f9ce202
1 parent bda3d5f
commit f9ce202
Show file tree

Hide file tree

Showing 7 changed files with 136 additions and 34 deletions.
diff --git a/src/appconfig.rs b/src/appconfig.rs
@@ -48,7 +48,9 @@ pub struct AppConfig {
     pub system: String,
     pub insecure: bool,
     pub events_lock: Arc<Mutex<bool>>,
-    pub log_lock: Arc<Mutex<bool>>
+    pub log_lock: Arc<Mutex<bool>>,
+    pub hashscanner_interval: usize,
+    pub engine: String
 }
 
 impl AppConfig {
@@ -77,7 +79,9 @@ impl AppConfig {
             system: self.system.clone(),
             insecure: self.insecure,
             events_lock: self.events_lock.clone(),
-            log_lock: self.log_lock.clone()
+            log_lock: self.log_lock.clone(),
+            hashscanner_interval: self.hashscanner_interval.clone(),
+            engine: self.engine.clone()
         }
     }
 
@@ -211,11 +215,13 @@ impl AppConfig {
         };
 
         // Manage null value on audit value
+        let mut engine = String::from("monitor");
         let audit = match yaml[0]["audit"].as_vec() {
             Some(value) => {
                 if utils::get_os() != "linux" {
                     panic!("Audit only supported in Linux systems.");
                 }
+                engine = String::from("audit");
                 value.to_vec()
             },
             None => {
@@ -271,6 +277,14 @@ impl AppConfig {
             None => 64
         };
 
+        let hashscanner_interval = match yaml[0]["hashscanner"]["interval"].as_i64() {
+            Some(value) => {
+                let interval = usize::try_from(value).unwrap();
+                if interval >= 5 { interval * 60 }else{ 300 } // Five minutes 
+            },
+            None => 3600 // One hour
+        };
+
         AppConfig {
             version: String::from(VERSION),
             path: cfg,
@@ -295,6 +309,8 @@ impl AppConfig {
             insecure,
             events_lock: Arc::new(Mutex::new(false)),
             log_lock: Arc::new(Mutex::new(false)),
+            hashscanner_interval,
+            engine
         }
     }
 
@@ -538,6 +554,8 @@ mod tests {
             insecure: true,
             events_lock: Arc::new(Mutex::new(false)),
             log_lock: Arc::new(Mutex::new(false)),
+            hashscanner_interval: 60,
+            engine: String::from("monitor")
         }
     }
 
@@ -566,6 +584,9 @@ mod tests {
         assert_eq!(cfg.log_max_file_size, cloned.log_max_file_size);
         assert_eq!(cfg.system, cloned.system);
         assert_eq!(cfg.insecure, cloned.insecure);
+        //
+        assert_eq!(cfg.hashscanner_interval, cloned.hashscanner_interval);
+        assert_eq!(cfg.engine, cloned.engine);
     }
 
     // ------------------------------------------------------------------------
@@ -593,6 +614,9 @@ mod tests {
         assert_eq!(cfg.log_max_file_size, 64);
         assert_eq!(cfg.system, String::from("windows"));
         assert_eq!(cfg.insecure, false);
+        //
+        assert_eq!(cfg.hashscanner_interval, 60);
+        assert_eq!(cfg.engine, String::from("monitor"));
     }
 
     // ------------------------------------------------------------------------
@@ -982,6 +1006,9 @@ mod tests {
             assert_eq!(cfg.log_max_file_size, 64);
             assert_eq!(cfg.system, String::from("linux"));
             assert_eq!(cfg.insecure, false);
+            //
+            assert_eq!(cfg.hashscanner_interval, 60);
+            assert_eq!(cfg.engine, String::from("monitor"));
         }
     }
 
@@ -1007,6 +1034,9 @@ mod tests {
         assert_eq!(cfg.log_max_file_size, 64);
         assert_eq!(cfg.system, String::from("macos"));
         assert_eq!(cfg.insecure, false);
+        //
+        assert_eq!(cfg.hashscanner_interval, 60);
+        assert_eq!(cfg.engine, String::from("monitor"));
     }
 
     // ------------------------------------------------------------------------

diff --git a/src/db.rs b/src/db.rs
@@ -175,19 +175,25 @@ impl DB {
     pub fn get_file_list(&self, path: String) -> Vec<DBFile> {
         let connection = self.open();
         let mut list = Vec::new();
-        let mut statement = connection.prepare_cached("SELECT * FROM files WHERE path LIKE '?1%'").unwrap();
-        let file_iter = statement.query_map([path], |row| {
+        let query = format!("SELECT * FROM files WHERE path LIKE '{}%'", path);
+        let mut statement = connection.prepare_cached(&query).unwrap();
+        let result = statement.query_map([], |row| {
             Ok(DBFile {
                 id: row.get(0).unwrap(),
                 timestamp: row.get(1).unwrap(),
                 hash: row.get(2).unwrap(),
                 path: row.get(3).unwrap(),
                 size: row.get(4).unwrap()
             })
-        }).unwrap();
-        for file in file_iter {
-            list.push(file.unwrap())
-        }
+        });
+        match result {
+            Ok(mapped_rows) => {
+                for file in mapped_rows {
+                    list.push(file.unwrap())
+                }
+            },
+            Err(e) => error!("Could not get database file list, error: {:?}", e)
+        };
         list
     }
 
@@ -219,6 +225,26 @@ impl DB {
 
     // ------------------------------------------------------------------------
 
+    pub fn delete_file(&self, dbfile: DBFile) -> Result<u8, DBFileError>{
+        let connection = self.open();
+        let query = "DELETE FROM files WHERE id = ?1";
+
+        let mut statement = connection.prepare(&query).unwrap();
+        let result = statement.execute(params![dbfile.id]);
+        match result {
+            Ok(_v) => {
+                debug!("File '{}', delete from database.", dbfile.path);
+                Ok(0)
+            },
+            Err(e) => {
+                error!("Cannot delete file '{}' information, Error: {:?}", dbfile.path, e);
+                Err(DBFileError::from(e))
+            }
+        }
+    }
+
+    // ------------------------------------------------------------------------
+
     pub fn print(&self) {
         let connection = self.open();
         let mut query = connection.prepare(

diff --git a/src/hash.rs b/src/hash.rs
@@ -135,6 +135,7 @@ mod tests {
     use std::fs;
     use std::fs::File;
     use std::io::prelude::*;
+    use sha3::{Digest, Sha3_512};
 
     fn create_test_file(filename: String) {
         File::create(filename).unwrap().write_all(b"This is a test!").unwrap();
@@ -150,16 +151,16 @@ mod tests {
     fn test_get_checksum_file() {
         let filename = String::from("test_get_checksum_file");
         create_test_file(filename.clone());
-        assert_eq!(get_checksum(filename.clone(), MAX_FILE_READ), String::from("46512636eeeb22dee0d60f3aba6473b1fb3258dc0c9ed6fbdbf26bed06df796bc70d4c1f6d50ca977b45f35b494e4bd9fb34e55a1576d6d9a3b5e1ab059953ee"));
+        assert_eq!(get_checksum(filename.clone(), MAX_FILE_READ, Sha3_512::new()), String::from("46512636eeeb22dee0d60f3aba6473b1fb3258dc0c9ed6fbdbf26bed06df796bc70d4c1f6d50ca977b45f35b494e4bd9fb34e55a1576d6d9a3b5e1ab059953ee"));
         remove_test_file(filename.clone());
     }
 
     // ------------------------------------------------------------------------
 
     #[test]
     fn test_get_checksum_not_exists() {
-        assert_ne!(get_checksum(String::from("not_exists"), MAX_FILE_READ), String::from("This is a test"));
-        assert_eq!(get_checksum(String::from("not_exists"), MAX_FILE_READ), String::from("UNKNOWN"));
+        assert_ne!(get_checksum(String::from("not_exists"), MAX_FILE_READ, Sha3_512::new()), String::from("This is a test"));
+        assert_eq!(get_checksum(String::from("not_exists"), MAX_FILE_READ, Sha3_512::new()), String::from("UNKNOWN"));
     }
 
     // ------------------------------------------------------------------------
@@ -168,7 +169,7 @@ mod tests {
     fn test_get_checksum_bad() {
         let filename = String::from("test_get_checksum_bad");
         create_test_file(filename.clone());
-        assert_ne!(get_checksum(filename.clone(), MAX_FILE_READ), String::from("This is a test"));
+        assert_ne!(get_checksum(filename.clone(), MAX_FILE_READ, Sha3_512::new()), String::from("This is a test"));
         remove_test_file(filename.clone());
     }
 

diff --git a/src/scanner.rs → src/hashscanner.rs b/src/scanner.rs → src/hashscanner.rs
@@ -9,6 +9,9 @@ use crate::utils;
 use walkdir::WalkDir;
 use log::*;
 use std::collections::HashSet;
+use std::time::Duration;
+use std::thread;
+use tokio::runtime::Runtime;
 
 pub fn scan_path(cfg: AppConfig, root: String) {
     let db = db::DB::new();
@@ -66,27 +69,62 @@ pub async fn check_path(cfg: AppConfig, root: String) {
 
 // ----------------------------------------------------------------------------
 
-pub fn update_db(cfg: AppConfig, root: String) {
+pub fn update_db(root: String) {
     let db = db::DB::new();
 
-    let list = db.get_file_list(root.clone());
-    let path_list = utils::get_path_file_list(root);
-
-    //path_list.iter().filter()
+    let db_list = db.get_file_list(root.clone());
+    let path_list = utils::get_fs_list(root);
 
     let path_set: HashSet<_> = path_list.iter().collect();
-    let diff: Vec<_> = list.iter().filter(|item| !path_set.contains(&item.path)).collect();
-    println!("DIFF: {:?}", diff);
+    let diff: Vec<_> = db_list.iter().filter(|item| !path_set.contains(&item.path)).collect();
+
+    for file in diff {
+        let result = db.delete_file(DBFile {
+            id: file.id.clone(),
+            timestamp: file.timestamp.clone(),
+            hash: file.hash.clone(),
+            path: file.path.clone(),
+            size: file.size
+        });
+        match result {
+            Ok(_v) => {
+                // In this case we don't trigger an event due to the watcher will trigger file deleted event in monitoring path.
+                debug!("File {} deleted from databse", file.path)
+            },
+            Err(e) => error!("Could not delete file {} from database, error: {:?}", file.path, e)
+        }
+    }
 }
 
 // ----------------------------------------------------------------------------
 
-pub async fn first_scan(cfg: AppConfig, root: String) {
+#[cfg(not(tarpaulin_include))]
+pub fn scan(cfg: AppConfig) {
     let db = db::DB::new();
-    if db.is_empty() {
-        scan_path(cfg, root);
-    } else {
-        check_path(cfg.clone(), root.clone()).await;
-        update_db(cfg, root);
+    let rt = Runtime::new().unwrap();
+    let interval = cfg.clone().hashscanner_interval;
+    debug!("Starting file scan to create hash database.");
+
+    let config_paths = match cfg.clone().engine.as_str() {
+        "audit" => cfg.clone().audit,
+        _ => cfg.clone().monitor,
+    };
+
+    loop{
+
+        for element in config_paths.clone() {
+            let path = String::from(element["path"].as_str().unwrap());
+            if db.is_empty() {
+                scan_path(cfg.clone(), path.clone());
+            } else {
+                rt.block_on(check_path(cfg.clone(), path.clone()));
+                update_db(path.clone());
+            }
+            debug!("Path '{}' scanned all files are hashed in DB.", path.clone());
+        }
+
+        debug!("Sleeping HashScanner thread for {} minutes", interval.clone());
+        thread::sleep(Duration::from_secs(interval.try_into().unwrap()));
     }
+
 }
diff --git a/src/main.rs b/src/main.rs
@@ -38,7 +38,7 @@ mod rotator;
 mod init;
 mod db;
 mod dbfile;
-mod scanner;
+mod hashscanner;
 mod hashevent;
 
 // ----------------------------------------------------------------------------
@@ -51,11 +51,17 @@ async fn main() {
 
     let (tx, rx) = mpsc::channel();
     let rotator_cfg = cfg.clone();
+    let hashscanner_cfg = cfg.clone();
     match thread::Builder::new()
         .name("FIM_Rotator".to_string()).spawn(|| rotator::rotator(rotator_cfg)){
         Ok(_v) => info!("FIM rotator thread started."),
         Err(e) => error!("Could not start FIM rotator thread, error: {}", e)
     };
+    match thread::Builder::new()
+        .name("FIM_HashScanner".to_string()).spawn(|| hashscanner::scan(hashscanner_cfg)){
+        Ok(_v) => info!("FIM HashScanner thread started."),
+        Err(e) => error!("Could not start FIM HashScanner thread, error: {}", e)
+    };
     monitor::monitor(tx, rx, cfg, ruleset).await;
 }
 
@@ -73,12 +79,19 @@ async fn main() -> windows_service::Result<()> {
                 let (tx, rx) = mpsc::channel();
                 let (cfg, ruleset) = init();
                 let rotator_cfg = cfg.clone();
+                let hashscanner_cfg = cfg.clone();
                 match thread::Builder::new()
                     .name("FIM_Rotator".to_string())
                     .spawn(|| rotator::rotator(rotator_cfg)){
                         Ok(_v) => info!("FIM rotator thread started."),
                         Err(e) => error!("Could not start FIM rotator thread, error: {}", e)
                     };
+                match thread::Builder::new()
+                    .name("FIM_HashScanner".to_string())
+                    .spawn(|| hashscanner::scan(hashscanner_cfg)){
+                        Ok(_v) => info!("FIM HashScanner thread started."),
+                        Err(e) => error!("Could not start FIM HashScanner thread, error: {}", e)
+                    };
                 monitor::monitor(tx, rx, cfg, ruleset).await;
                 Ok(())
             },

diff --git a/src/monitor.rs b/src/monitor.rs
@@ -35,7 +35,6 @@ use crate::logreader;
 // integrations checker
 use crate::launcher;
 use crate::multiwatcher::MultiWatcher;
-use crate::scanner;
 
 // ----------------------------------------------------------------------------
 
@@ -130,12 +129,7 @@ pub async fn monitor(
             }
 
             match watcher.watch(Path::new(path), RecursiveMode::Recursive) {
-                Ok(_d) => {
-                    debug!("Monitoring '{}' path.", path);
-                    debug!("Starting file scan to create hash database.");
-                    scanner::first_scan(cfg.clone(), String::from(path)).await;
-                    debug!("Path '{}' scanned all files are hashed in DB.", path);
-                },
+                Ok(_d) => debug!("Monitoring '{}' path.", path),
                 Err(e) => warn!("Could not monitor given path '{}', description: {}", path, e)
             };
         }

diff --git a/src/utils.rs b/src/utils.rs
@@ -279,7 +279,7 @@ pub fn get_current_time_millis() -> String {
 
 // ----------------------------------------------------------------------------
 
-pub fn get_path_file_list(root: String) -> Vec<String> {
+pub fn get_fs_list(root: String) -> Vec<String> {
     let mut list = Vec::new();
     for result in WalkDir::new(root) {
         list.push(String::from(result.unwrap().path().to_str().unwrap()))