From 45116c56c284b826cffed6dc7ae490370da54bbc Mon Sep 17 00:00:00 2001 From: Toni Peter Date: Wed, 29 Jan 2025 15:56:12 +0100 Subject: [PATCH 1/5] Add filter functionality to scannerctl. This takes as input a path to a feed and prints out those scripts that could presumably be executed with the current state of the rust scanner (filtering out those that either use non-implemented builtins or include other scripts that do.) --- rust/Cargo.lock | 7 + rust/Cargo.toml | 1 + rust/src/nasl/utils/executor/mod.rs | 10 + .../scannerctl/feed/filter/all_builtins.rs | 341 ++++++++++++++++++ rust/src/scannerctl/feed/filter/mod.rs | 222 ++++++++++++ rust/src/scannerctl/feed/mod.rs | 4 + 6 files changed, 585 insertions(+) create mode 100644 rust/src/scannerctl/feed/filter/all_builtins.rs create mode 100644 rust/src/scannerctl/feed/filter/mod.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index f5ecf2591..6aae1787b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2568,6 +2568,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pbkdf2" version = "0.11.0" @@ -3472,6 +3478,7 @@ dependencies = [ "num_cpus", "once_cell", "openssl", + "pathdiff", "pbkdf2 0.12.2", "pcap", "pkcs8", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b700e6415..46a94e38e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -96,6 +96,7 @@ nasl-c-lib = { path = "crates/nasl-c-lib", optional = true } openssl = { version = "0.10.66", features = ["vendored"] } blowfish = "0.9.1" rc4 = "0.1.0" +pathdiff = "0.2.3" [workspace] resolver = "2" diff --git a/rust/src/nasl/utils/executor/mod.rs b/rust/src/nasl/utils/executor/mod.rs index 17406d378..9d3b2a47d 100644 --- a/rust/src/nasl/utils/executor/mod.rs +++ b/rust/src/nasl/utils/executor/mod.rs @@ -73,6 +73,10 @@ impl Executor { pub fn contains(&self, k: &str) -> bool { self.sets.iter().any(|set| set.contains(k)) } + + pub fn iter(&self) -> impl Iterator { + self.sets.iter().flat_map(|set| set.iter()) + } } pub struct StoredFunctionSet { @@ -191,6 +195,8 @@ pub trait FunctionSet { ) -> NaslResult; fn contains(&self, k: &str) -> bool; + + fn iter(&self) -> Box + '_>; } #[async_trait] @@ -227,6 +233,10 @@ impl FunctionSet for StoredFunctionSet { fn contains(&self, k: &str) -> bool { self.fns.contains_key(k) } + + fn iter(&self) -> Box + '_> { + Box::new(self.fns.keys().map(|x| x.as_str())) + } } /// Anything that can be converted into a `StoredFunctionSet`. diff --git a/rust/src/scannerctl/feed/filter/all_builtins.rs b/rust/src/scannerctl/feed/filter/all_builtins.rs new file mode 100644 index 000000000..60abdd276 --- /dev/null +++ b/rust/src/scannerctl/feed/filter/all_builtins.rs @@ -0,0 +1,341 @@ +pub const ALL_BUILTINS: &'static [&'static str] = &[ + "plugin_run_find_service", + "plugin_run_openvas_tcp_scanner", + "plugin_run_synscan", + "cert_close", + "cert_open", + "cert_query", + "aes128_cbc_encrypt", + "aes128_ccm_decrypt_auth", + "aes128_ccm_decrypt", + "aes128_ccm_encrypt_auth", + "aes128_ccm_encrypt", + "aes128_ctr_encrypt", + "aes128_gcm_decrypt_auth", + "aes128_gcm_decrypt", + "aes128_gcm_encrypt_auth", + "aes128_gcm_encrypt", + "aes256_cbc_encrypt", + "aes256_ccm_decrypt_auth", + "aes256_ccm_decrypt", + "aes256_ccm_encrypt_auth", + "aes256_ccm_encrypt", + "aes256_ctr_encrypt", + "aes256_gcm_decrypt_auth", + "aes256_gcm_decrypt", + "aes256_gcm_encrypt_auth", + "aes256_gcm_encrypt", + "aes_mac_cbc", + "aes_mac_gcm", + "bf_cbc_decrypt", + "bf_cbc_encrypt", + "bn_cmp", + "bn_random", + "close_stream_cipher", + "des_ede_cbc_encrypt", + "DES", + "dh_compute_key", + "dh_generate_key", + "dsa_do_sign", + "dsa_do_verify", + "get_signature", + "get_smb2_signature", + "HMAC_MD2", + "HMAC_MD5", + "HMAC_RIPEMD160", + "HMAC_SHA1", + "HMAC_SHA256", + "HMAC_SHA384", + "HMAC_SHA512", + "insert_hexzeros", + "key_exchange", + "lm_owf_gen", + "MD2", + "MD4", + "MD5", + "ntlm2_response", + "ntlm_response", + "NTLMv1_HASH", + "NTLMv2_HASH", + "ntlmv2_response", + "nt_owf_gen", + "ntv2_owf_gen", + "open_rc4_cipher", + "pem_to_dsa", + "pem_to_rsa", + "prf_sha256", + "prf_sha384", + "rc4_encrypt", + "RIPEMD160", + "rsa_private_decrypt", + "rsa_public_decrypt", + "rsa_public_encrypt", + "rsa_sign", + "SHA1", + "SHA256", + "SHA512", + "smb3kdf", + "smb_cmac_aes_signature", + "smb_gmac_aes_signature", + "tls1_prf", + "script_add_preference", + "script_category", + "script_copyright", + "script_cve_id", + "script_dependencies", + "script_exclude_keys", + "script_family", + "script_mandatory_keys", + "script_name", + "script_oid", + "script_require_keys", + "script_require_ports", + "script_require_udp_ports", + "script_tag", + "script_timeout", + "script_version", + "script_xref", + "get_preference", + "get_script_oid", + "script_get_preference_file_content", + "script_get_preference_file_location", + "script_get_preference", + "vendor_version", + "add_host_name", + "get_host_names", + "get_host_name_source", + "resolve_host_name", + "resolve_hostname_to_multiple_ips", + "same_host", + "TARGET_IS_IPV6", + "cgibin", + "http_close_socket", + "http_delete", + "http_get", + "http_head", + "http_open_socket", + "http_post", + "http_put", + "isotime_add", + "isotime_is_valid", + "isotime_now", + "isotime_print", + "isotime_scan", + "get_host_kb_index", + "get_kb_item", + "get_kb_list", + "replace_kb_item", + "set_kb_item", + "dec2str", + "defined_func", + "dump_ctxt", + "exit", + "get_byte_order", + "gettimeofday", + "gunzip", + "gzip", + "isnull", + "keys", + "localtime", + "make_array", + "make_list", + "max_index", + "mktime", + "open_sock_kdc", + "rand", + "safe_checks", + "sleep", + "sort", + "typeof", + "unixtime", + "usleep", + "close", + "end_denial", + "ftp_get_pasv_port", + "ftp_log_in", + "get_host_ip", + "get_host_name", + "get_host_open_port", + "get_mtu", + "get_port_state", + "get_port_transport", + "get_source_port", + "get_tcp_port_state", + "get_udp_port_state", + "islocalhost", + "islocalnet", + "join_multicast_group", + "leave_multicast_group", + "open_priv_sock_tcp", + "open_priv_sock_udp", + "open_sock_tcp", + "open_sock_udp", + "recv_line", + "recv", + "scanner_add_port", + "scanner_get_port", + "send", + "start_denial", + "tcp_ping", + "telnet_init", + "this_host", + "this_host_name", + "dump_frame", + "dump_icmp_packet", + "dump_icmp_v6_packet", + "dump_ip_packet", + "dump_ip_v6_packet", + "dump_ipv6_packet", + "dump_tcp_packet", + "dump_tcp_v6_packet", + "dump_udp_packet", + "dump_udp_v6_packet", + "forge_frame", + "forge_icmp_packet", + "forge_icmp_v6_packet", + "forge_igmp_packet", + "forge_igmp_v6_packet", + "forge_ip_packet", + "forge_ip_v6_packet", + "forge_ipv6_packet", + "forge_tcp_packet", + "forge_tcp_v6_packet", + "forge_udp_packet", + "forge_udp_v6_packet", + "get_icmp_element", + "get_icmp_v6_element", + "get_ip_element", + "get_ip_v6_element", + "get_ipv6_element", + "get_local_mac_address_from_ip", + "get_tcp_element", + "get_tcp_option", + "get_tcp_v6_element", + "get_tcp_v6_option", + "get_udp_element", + "get_udp_v6_element", + "insert_ip_options", + "insert_ip_v6_options", + "insert_ipv6_options", + "insert_tcp_options", + "insert_tcp_v6_options", + "pcap_next", + "send_arp_request", + "send_capture", + "send_frame", + "send_packet", + "send_v6packet", + "set_ip_elements", + "set_ip_v6_elements", + "set_ipv6_elements", + "set_tcp_elements", + "set_tcp_v6_elements", + "set_udp_elements", + "set_udp_v6_elements", + "tcp_ping", + "tcp_v6_ping", + "egrep", + "eregmatch", + "ereg", + "ereg_replace", + "error_message", + "log_message", + "scanner_status", + "security_message", + "smb_close", + "smb_connect", + "smb_file_group_sid", + "smb_file_owner_sid", + "smb_file_SDDL", + "smb_file_trustee_rights", + "smb_versioninfo", + "win_cmd_exec", + "snmpv1_get", + "snmpv1_getnext", + "snmpv2c_get", + "snmpv2c_getnext", + "snmpv3_get", + "snmpv3_getnext", + "sftp_enabled_check", + "ssh_connect", + "ssh_disconnect", + "ssh_get_auth_methods", + "ssh_get_host_key", + "ssh_get_issue_banner", + "ssh_get_server_banner", + "ssh_get_sock", + "ssh_login_interactive", + "ssh_login_interactive_pass", + "ssh_request_exec", + "ssh_session_id_from_sock", + "ssh_set_login", + "ssh_shell_close", + "ssh_shell_open", + "ssh_shell_read", + "ssh_shell_write", + "ssh_userauth", + "chomp", + "crap", + "display", + "hex", + "hexstr", + "insstr", + "int", + "match", + "ord", + "raw_string", + "split", + "strcat", + "stridx", + "string", + "strlen", + "str_replace", + "strstr", + "substr", + "tolower", + "toupper", + "get_sock_info", + "socket_cert_verify", + "socket_check_ssl_safe_renegotiation", + "socket_get_cert", + "socket_get_error", + "socket_get_ssl_ciphersuite", + "socket_get_ssl_session_id", + "socket_get_ssl_version", + "socket_negotiate_ssl", + "socket_ssl_do_handshake", + "file_close", + "file_open", + "file_read", + "file_seek", + "file_stat", + "file_write", + "find_in_path", + "fread", + "fwrite", + "get_tmp_dir", + "pread", + "unlink", + "openvas-smb", + "wmi_close", + "wmi_connect", + "wmi_connect_reg", + "wmi_connect_rsop", + "wmi_query", + "wmi_query_rsop", + "wmi_reg_create_key", + "wmi_reg_delete_key", + "wmi_reg_enum_key", + "wmi_reg_enum_value", + "wmi_reg_get_bin_val", + "wmi_reg_get_dword_val", + "wmi_reg_get_ex_string_val", + "wmi_reg_get_mul_string_val", + "wmi_reg_get_qword_val", + "wmi_reg_get_sz", + "wmi_reg_set_dword_val", + "wmi_reg_set_ex_string_val", + "wmi_reg_set_qword_val", + "wmi_reg_set_string_val", + "wmi_versioninfo", +]; diff --git a/rust/src/scannerctl/feed/filter/mod.rs b/rust/src/scannerctl/feed/filter/mod.rs new file mode 100644 index 000000000..d8c8ef68c --- /dev/null +++ b/rust/src/scannerctl/feed/filter/mod.rs @@ -0,0 +1,222 @@ +mod all_builtins; + +use all_builtins::ALL_BUILTINS; +use regex::Regex; +use scannerlib::nasl::nasl_std_functions; +use std::{ + collections::{HashMap, HashSet}, + fs, + path::{Path, PathBuf}, +}; +use tracing::error; +use walkdir::WalkDir; + +use crate::CliError; + +#[derive(clap::Parser)] +pub struct FilterArgs { + feed_path: PathBuf, +} + +pub type Builtins<'a> = &'a [String]; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Script { + Runnable, + NotRunnable, + Includes(Vec), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ScriptPath(String); + +impl ScriptPath { + fn new(feed_path: &Path, path: &Path) -> Self { + let relative = pathdiff::diff_paths(path, feed_path).unwrap(); + Self(relative.as_os_str().to_str().unwrap().to_string()) + } +} + +impl Script { + fn new(builtins: Builtins, path: &Path) -> Option { + let contents = fs::read_to_string(path); + match contents { + Err(e) => { + error!("Error reading file {path:?}: {e:?}"); + return None; + } + Ok(contents) => Some(Script::from_contents(builtins, &contents)), + } + } + + pub fn from_contents(builtins: Builtins, contents: &str) -> Self { + let is_runnable = builtins.iter().all(|builtin| !contents.contains(builtin)); + let includes = get_includes(contents); + if !is_runnable { + Self::NotRunnable + } else if includes.is_empty() { + Self::Runnable + } else { + Self::Includes(includes) + } + } + + fn includes(&self) -> impl Iterator { + if let Self::Includes(includes) = self { + includes.iter() + } else { + panic!("includes() called on non-includes variant") + } + } +} + +fn get_includes(contents: &str) -> Vec { + let mut includes = vec![]; + for line in contents.lines() { + if line.contains("include") { + if let Some(include) = parse_include(line) { + includes.push(include); + } + } + } + includes +} + +fn parse_include(line: &str) -> Option { + let re = Regex::new(r#"\binclude\("([^"]+)"\)"#).unwrap(); + if let Some(capture) = re.captures_iter(line).next() { + if let Some(match_) = capture.get(1) { + return Some(ScriptPath(match_.as_str().to_string())); + } + } + None +} + +fn resolve_includes(mut unresolved: HashMap) -> HashMap { + let mut resolved: HashMap = HashMap::default(); + loop { + let (newly_resolved, newly_unresolved) = unresolved + .into_iter() + .partition::, _>(|(_, script)| { + matches!(script, Script::Runnable | Script::NotRunnable) + }); + unresolved = newly_unresolved; + let new_length = newly_resolved.len(); + resolved.extend(newly_resolved.into_iter()); + if unresolved.is_empty() { + return resolved; + } + if new_length == 0 { + panic!("Unresolvable."); + } + for v in unresolved.values_mut() { + let any_not_runnable = v + .includes() + .any(|include| matches!(resolved.get(include), Some(Script::NotRunnable))); + if any_not_runnable { + *v = Script::NotRunnable; + continue; + } + let all_resolved = v.includes().all(|include| resolved.contains_key(include)); + if !all_resolved { + continue; + } + *v = Script::Runnable; + } + } +} + +fn get_unimplemented_builtins() -> Vec { + let exec = nasl_std_functions(); + let implemented: Vec<_> = exec.iter().collect(); + let mut unimplemented: HashSet<_> = ALL_BUILTINS.iter().map(|x| x.to_string()).collect(); + for f in implemented { + unimplemented.remove(f); + } + unimplemented.into_iter().collect() +} + +pub fn run(args: FilterArgs) -> Result<(), CliError> { + let mut scripts = HashMap::new(); + let builtins = get_unimplemented_builtins(); + for entry in WalkDir::new(&args.feed_path).into_iter() { + let entry = entry.unwrap(); + if entry.file_type().is_file() { + let path = entry.path(); + if let Some(script) = Script::new(&builtins, path) { + let script_path = ScriptPath::new(&args.feed_path, path); + scripts.insert(script_path, script); + } + } + } + let resolved = resolve_includes(scripts); + for (path, script) in resolved.into_iter() { + if matches!(script, Script::Runnable) { + println!("{:?}", &path.0); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::feed::filter::{resolve_includes, ScriptPath}; + + use super::Script; + use super::Script::*; + + fn path(path_str: &str) -> ScriptPath { + ScriptPath(path_str.to_string()) + } + + fn includes(paths: &[&str]) -> Script { + Includes(paths.into_iter().map(|p| path(p)).collect()) + } + + fn make_scripts(scripts: &[(&str, Script)]) -> HashMap { + scripts + .into_iter() + .map(|(name, script)| (ScriptPath(name.to_string()), script.clone())) + .collect() + } + + #[test] + fn resolution() { + let scripts = [ + ("a", Runnable), + ("b", Runnable), + ("c", NotRunnable), + ("d", includes(&["a", "b", "c"])), + ]; + let resolved = resolve_includes(make_scripts(&scripts)); + assert_eq!(resolved[&path("a")], Runnable); + assert_eq!(resolved[&path("b")], Runnable); + assert_eq!(resolved[&path("c")], NotRunnable); + assert_eq!(resolved[&path("d")], NotRunnable); + } + + #[test] + fn transitive_resolution() { + let scripts = [ + ("a1", Runnable), + ("b1", Includes(vec![path("a1")])), + ("c1", Includes(vec![path("b1")])), + ("d1", Includes(vec![path("c1")])), + ("a2", NotRunnable), + ("b2", Includes(vec![path("a2")])), + ("c2", Includes(vec![path("b2")])), + ("d2", Includes(vec![path("c2")])), + ]; + let resolved = resolve_includes(make_scripts(&scripts)); + assert_eq!(resolved[&path("a1")], Runnable); + assert_eq!(resolved[&path("b1")], Runnable); + assert_eq!(resolved[&path("c1")], Runnable); + assert_eq!(resolved[&path("d1")], Runnable); + assert_eq!(resolved[&path("a2")], NotRunnable); + assert_eq!(resolved[&path("b2")], NotRunnable); + assert_eq!(resolved[&path("c2")], NotRunnable); + assert_eq!(resolved[&path("d2")], NotRunnable); + } +} diff --git a/rust/src/scannerctl/feed/mod.rs b/rust/src/scannerctl/feed/mod.rs index 861d64d18..91abf9679 100644 --- a/rust/src/scannerctl/feed/mod.rs +++ b/rust/src/scannerctl/feed/mod.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception +mod filter; mod transpile; pub mod update; use std::{ @@ -12,6 +13,7 @@ use std::{ // re-export to work around name conflict use clap::Subcommand; +use filter::FilterArgs; use scannerlib::{ nasl::{syntax::LoadError, WithErrorInfo}, storage::{ @@ -42,6 +44,7 @@ enum Action { Update(UpdateArgs), Transform(TransformArgs), Transpile(TranspileArgs), + Filter(FilterArgs), } /// Runs nasl scripts in description mode and updates data into Redis @@ -173,6 +176,7 @@ pub async fn run(args: FeedArgs) -> Result<(), CliError> { Action::Update(args) => update(args).await?, Action::Transform(args) => transform(args).await?, Action::Transpile(args) => transpile::run(args).await?, + Action::Filter(args) => filter::run(args)?, } Ok(()) } From 1dab963c2e580b6617288ee0e8de1ad5dfa600bf Mon Sep 17 00:00:00 2001 From: Toni Peter Date: Thu, 30 Jan 2025 12:12:55 +0100 Subject: [PATCH 2/5] Remove 'exit' from the list of builtins. --- rust/doc/misc/builtin_coverage.nasl | 2 +- rust/src/scannerctl/feed/filter/all_builtins.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/doc/misc/builtin_coverage.nasl b/rust/doc/misc/builtin_coverage.nasl index a8fcf45e3..f0cf8494d 100644 --- a/rust/doc/misc/builtin_coverage.nasl +++ b/rust/doc/misc/builtin_coverage.nasl @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception -functions = make_array("built-in-plugins",make_list("plugin_run_find_service","plugin_run_openvas_tcp_scanner","plugin_run_synscan"),"cert-functions",make_list("cert_close","cert_open","cert_query"),"cryptographic",make_list("aes128_cbc_encrypt","aes128_ccm_decrypt_auth","aes128_ccm_decrypt","aes128_ccm_encrypt_auth","aes128_ccm_encrypt","aes128_ctr_encrypt","aes128_gcm_decrypt_auth","aes128_gcm_decrypt","aes128_gcm_encrypt_auth","aes128_gcm_encrypt","aes256_cbc_encrypt","aes256_ccm_decrypt_auth","aes256_ccm_decrypt","aes256_ccm_encrypt_auth","aes256_ccm_encrypt","aes256_ctr_encrypt","aes256_gcm_decrypt_auth","aes256_gcm_decrypt","aes256_gcm_encrypt_auth","aes256_gcm_encrypt","aes_mac_cbc","aes_mac_gcm","bf_cbc_decrypt","bf_cbc_encrypt","bn_cmp","bn_random","close_stream_cipher","des_ede_cbc_encrypt","DES","dh_compute_key","dh_generate_key","dsa_do_sign","dsa_do_verify","get_signature","get_smb2_signature","HMAC_MD2","HMAC_MD5","HMAC_RIPEMD160","HMAC_SHA1","HMAC_SHA256","HMAC_SHA384","HMAC_SHA512","insert_hexzeros","key_exchange","lm_owf_gen","MD2","MD4","MD5","ntlm2_response","ntlm_response","NTLMv1_HASH","NTLMv2_HASH","ntlmv2_response","nt_owf_gen","ntv2_owf_gen","open_rc4_cipher","pem_to_dsa","pem_to_rsa","prf_sha256","prf_sha384","rc4_encrypt","RIPEMD160","rsa_private_decrypt","rsa_public_decrypt","rsa_public_encrypt","rsa_sign","SHA1","SHA256","SHA512","smb3kdf","smb_cmac_aes_signature","smb_gmac_aes_signature","tls1_prf"),"description-functions",make_list("script_add_preference","script_category","script_copyright","script_cve_id","script_dependencies","script_exclude_keys","script_family","script_mandatory_keys","script_name","script_oid","script_require_keys","script_require_ports","script_require_udp_ports","script_tag","script_timeout","script_version","script_xref"),"glue-functions",make_list("get_preference","get_script_oid","script_get_preference_file_content","script_get_preference_file_location","script_get_preference","vendor_version"),"host-functions",make_list("add_host_name","get_host_names","get_host_name_source","resolve_host_name","resolve_hostname_to_multiple_ips","same_host","TARGET_IS_IPV6"),"http-functions",make_list("cgibin","http_close_socket","http_delete","http_get","http_head","http_open_socket","http_post","http_put"),"isotime-functions",make_list("isotime_add","isotime_is_valid","isotime_now","isotime_print","isotime_scan"),"knowledge-base",make_list("get_host_kb_index","get_kb_item","get_kb_list","replace_kb_item","set_kb_item"),"misc",make_list("dec2str","defined_func","dump_ctxt","exit","get_byte_order","gettimeofday","gunzip","gzip","isnull","keys","localtime","make_array","make_list","max_index","mktime","open_sock_kdc","rand","safe_checks","sleep","sort","typeof","unixtime","usleep"),"network-functions",make_list("close","end_denial","ftp_get_pasv_port","ftp_log_in","get_host_ip","get_host_name","get_host_open_port","get_mtu","get_port_state","get_port_transport","get_source_port","get_tcp_port_state","get_udp_port_state","islocalhost","islocalnet","join_multicast_group","leave_multicast_group","open_priv_sock_tcp","open_priv_sock_udp","open_sock_tcp","open_sock_udp","recv_line","recv","scanner_add_port","scanner_get_port","send","start_denial","tcp_ping","telnet_init","this_host","this_host_name"),"raw-ip-functions",make_list("dump_frame","dump_icmp_packet","dump_icmp_v6_packet","dump_ip_packet","dump_ip_v6_packet","dump_ipv6_packet","dump_tcp_packet","dump_tcp_v6_packet","dump_udp_packet","dump_udp_v6_packet","forge_frame","forge_icmp_packet","forge_icmp_v6_packet","forge_igmp_packet","forge_igmp_v6_packet","forge_ip_packet","forge_ip_v6_packet","forge_ipv6_packet","forge_tcp_packet","forge_tcp_v6_packet","forge_udp_packet","forge_udp_v6_packet","get_icmp_element","get_icmp_v6_element","get_ip_element","get_ip_v6_element","get_ipv6_element","get_local_mac_address_from_ip","get_tcp_element","get_tcp_option","get_tcp_v6_element","get_tcp_v6_option","get_udp_element","get_udp_v6_element","insert_ip_options","insert_ip_v6_options","insert_ipv6_options","insert_tcp_options","insert_tcp_v6_options","pcap_next","send_arp_request","send_capture","send_frame","send_packet","send_v6packet","set_ip_elements","set_ip_v6_elements","set_ipv6_elements","set_tcp_elements","set_tcp_v6_elements","set_udp_elements","set_udp_v6_elements","tcp_ping","tcp_v6_ping"),"regular-expressions",make_list("egrep","eregmatch","ereg","ereg_replace"),"report-functions",make_list("error_message","log_message","scanner_status","security_message"),"smb-functions",make_list("smb_close","smb_connect","smb_file_group_sid","smb_file_owner_sid","smb_file_SDDL","smb_file_trustee_rights","smb_versioninfo","win_cmd_exec"),"snmp-functions",make_list("snmpv1_get","snmpv1_getnext","snmpv2c_get","snmpv2c_getnext","snmpv3_get","snmpv3_getnext"),"ssh-functions",make_list("sftp_enabled_check","ssh_connect","ssh_disconnect","ssh_get_auth_methods","ssh_get_host_key","ssh_get_issue_banner","ssh_get_server_banner","ssh_get_sock","ssh_login_interactive","ssh_login_interactive_pass","ssh_request_exec","ssh_session_id_from_sock","ssh_set_login","ssh_shell_close","ssh_shell_open","ssh_shell_read","ssh_shell_write","ssh_userauth"),"string-functions",make_list("chomp","crap","display","hex","hexstr","insstr","int","match","ord","raw_string","split","strcat","stridx","string","strlen","str_replace","strstr","substr","tolower","toupper"),"tls",make_list("get_sock_info","socket_cert_verify","socket_check_ssl_safe_renegotiation","socket_get_cert","socket_get_error","socket_get_ssl_ciphersuite","socket_get_ssl_session_id","socket_get_ssl_version","socket_negotiate_ssl","socket_ssl_do_handshake"),"unsafe",make_list("file_close","file_open","file_read","file_seek","file_stat","file_write","find_in_path","fread","fwrite","get_tmp_dir","pread","unlink"),"wmi-functions",make_list("openvas-smb","wmi_close","wmi_connect","wmi_connect_reg","wmi_connect_rsop","wmi_query","wmi_query_rsop","wmi_reg_create_key","wmi_reg_delete_key","wmi_reg_enum_key","wmi_reg_enum_value","wmi_reg_get_bin_val","wmi_reg_get_dword_val","wmi_reg_get_ex_string_val","wmi_reg_get_mul_string_val","wmi_reg_get_qword_val","wmi_reg_get_sz","wmi_reg_set_dword_val","wmi_reg_set_ex_string_val","wmi_reg_set_qword_val","wmi_reg_set_string_val","wmi_versioninfo")); +functions = make_array("built-in-plugins",make_list("plugin_run_find_service","plugin_run_openvas_tcp_scanner","plugin_run_synscan"),"cert-functions",make_list("cert_close","cert_open","cert_query"),"cryptographic",make_list("aes128_cbc_encrypt","aes128_ccm_decrypt_auth","aes128_ccm_decrypt","aes128_ccm_encrypt_auth","aes128_ccm_encrypt","aes128_ctr_encrypt","aes128_gcm_decrypt_auth","aes128_gcm_decrypt","aes128_gcm_encrypt_auth","aes128_gcm_encrypt","aes256_cbc_encrypt","aes256_ccm_decrypt_auth","aes256_ccm_decrypt","aes256_ccm_encrypt_auth","aes256_ccm_encrypt","aes256_ctr_encrypt","aes256_gcm_decrypt_auth","aes256_gcm_decrypt","aes256_gcm_encrypt_auth","aes256_gcm_encrypt","aes_mac_cbc","aes_mac_gcm","bf_cbc_decrypt","bf_cbc_encrypt","bn_cmp","bn_random","close_stream_cipher","des_ede_cbc_encrypt","DES","dh_compute_key","dh_generate_key","dsa_do_sign","dsa_do_verify","get_signature","get_smb2_signature","HMAC_MD2","HMAC_MD5","HMAC_RIPEMD160","HMAC_SHA1","HMAC_SHA256","HMAC_SHA384","HMAC_SHA512","insert_hexzeros","key_exchange","lm_owf_gen","MD2","MD4","MD5","ntlm2_response","ntlm_response","NTLMv1_HASH","NTLMv2_HASH","ntlmv2_response","nt_owf_gen","ntv2_owf_gen","open_rc4_cipher","pem_to_dsa","pem_to_rsa","prf_sha256","prf_sha384","rc4_encrypt","RIPEMD160","rsa_private_decrypt","rsa_public_decrypt","rsa_public_encrypt","rsa_sign","SHA1","SHA256","SHA512","smb3kdf","smb_cmac_aes_signature","smb_gmac_aes_signature","tls1_prf"),"description-functions",make_list("script_add_preference","script_category","script_copyright","script_cve_id","script_dependencies","script_exclude_keys","script_family","script_mandatory_keys","script_name","script_oid","script_require_keys","script_require_ports","script_require_udp_ports","script_tag","script_timeout","script_version","script_xref"),"glue-functions",make_list("get_preference","get_script_oid","script_get_preference_file_content","script_get_preference_file_location","script_get_preference","vendor_version"),"host-functions",make_list("add_host_name","get_host_names","get_host_name_source","resolve_host_name","resolve_hostname_to_multiple_ips","same_host","TARGET_IS_IPV6"),"http-functions",make_list("cgibin","http_close_socket","http_delete","http_get","http_head","http_open_socket","http_post","http_put"),"isotime-functions",make_list("isotime_add","isotime_is_valid","isotime_now","isotime_print","isotime_scan"),"knowledge-base",make_list("get_host_kb_index","get_kb_item","get_kb_list","replace_kb_item","set_kb_item"),"misc",make_list("dec2str","defined_func","dump_ctxt","get_byte_order","gettimeofday","gunzip","gzip","isnull","keys","localtime","make_array","make_list","max_index","mktime","open_sock_kdc","rand","safe_checks","sleep","sort","typeof","unixtime","usleep"),"network-functions",make_list("close","end_denial","ftp_get_pasv_port","ftp_log_in","get_host_ip","get_host_name","get_host_open_port","get_mtu","get_port_state","get_port_transport","get_source_port","get_tcp_port_state","get_udp_port_state","islocalhost","islocalnet","join_multicast_group","leave_multicast_group","open_priv_sock_tcp","open_priv_sock_udp","open_sock_tcp","open_sock_udp","recv_line","recv","scanner_add_port","scanner_get_port","send","start_denial","tcp_ping","telnet_init","this_host","this_host_name"),"raw-ip-functions",make_list("dump_frame","dump_icmp_packet","dump_icmp_v6_packet","dump_ip_packet","dump_ip_v6_packet","dump_ipv6_packet","dump_tcp_packet","dump_tcp_v6_packet","dump_udp_packet","dump_udp_v6_packet","forge_frame","forge_icmp_packet","forge_icmp_v6_packet","forge_igmp_packet","forge_igmp_v6_packet","forge_ip_packet","forge_ip_v6_packet","forge_ipv6_packet","forge_tcp_packet","forge_tcp_v6_packet","forge_udp_packet","forge_udp_v6_packet","get_icmp_element","get_icmp_v6_element","get_ip_element","get_ip_v6_element","get_ipv6_element","get_local_mac_address_from_ip","get_tcp_element","get_tcp_option","get_tcp_v6_element","get_tcp_v6_option","get_udp_element","get_udp_v6_element","insert_ip_options","insert_ip_v6_options","insert_ipv6_options","insert_tcp_options","insert_tcp_v6_options","pcap_next","send_arp_request","send_capture","send_frame","send_packet","send_v6packet","set_ip_elements","set_ip_v6_elements","set_ipv6_elements","set_tcp_elements","set_tcp_v6_elements","set_udp_elements","set_udp_v6_elements","tcp_ping","tcp_v6_ping"),"regular-expressions",make_list("egrep","eregmatch","ereg","ereg_replace"),"report-functions",make_list("error_message","log_message","scanner_status","security_message"),"smb-functions",make_list("smb_close","smb_connect","smb_file_group_sid","smb_file_owner_sid","smb_file_SDDL","smb_file_trustee_rights","smb_versioninfo","win_cmd_exec"),"snmp-functions",make_list("snmpv1_get","snmpv1_getnext","snmpv2c_get","snmpv2c_getnext","snmpv3_get","snmpv3_getnext"),"ssh-functions",make_list("sftp_enabled_check","ssh_connect","ssh_disconnect","ssh_get_auth_methods","ssh_get_host_key","ssh_get_issue_banner","ssh_get_server_banner","ssh_get_sock","ssh_login_interactive","ssh_login_interactive_pass","ssh_request_exec","ssh_session_id_from_sock","ssh_set_login","ssh_shell_close","ssh_shell_open","ssh_shell_read","ssh_shell_write","ssh_userauth"),"string-functions",make_list("chomp","crap","display","hex","hexstr","insstr","int","match","ord","raw_string","split","strcat","stridx","string","strlen","str_replace","strstr","substr","tolower","toupper"),"tls",make_list("get_sock_info","socket_cert_verify","socket_check_ssl_safe_renegotiation","socket_get_cert","socket_get_error","socket_get_ssl_ciphersuite","socket_get_ssl_session_id","socket_get_ssl_version","socket_negotiate_ssl","socket_ssl_do_handshake"),"unsafe",make_list("file_close","file_open","file_read","file_seek","file_stat","file_write","find_in_path","fread","fwrite","get_tmp_dir","pread","unlink"),"wmi-functions",make_list("openvas-smb","wmi_close","wmi_connect","wmi_connect_reg","wmi_connect_rsop","wmi_query","wmi_query_rsop","wmi_reg_create_key","wmi_reg_delete_key","wmi_reg_enum_key","wmi_reg_enum_value","wmi_reg_get_bin_val","wmi_reg_get_dword_val","wmi_reg_get_ex_string_val","wmi_reg_get_mul_string_val","wmi_reg_get_qword_val","wmi_reg_get_sz","wmi_reg_set_dword_val","wmi_reg_set_ex_string_val","wmi_reg_set_qword_val","wmi_reg_set_string_val","wmi_versioninfo")); categories = make_list("built-in-plugins","cert-functions","cryptographic","description-functions","glue-functions","host-functions","http-functions","isotime-functions","knowledge-base","misc","network-functions","raw-ip-functions","regular-expressions","report-functions","smb-functions","snmp-functions","ssh-functions","string-functions","tls","unsafe","wmi-functions"); covered = make_array(); diff --git a/rust/src/scannerctl/feed/filter/all_builtins.rs b/rust/src/scannerctl/feed/filter/all_builtins.rs index 60abdd276..6e7923da1 100644 --- a/rust/src/scannerctl/feed/filter/all_builtins.rs +++ b/rust/src/scannerctl/feed/filter/all_builtins.rs @@ -129,7 +129,6 @@ pub const ALL_BUILTINS: &'static [&'static str] = &[ "dec2str", "defined_func", "dump_ctxt", - "exit", "get_byte_order", "gettimeofday", "gunzip", From 0323d371249255a1811545712dfb73c341a3d1c8 Mon Sep 17 00:00:00 2001 From: Toni Peter Date: Thu, 30 Jan 2025 12:39:04 +0100 Subject: [PATCH 3/5] Write feed-filter outputs to file. --- rust/src/scannerctl/feed/filter/mod.rs | 124 +++++++++++++++++-------- 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/rust/src/scannerctl/feed/filter/mod.rs b/rust/src/scannerctl/feed/filter/mod.rs index d8c8ef68c..63ae9e0a4 100644 --- a/rust/src/scannerctl/feed/filter/mod.rs +++ b/rust/src/scannerctl/feed/filter/mod.rs @@ -3,10 +3,12 @@ mod all_builtins; use all_builtins::ALL_BUILTINS; use regex::Regex; use scannerlib::nasl::nasl_std_functions; +use std::io::{self, Write}; +use std::path::PathBuf; use std::{ collections::{HashMap, HashSet}, - fs, - path::{Path, PathBuf}, + fs::{self, OpenOptions}, + path::Path, }; use tracing::error; use walkdir::WalkDir; @@ -16,12 +18,51 @@ use crate::CliError; #[derive(clap::Parser)] pub struct FilterArgs { feed_path: PathBuf, + output_file: PathBuf, } -pub type Builtins<'a> = &'a [String]; +struct Builtins { + builtins: Vec, + counts: HashMap, +} + +impl Builtins { + fn unimplemented() -> Self { + let exec = nasl_std_functions(); + let implemented: Vec<_> = exec.iter().collect(); + let mut unimplemented: HashSet<_> = ALL_BUILTINS.iter().map(|x| x.to_string()).collect(); + for f in implemented { + unimplemented.remove(f); + } + let counts = unimplemented.iter().map(|name| (name.clone(), 0)).collect(); + Self { + builtins: unimplemented.into_iter().collect(), + counts, + } + } + + fn script_is_runnable(&mut self, contents: &str) -> bool { + let mut is_runnable = true; + for builtin in self.builtins.iter() { + if contents.contains(builtin) { + is_runnable = false; + *self.counts.get_mut(builtin).unwrap() += 1; + } + } + is_runnable + } + + fn print_counts(&self) { + let mut sorted: Vec<_> = self.counts.iter().collect(); + sorted.sort_by_key(|(_, count)| **count); + for (builtin, count) in sorted.iter() { + println!("{builtin} {count}"); + } + } +} #[derive(Clone, Debug, PartialEq, Eq)] -pub enum Script { +enum Script { Runnable, NotRunnable, Includes(Vec), @@ -38,7 +79,7 @@ impl ScriptPath { } impl Script { - fn new(builtins: Builtins, path: &Path) -> Option { + fn new(builtins: &mut Builtins, path: &Path) -> Option { let contents = fs::read_to_string(path); match contents { Err(e) => { @@ -49,8 +90,8 @@ impl Script { } } - pub fn from_contents(builtins: Builtins, contents: &str) -> Self { - let is_runnable = builtins.iter().all(|builtin| !contents.contains(builtin)); + pub fn from_contents(builtins: &mut Builtins, contents: &str) -> Self { + let is_runnable = builtins.script_is_runnable(contents); let includes = get_includes(contents); if !is_runnable { Self::NotRunnable @@ -92,6 +133,43 @@ fn parse_include(line: &str) -> Option { None } +fn write_scripts( + output_file: &Path, + resolved: HashMap, +) -> Result<(), io::Error> { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .create(true) + .open(output_file) + .unwrap(); + for (path, script) in resolved.into_iter() { + if matches!(script, Script::Runnable) { + writeln!(file, "{:?}", &path.0)?; + } + } + Ok(()) +} + +pub fn run(args: FilterArgs) -> Result<(), CliError> { + let mut scripts = HashMap::new(); + let mut builtins = Builtins::unimplemented(); + for entry in WalkDir::new(&args.feed_path).into_iter() { + let entry = entry.unwrap(); + if entry.file_type().is_file() { + let path = entry.path(); + if let Some(script) = Script::new(&mut builtins, path) { + let script_path = ScriptPath::new(&args.feed_path, path); + scripts.insert(script_path, script); + } + } + } + let resolved = resolve_includes(scripts); + write_scripts(&args.output_file, resolved)?; + builtins.print_counts(); + Ok(()) +} + fn resolve_includes(mut unresolved: HashMap) -> HashMap { let mut resolved: HashMap = HashMap::default(); loop { @@ -126,38 +204,6 @@ fn resolve_includes(mut unresolved: HashMap) -> HashMap Vec { - let exec = nasl_std_functions(); - let implemented: Vec<_> = exec.iter().collect(); - let mut unimplemented: HashSet<_> = ALL_BUILTINS.iter().map(|x| x.to_string()).collect(); - for f in implemented { - unimplemented.remove(f); - } - unimplemented.into_iter().collect() -} - -pub fn run(args: FilterArgs) -> Result<(), CliError> { - let mut scripts = HashMap::new(); - let builtins = get_unimplemented_builtins(); - for entry in WalkDir::new(&args.feed_path).into_iter() { - let entry = entry.unwrap(); - if entry.file_type().is_file() { - let path = entry.path(); - if let Some(script) = Script::new(&builtins, path) { - let script_path = ScriptPath::new(&args.feed_path, path); - scripts.insert(script_path, script); - } - } - } - let resolved = resolve_includes(scripts); - for (path, script) in resolved.into_iter() { - if matches!(script, Script::Runnable) { - println!("{:?}", &path.0); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use std::collections::HashMap; From 1f240f31640c88031ccb586fc34d73f5228f4a53 Mon Sep 17 00:00:00 2001 From: Toni Peter Date: Thu, 30 Jan 2025 12:56:35 +0100 Subject: [PATCH 4/5] Create sub-feed directly instead of printing --- rust/src/scannerctl/feed/filter/mod.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/rust/src/scannerctl/feed/filter/mod.rs b/rust/src/scannerctl/feed/filter/mod.rs index 63ae9e0a4..1b17ebe6b 100644 --- a/rust/src/scannerctl/feed/filter/mod.rs +++ b/rust/src/scannerctl/feed/filter/mod.rs @@ -3,11 +3,11 @@ mod all_builtins; use all_builtins::ALL_BUILTINS; use regex::Regex; use scannerlib::nasl::nasl_std_functions; -use std::io::{self, Write}; +use std::io::{self}; use std::path::PathBuf; use std::{ collections::{HashMap, HashSet}, - fs::{self, OpenOptions}, + fs, path::Path, }; use tracing::error; @@ -133,19 +133,18 @@ fn parse_include(line: &str) -> Option { None } -fn write_scripts( - output_file: &Path, +fn copy_feed( + feed_path: &Path, + output_path: &Path, resolved: HashMap, ) -> Result<(), io::Error> { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(output_file) - .unwrap(); - for (path, script) in resolved.into_iter() { + fs::create_dir(output_path)?; + for (path, script) in resolved { if matches!(script, Script::Runnable) { - writeln!(file, "{:?}", &path.0)?; + let src = feed_path.join(&path.0); + let dst = output_path.join(&path.0); + fs::create_dir_all(dst.parent().unwrap())?; + std::fs::copy(src, dst)?; } } Ok(()) @@ -165,7 +164,7 @@ pub fn run(args: FilterArgs) -> Result<(), CliError> { } } let resolved = resolve_includes(scripts); - write_scripts(&args.output_file, resolved)?; + copy_feed(&args.feed_path, &args.output_file, resolved)?; builtins.print_counts(); Ok(()) } From 01ea620208799e998e25f8778e00b13cb5cad1ac Mon Sep 17 00:00:00 2001 From: Toni Peter Date: Thu, 30 Jan 2025 14:07:11 +0100 Subject: [PATCH 5/5] Take script dependencies into account. --- rust/src/scannerctl/feed/filter/mod.rs | 175 +++++++++++++++---------- 1 file changed, 103 insertions(+), 72 deletions(-) diff --git a/rust/src/scannerctl/feed/filter/mod.rs b/rust/src/scannerctl/feed/filter/mod.rs index 1b17ebe6b..61ca2bbb6 100644 --- a/rust/src/scannerctl/feed/filter/mod.rs +++ b/rust/src/scannerctl/feed/filter/mod.rs @@ -65,7 +65,7 @@ impl Builtins { enum Script { Runnable, NotRunnable, - Includes(Vec), + Dependencies(Vec), } #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -79,94 +79,100 @@ impl ScriptPath { } impl Script { - fn new(builtins: &mut Builtins, path: &Path) -> Option { + fn iter_dependencies(&self) -> impl Iterator { + if let Self::Dependencies(dependencies) = self { + dependencies.iter() + } else { + panic!("iter_dependencies() called on non-dependencies variant") + } + } +} + +struct FeedFilter { + builtins: Builtins, + feed_path: PathBuf, + output_path: PathBuf, + script_dependencies_regex: Regex, + include_regex: Regex, +} + +impl FeedFilter { + fn new(feed_path: PathBuf, output_path: PathBuf) -> Self { + let builtins = Builtins::unimplemented(); + let include_regex = Regex::new(r#"\binclude\("([^"]+)"\)"#).unwrap(); + let script_dependencies_regex = + Regex::new(r#"\bscript_dependencies\("([^"]+)"\)"#).unwrap(); + Self { + builtins, + feed_path, + output_path, + script_dependencies_regex, + include_regex, + } + } + + fn script_from_path(&mut self, path: &Path) -> Option