diff --git a/etc/saq.default.ini b/etc/saq.default.ini index 8882e96f..1662efa2 100644 --- a/etc/saq.default.ini +++ b/etc/saq.default.ini @@ -873,6 +873,8 @@ use_proxy = yes #>>> FIELDS #['IP', 'ASN', 'ORG', 'Continent', 'Country', 'Region', 'City', 'Time Zone', 'Latitude', 'Longitude', 'Accuracy Radius'] tag_list = Country +; Point to a local YAML or JSON config for any customizations +override_config_path = [analysis_module_mailbox_email_analyzer] module = saq.modules.email @@ -1008,6 +1010,18 @@ enabled = yes ; use this if you are in AWS and your target is inside target network ssh_host = +[analysis_module_parse_url] +; Parse URL and add FQDN observable +module = saq.modules.url +class = ParseURLAnalyzer +enabled = no + +[analysis_module_fqdn_analyzer] +; Add ip observables for FQDN resolutions +module = saq.modules.dns +class = FQDNAnalyzer +enabled = no + [analysis_module_dns_analyzer] module = saq.modules.asset class = DNSAnalyzer diff --git a/lib/saq/modules/dns.py b/lib/saq/modules/dns.py index d19eb088..768050bd 100644 --- a/lib/saq/modules/dns.py +++ b/lib/saq/modules/dns.py @@ -3,6 +3,7 @@ import csv import logging import os.path +import socket from urllib.parse import urlparse @@ -10,7 +11,7 @@ from saq.analysis import Analysis, Observable from saq.constants import * -from saq.modules import SplunkAnalysisModule, splunktime_to_saqtime +from saq.modules import AnalysisModule, SplunkAnalysisModule, splunktime_to_saqtime KEY_SOURCE_COUNT = 'src_count' KEY_REQUEST_BREAKDOWN = 'request_breakdown' @@ -20,6 +21,52 @@ KEY_REQUEST_BREAKDOWN_TOTAL_COUNT = 'total_count' KEY_DNS_REQUESTS = 'dns_requests' + +class FQDNAnalysis(Analysis): + """What IP adderss does this FQDN resolve to?""" + + def initialize_details(self): + self.details = { 'ip_address': None, + 'resolution_count': None, + 'aliaslist': [], + 'all_resolutions': []} + + def generate_summary(self): + message = f"Resolved to {self.details['ip_address']}" + if self.details['resolution_count'] > 1: + message += f", and {self.details['resolution_count']-1} other IP addresses" + return message + +class FQDNAnalyzer(AnalysisModule): + """What IP address does this FQDN resolve to?""" + # Add anything else you want to this FQDN Analyzer. + + @property + def generated_analysis_type(self): + return FQDNAnalysis + + @property + def valid_observable_types(self): + return F_FQDN + + def execute_analysis(self, observable): + try: + _hostname, _aliaslist, ipaddrlist = socket.gethostbyname_ex(observable.value) + if ipaddrlist: + # ipaddrlist should always be a list of strings + analysis = self.create_analysis(observable) + analysis.details['resolution_count'] = len(ipaddrlist) + analysis.details['all_resolutions'] = ipaddrlist + analysis.details['aliaslist'] = _aliaslist + # for now, just add the first ip address + analysis.details['ip_address'] = ipaddrlist[0] + analysis.add_observable(F_IPV4, ipaddrlist[0]) + return True + return False + except Exception as e: + logging.error(f"Problem resolving FQDN: {e}") + return False + # # Module: DNS Request Analysis # Question: Who requested DNS resolution for this FQDN? diff --git a/lib/saq/modules/ip_address.py b/lib/saq/modules/ip_address.py index 7037c883..f25fb5a8 100644 --- a/lib/saq/modules/ip_address.py +++ b/lib/saq/modules/ip_address.py @@ -1,8 +1,9 @@ - +import os import sys import logging from ip_inspector import maxmind +from ip_inspector.config import load as load_ipi_config from ip_inspector import Inspector, Inspected_IP import saq @@ -110,6 +111,10 @@ class IPIAnalyzer(AnalysisModule): """Lookup an IP address in MaxMind's free GeoLite2 databases and wrap those results around a whitelist/blacklist check. """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__ipi_config = None + @property def generated_analysis_type(self): return IpInspectorAnalysis @@ -127,6 +132,54 @@ def tag_list(self): tag_list = self.config['tag_list'] return tag_list.split(',') + @property + def override_config_path(self): + if 'override_config_path' not in self.config: + logging.warning("Missing expected default config field.") + return False + ocp = self.config['override_config_path'] + if not ocp: + # value not set + return None + if os.path.exists(ocp): + return ocp + ocp = os.path.join(saq.SAQ_HOME, ocp) + if os.path.exists(ocp): + return ocp + logging.warning("Can't find '{}'".format(self.config['override_config_path'])) + return False + + @property + def ipi_config(self): + if not self.__ipi_config: + if self.override_config_path: + self.__ipi_config = load_ipi_config(saved_config_path=self.override_config_path) + else: + self.__ipi_config = load_ipi_config() + return self.__ipi_config + + @property + def blacklist_maps(self): + _bl_map = {} + for bl_type, bl_path in self.ipi_config['default']['blacklists'].items(): + _bl_map[bl_type] = bl_path + if os.path.exists(bl_path): + continue + if os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)): + _bl_map[bl_type] = os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)) + return _bl_map + + @property + def whitelist_maps(self): + _bl_map = {} + for bl_type, bl_path in self.ipi_config['default']['whitelists'].items(): + _bl_map[bl_type] = bl_path + if os.path.exists(bl_path): + continue + if os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)): + _bl_map[bl_type] = os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)) + return _bl_map + @property def use_proxy(self): return self.config['use_proxy'] @@ -141,7 +194,9 @@ def execute_analysis(self, observable): try: proxies = saq.PROXIES if self.use_proxy else None # Create Inspector with MaxMind API - mmi = Inspector(maxmind.Client(license_key=self.license_key, proxies=proxies)) + mmi = Inspector(maxmind.Client(license_key=self.license_key, proxies=proxies), + blacklists=self.blacklist_maps, + whitelists=self.whitelist_maps) except Exception as e: logging.error("Failed to create MaxMind Inspector: {}".format(e)) return False diff --git a/lib/saq/modules/url.py b/lib/saq/modules/url.py index 5cd5c4ce..e8841d16 100644 --- a/lib/saq/modules/url.py +++ b/lib/saq/modules/url.py @@ -46,6 +46,46 @@ KEY_PROXY = 'proxy' KEY_PROXY_NAME = 'proxy_name' +class ParseURLAnalysis(Analysis): + """Add the FQDN of the URL as an observable.""" + def initialize_details(self): + self.details = { 'netloc': None, + 'scheme': None, + 'path': None, + 'query': None, + 'params': None, + 'fragment': None } + + #def generate_summary(self): + # return f"Parsed: {self.details['netloc']}" + +class ParseURLAnalyzer(AnalysisModule): + """Parse the URL and add the FQDN as an observable.""" + + @property + def generated_analysis_type(self): + return ParseURLAnalysis + + @property + def valid_observable_types(self): + return F_URL + + def execute_analysis(self, observable): + try: + parsed_url = urlparse(observable.value) + analysis = self.create_analysis(observable) + analysis.details['netloc'] = parsed_url.netloc + analysis.details['scheme'] = parsed_url.scheme + analysis.details['path'] = parsed_url.path + analysis.details['query'] = parsed_url.query + analysis.details['params'] = parsed_url.params + analysis.details['fragment'] = parsed_url.fragment + analysis.add_observable(F_FQDN, parsed_url.netloc) + return True + except Exception as e: + logging.error(f"Problem parsing URL: {e}") + return False + class GglsblAnalysis(Analysis): """URL matches against Google's SafeBrowsing List using the [gglsbl-rest](https://github.com/mlsecproject/gglsbl-rest) service. """