diff --git a/suricata/update/data/index.py b/suricata/update/data/index.py index 48d4ebb..02a9c4f 100644 --- a/suricata/update/data/index.py +++ b/suricata/update/data/index.py @@ -51,6 +51,28 @@ 'support-url': 'https://redmine.openinfosecfoundation.org/', 'url': 'https://openinfosecfoundation.org/rules/trafficid/trafficid.rules', 'vendor': 'OISF'}, + 'pawpatrules': { 'checksum': False, + 'description': 'PAW Patrules ruleset ' + 'permit to detect many ' + 'events on\n' + 'network. Suspicious ' + 'flow, malicious tool, ' + 'unsuported and\n' + 'vulnerable system, known ' + 'threat actors with ' + 'various IOCs,\n' + 'lateral movement, bad ' + 'practice, shadow IT... ' + 'Rules are\n' + 'frequently updated.\n', + 'homepage': 'https://pawpatrules.fr/', + 'license': 'CC-BY-SA-4.0', + 'min-version': '6.0.0', + 'summary': 'PAW Patrules is a collection ' + 'of rules for IDPS / NSM ' + 'Suricata engine', + 'url': 'https://rules.pawpatrules.fr/suricata/paw-patrules.tar.gz', + 'vendor': 'pawpatrules'}, 'ptresearch/attackdetection': { 'description': 'The ' 'Attack ' 'Detection ' @@ -261,6 +283,184 @@ 'support-url': 'https://discord.com/channels/911231224448712714/911238451842666546', 'url': 'https://ti.stamus-networks.io/open/stamus-lateral-rules.tar.gz', 'vendor': 'Stamus Networks'}, + 'stamus/nrd-14-open': { 'description': 'Newly Registered ' + 'Domains list ' + '(last 14 days) to ' + 'match on DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced by ' + 'Stamus Labs ' + 'research team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly Registered ' + 'Domains Open only - ' + '14 day list, complete', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-nrd-14.tar.gz', + 'vendor': 'Stamus Networks'}, + 'stamus/nrd-30-open': { 'description': 'Newly Registered ' + 'Domains list ' + '(last 30 days) to ' + 'match on DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced by ' + 'Stamus Labs ' + 'research team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly Registered ' + 'Domains Open only - ' + '30 day list, complete', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-nrd-30.tar.gz', + 'vendor': 'Stamus Networks'}, + 'stamus/nrd-entropy-14-open': { 'description': 'Suspicious ' + 'Newly ' + 'Registered ' + 'Domains ' + 'list with ' + 'high ' + 'entropy ' + '(last 14 ' + 'days) to ' + 'match on ' + 'DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced ' + 'by Stamus ' + 'Labs ' + 'research ' + 'team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly ' + 'Registered ' + 'Domains Open ' + 'only - 14 day ' + 'list, high ' + 'entropy', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-entropy-14.tar.gz', + 'vendor': 'Stamus ' + 'Networks'}, + 'stamus/nrd-entropy-30-open': { 'description': 'Suspicious ' + 'Newly ' + 'Registered ' + 'Domains ' + 'list with ' + 'high ' + 'entropy ' + '(last 30 ' + 'days) to ' + 'match on ' + 'DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced ' + 'by Stamus ' + 'Labs ' + 'research ' + 'team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly ' + 'Registered ' + 'Domains Open ' + 'only - 30 day ' + 'list, high ' + 'entropy', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-entropy-30.tar.gz', + 'vendor': 'Stamus ' + 'Networks'}, + 'stamus/nrd-phishing-14-open': { 'description': 'Suspicious ' + 'Newly ' + 'Registered ' + 'Domains ' + 'Phishing ' + 'list ' + '(last 14 ' + 'days) to ' + 'match on ' + 'DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced ' + 'by ' + 'Stamus ' + 'Labs ' + 'research ' + 'team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly ' + 'Registered ' + 'Domains Open ' + 'only - 14 ' + 'day list, ' + 'phishing', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-phishing-14.tar.gz', + 'vendor': 'Stamus ' + 'Networks'}, + 'stamus/nrd-phishing-30-open': { 'description': 'Suspicious ' + 'Newly ' + 'Registered ' + 'Domains ' + 'Phishing ' + 'list ' + '(last 30 ' + 'days) to ' + 'match on ' + 'DNS, TLS ' + 'and HTTP ' + 'communication.\n' + 'Produced ' + 'by ' + 'Stamus ' + 'Labs ' + 'research ' + 'team.\n', + 'license': 'Commercial', + 'min-version': '6.0.0', + 'parameters': { 'secret-code': { 'prompt': 'Stamus ' + 'Networks ' + 'License ' + 'code'}}, + 'subscribe-url': 'https://www.stamus-networks.com/stamus-labs/subscribe-to-threat-intel-feed', + 'summary': 'Newly ' + 'Registered ' + 'Domains Open ' + 'only - 30 ' + 'day list, ' + 'phishing', + 'url': 'https://ti.stamus-networks.io/%(secret-code)s/sti-domains-phishing-30.tar.gz', + 'vendor': 'Stamus ' + 'Networks'}, 'tgreen/hunting': { 'checksum': False, 'description': 'Heuristic ruleset for ' 'hunting. Focus on ' diff --git a/suricata/update/main.py b/suricata/update/main.py index a1e9e70..18af7a8 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -447,7 +447,7 @@ def handle_dataset_files(rule, dep_files): prefix = os.path.dirname(rule.group) # Construct the source filename. - source_filename = "{}/{}".format(prefix, dataset_filename) + source_filename = os.path.join(prefix, dataset_filename) # If a source filename starts with a "/", look for it on the filesystem. The archive # unpackers will take care of removing a leading / so this shouldn't happen for @@ -483,10 +483,19 @@ def handle_filehash_files(rule, dep_files, fhash): if not rule.enabled: return filehash_fname = rule.get(fhash) - filename = [fname for fname, content in dep_files.items() if os.path.join(*(fname.split(os.path.sep)[1:])) == filehash_fname] - if filename: + + # Get the directory name the rule is from. + prefix = os.path.dirname(rule.group) + + source_filename = os.path.join(prefix, filehash_fname) + dest_filename = source_filename[len(prefix) + len(os.path.sep):] + logger.debug("dest_filename={}".format(dest_filename)) + + if source_filename not in dep_files: + logger.error("{} file {} was not found".format(fhash, filehash_fname)) + else: logger.debug("Copying %s file %s to output directory" % (fhash, filehash_fname)) - filepath = os.path.join(config.get_state_dir(), os.path.dirname(filename[0])) + filepath = os.path.join(config.get_output_dir(), os.path.dirname(dest_filename)) logger.debug("filepath: %s" % filepath) try: os.makedirs(filepath) @@ -494,11 +503,10 @@ def handle_filehash_files(rule, dep_files, fhash): if oserr.errno != errno.EEXIST: logger.error(oserr) sys.exit(1) - logger.debug("output fname: %s" % os.path.join(filepath, os.path.basename(filehash_fname))) - with open(os.path.join(filepath, os.path.basename(filehash_fname)), "w+") as fp: - fp.write(dep_files[os.path.join("rules", filehash_fname)].decode("utf-8")) - else: - logger.error("{} file {} was not found".format(fhash, filehash_fname)) + output_filename = os.path.join(filepath, os.path.basename(filehash_fname)) + logger.debug("output fname: %s" % output_filename) + with open(output_filename, "w") as fp: + fp.write(dep_files[source_filename].decode("utf-8")) def write_merged(filename, rulemap, dep_files): @@ -991,7 +999,7 @@ def load_sources(suricata_version): source_files = Fetch().run(url) for key in source_files: content = source_files[key] - key = format("{}/{}".format(prefix, key)) + key = os.path.join(prefix, key) files.append(SourceFile(key, content)) # Now load local rules. diff --git a/tests/integration_tests.py b/tests/integration_tests.py index 8970585..c4b119b 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -118,6 +118,15 @@ def delete(path): "testing-header-with-spaces", "file:///doesnotexist" ]) +run(common_args + [ + "add-source", + "suricata-test-rules", + "file://{}/tests/suricata-test-rules.zip".format(os.getcwd()), +]) +run(common_args) +assert(os.path.exists(os.path.join(DATA_DIR, "rules/testmyids.md5"))) +assert(os.path.exists(os.path.join(DATA_DIR, "rules/testmyids.sha1"))) +assert(os.path.exists(os.path.join(DATA_DIR, "rules/testmyids.sha256"))) class IntegrationTest: def __init__(self, configs={}): diff --git a/tests/suricata-test-rules.zip b/tests/suricata-test-rules.zip new file mode 100644 index 0000000..4f834f8 Binary files /dev/null and b/tests/suricata-test-rules.zip differ