From d8f12f0be2495b8c2639148c94880c0c4139de92 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Thu, 29 Feb 2024 15:47:07 -0300 Subject: [PATCH 1/6] Add codeql agent --- .../static/executors/official/codeql.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 faraday_agent_dispatcher/static/executors/official/codeql.py diff --git a/faraday_agent_dispatcher/static/executors/official/codeql.py b/faraday_agent_dispatcher/static/executors/official/codeql.py new file mode 100644 index 00000000..b3ee7149 --- /dev/null +++ b/faraday_agent_dispatcher/static/executors/official/codeql.py @@ -0,0 +1,92 @@ +import http +import json +import re +from pprint import pprint + +import requests +import os +import logging + +logger = logging.getLogger(__name__) + +CWE_PATTERN = ".*(cwe-\d+)" + + +def get_codeql_alerts(owner, repository, token, vuln_tag=None, host_tag=None): + url = f"https://api.github.com/repos/{owner}/{repository}/code-scanning/alerts" + auth = {"Authorization": f"Bearer {token}"} + + response = requests.get(url, headers=auth) + if response.status_code != http.HTTPStatus.OK: + print(f"Response from server {response.status_code} / repo {repository} / owner {owner}") + return [] + + if response.status_code == http.HTTPStatus.OK: + security_events = response.json() + pprint(security_events) + hosts_ips = list({security_event["most_recent_instance"]["location"]['path'] + for security_event in security_events}) + hosts = [] + + for ip in hosts_ips: + host_vulns = [] + for security_event in security_events: + if security_event["most_recent_instance"]["location"]['path'] == ip: + if security_event["state"] != "open": + logger.warning(f"Vulnerability {security_event['number']} already closed...") + continue + cwe_list = [] + _vuln_tag = set() + for tag in security_event['rule']['tags']: + if 'cwe-' in tag: + cwe = re.search(CWE_PATTERN, tag)[1] + cwe_list.append(cwe) + else: + _vuln_tag.add(tag) + + vulnerability = { + "name": f"{security_event['rule']['name']}", + "desc": f"{security_event['rule']['description']}\n", + "severity": f"{security_event['rule']['security_severity_level']}", + "type": "Vulnerability", + "impact": { + "accountability": False, + "availability": False, + }, + "cwe": cwe_list, + "cve": [], + "refs": [], + "tags": vuln_tag + list(_vuln_tag), + } + host_vulns.append(vulnerability) + + hosts.append( + { + "ip": f"{owner}/{repository}/{ip}", + "description": "", + "hostnames": [], + "vulnerabilities": host_vulns, + "tags": host_tag, + } + ) + + +def main(): + token = os.getenv("GITHUB_TOKEN") + owner = os.getenv("GITHUB_OWNER") + repository = os.getenv("EXECUTOR_CONFIG_GITHUB_REPOSITORY") + + vuln_tag = os.getenv("AGENT_CONFIG_VULN_TAG", []) + if vuln_tag: + vuln_tag = vuln_tag.split(",") + host_tag = os.getenv("AGENT_CONFIG_HOSTNAME_TAG", []) + if host_tag: + host_tag = host_tag.split(",") + + hosts = get_codeql_alerts(owner, repository, token, vuln_tag, host_tag) + data = {"hosts": hosts} + print(json.dumps(data)) + + +if __name__ == "__main__": + main() From e0a0bf15d72c3ab0591559f0d17515b72bdaba83 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Tue, 5 Mar 2024 11:03:44 -0300 Subject: [PATCH 2/6] Add codeql implementation --- .../static/executors/official/codeql.py | 268 +++++++++++++----- 1 file changed, 198 insertions(+), 70 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/codeql.py b/faraday_agent_dispatcher/static/executors/official/codeql.py index b3ee7149..abf6dd4c 100644 --- a/faraday_agent_dispatcher/static/executors/official/codeql.py +++ b/faraday_agent_dispatcher/static/executors/official/codeql.py @@ -1,7 +1,8 @@ import http import json import re -from pprint import pprint +import sys +from dataclasses import dataclass import requests import os @@ -9,83 +10,210 @@ logger = logging.getLogger(__name__) -CWE_PATTERN = ".*(cwe-\d+)" - - -def get_codeql_alerts(owner, repository, token, vuln_tag=None, host_tag=None): - url = f"https://api.github.com/repos/{owner}/{repository}/code-scanning/alerts" +CVE_PATTERN = r'.*(CVE-\d{4}-\d{4,7}).*' +CWE_PATTERN = r'.*(cwe-\d+)' +REFS_PATTERN = r'.*?\((https://.*?)\).*' + +token = os.getenv("GITHUB_TOKEN") +owner = os.getenv("GITHUB_OWNER") +repository = os.getenv("EXECUTOR_CONFIG_GITHUB_REPOSITORY") + + +@dataclass +class SecurityEvent: + name: str + description: str + severity: str + data: str + resolution: str + tags: list + cwe: list + cve: list + refs: list + + +def get_custom_description(vulnerability_data): + custom_description = f'{vulnerability_data["rule"]["full_description"]}' + most_recent_instance_data = vulnerability_data['most_recent_instance'] + + if 'location' in most_recent_instance_data: + location_data = most_recent_instance_data["location"] + if location_data: + commit_sha = None + path = None + github_link = "" + if 'commit_sha' in most_recent_instance_data: + commit_sha = most_recent_instance_data['commit_sha'] + if 'path' in most_recent_instance_data['location']: + path = most_recent_instance_data['location']['path'] + if commit_sha and path: + github_link = f"[View it on Github](https://github.com/{owner}/{repository}/blob/{commit_sha}/{path}" \ + f"#L{location_data['start_line']}-{location_data['end_line']})" + + custom_description = f'{custom_description}\n\n' \ + f'Column: {location_data["start_column"]} - {location_data["end_column"]}\n' \ + f'Line: {location_data["start_line"]} - {location_data["end_line"]}\n\n' \ + f'{github_link}' + return custom_description + + +def get_security_event_obj(event_id): + url = f"https://api.github.com/repos/{owner}/{repository}/code-scanning/alerts/{event_id}" auth = {"Authorization": f"Bearer {token}"} - response = requests.get(url, headers=auth) if response.status_code != http.HTTPStatus.OK: - print(f"Response from server {response.status_code} / repo {repository} / owner {owner}") + print(f"Response from server {response.status_code} / repo {repository} / owner {owner}", file=sys.stderr) + return None + security_event_obj = None + event = response.json() + if event: + name = event['rule']['description'] + description = get_custom_description(event) + tags = get_tags(event) + cwe = get_cwe(event) + + resolution, unparsed_data = parse_resolution(event) + + cve = [] + refs = [] + # unparsed data contains cve and references + if unparsed_data: + cve = parse_cve(unparsed_data) + refs = parse_references(unparsed_data) + + security_event_obj = SecurityEvent( + name=name, + description=description, + tags=tags, + cwe=cwe, + severity=event['rule']['security_severity_level'], + data=event['most_recent_instance']['message']['text'], + refs=refs, + cve=cve, + resolution=resolution + ) + + return security_event_obj + + +def get_cwe(alert): + cwe_set = set() + for cwe in alert['rule']['tags']: + if 'cwe-' in cwe: + try: + parsed_cwe = re.search(CWE_PATTERN, cwe)[1] + cwe_set.add(parsed_cwe) + except ValueError: + print(f"Could not parse cwe {cwe}", file=sys.stderr) + continue + return list(cwe_set) + + +def get_tags(alert): + tags = set() + for tag in alert['rule']['tags']: + if 'cwe-' in tag: + continue + else: + tags.add(tag) + if alert["most_recent_instance"]['category'].startswith("/language"): + tags.add(alert["most_recent_instance"]['category'].split(":")[1]) + return list(tags) + + +def get_resolution(alert): + try: + resolution, _ = alert['rule']['help'].split('## References') + except ValueError: + resolution = alert['rule']['help'] + return resolution + + +def parse_resolution(alert): + try: + resolution, unparsed_data = alert['rule']['help'].split('## References') + except ValueError: + resolution = alert['rule']['help'] + unparsed_data = "" + return resolution, unparsed_data + + +def parse_references(unparsed_data): + refs = [] + for ref in re.findall(REFS_PATTERN, unparsed_data): + if "cwe.mitre" in ref: + continue + refs.append({'type': 'other', 'name': ref}) + return refs + + +def parse_cve(unparsed_data): + cves_found = re.findall(CVE_PATTERN, unparsed_data) + + return list(set(cves_found)) + + +def get_security_events(): + url = f"https://api.github.com/repos/{owner}/{repository}/code-scanning/alerts" + auth = {"Authorization": f"Bearer {token}"} + data = {"state": "open"} + response = requests.get(url, headers=auth, data=data) + if response.status_code != http.HTTPStatus.OK: + print(f"Could not get {owner} alerts " + f"from {repository} repository. " + f"Response code was {response.status_code}", file=sys.stderr) return [] - - if response.status_code == http.HTTPStatus.OK: - security_events = response.json() - pprint(security_events) - hosts_ips = list({security_event["most_recent_instance"]["location"]['path'] - for security_event in security_events}) - hosts = [] - - for ip in hosts_ips: - host_vulns = [] - for security_event in security_events: - if security_event["most_recent_instance"]["location"]['path'] == ip: - if security_event["state"] != "open": - logger.warning(f"Vulnerability {security_event['number']} already closed...") - continue - cwe_list = [] - _vuln_tag = set() - for tag in security_event['rule']['tags']: - if 'cwe-' in tag: - cwe = re.search(CWE_PATTERN, tag)[1] - cwe_list.append(cwe) - else: - _vuln_tag.add(tag) - - vulnerability = { - "name": f"{security_event['rule']['name']}", - "desc": f"{security_event['rule']['description']}\n", - "severity": f"{security_event['rule']['security_severity_level']}", - "type": "Vulnerability", - "impact": { - "accountability": False, - "availability": False, - }, - "cwe": cwe_list, - "cve": [], - "refs": [], - "tags": vuln_tag + list(_vuln_tag), - } - host_vulns.append(vulnerability) - - hosts.append( - { - "ip": f"{owner}/{repository}/{ip}", - "description": "", - "hostnames": [], - "vulnerabilities": host_vulns, - "tags": host_tag, + return response.json() + + +def get_assets_to_create(vulnerability_tags: list, asset_tags: list) -> list: + security_events = get_security_events() + assets = list({security_event["most_recent_instance"]["location"]['path'] + for security_event in security_events}) + assets_to_create = [] + + for asset in assets: + asset_vulnerabilities = [] + for security_event in security_events: + if security_event["most_recent_instance"]["location"]['path'] == asset: + security_event_obj = get_security_event_obj(security_event['number']) + if not security_event_obj: + print(f"Could not get details of event with id {security_event['number']}") + continue + vulnerability = { + "name": f"{security_event_obj.name}", + "desc": f"{security_event_obj.description}\n", + "severity": f"{security_event_obj.severity}", + "data": f"{security_event_obj.data}", + "type": "Vulnerability", + "cwe": security_event_obj.cwe, + "cve": security_event_obj.cve, + "refs": security_event_obj.refs, + "tags": vulnerability_tags + list(security_event_obj.tags), + "resolution": security_event_obj.resolution } - ) + asset_vulnerabilities.append(vulnerability) + assets_to_create.append( + { + "ip": f"{owner}/{repository}/{asset}", + "description": "", + "hostnames": [], + "vulnerabilities": asset_vulnerabilities, + "tags": asset_tags, + } + ) + return assets_to_create def main(): - token = os.getenv("GITHUB_TOKEN") - owner = os.getenv("GITHUB_OWNER") - repository = os.getenv("EXECUTOR_CONFIG_GITHUB_REPOSITORY") - - vuln_tag = os.getenv("AGENT_CONFIG_VULN_TAG", []) - if vuln_tag: - vuln_tag = vuln_tag.split(",") - host_tag = os.getenv("AGENT_CONFIG_HOSTNAME_TAG", []) - if host_tag: - host_tag = host_tag.split(",") - - hosts = get_codeql_alerts(owner, repository, token, vuln_tag, host_tag) - data = {"hosts": hosts} - print(json.dumps(data)) + vulnerability_tags = os.getenv("AGENT_CONFIG_VULN_TAG") or [] + if vulnerability_tags: + vulnerability_tags = vulnerability_tags.split(",") + asset_tags = os.getenv("AGENT_CONFIG_HOSTNAME_TAG") or [] + if asset_tags: + asset_tags = asset_tags.split(",") + assets = get_assets_to_create(vulnerability_tags, asset_tags) + print(json.dumps({"hosts": assets})) if __name__ == "__main__": From 679074c216f182b2b1a8faff279cfe33a8f9a380 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Tue, 5 Mar 2024 11:07:40 -0300 Subject: [PATCH 3/6] Add changelog --- CHANGELOG/current/208.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGELOG/current/208.md diff --git a/CHANGELOG/current/208.md b/CHANGELOG/current/208.md new file mode 100644 index 00000000..6b9bc43f --- /dev/null +++ b/CHANGELOG/current/208.md @@ -0,0 +1 @@ +[ADD] New Github CodeQL agent. #208 From c573cf0fbbd2229150b3493ec64d6a28e589fe85 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Tue, 5 Mar 2024 11:56:34 -0300 Subject: [PATCH 4/6] Modify format with black --- CHANGELOG/current/208.md | 1 + .../static/executors/official/codeql.py | 98 +++++++++++-------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/CHANGELOG/current/208.md b/CHANGELOG/current/208.md index 6b9bc43f..9c7269a1 100644 --- a/CHANGELOG/current/208.md +++ b/CHANGELOG/current/208.md @@ -1 +1,2 @@ [ADD] New Github CodeQL agent. #208 +`` \ No newline at end of file diff --git a/faraday_agent_dispatcher/static/executors/official/codeql.py b/faraday_agent_dispatcher/static/executors/official/codeql.py index abf6dd4c..5a6d4fb8 100644 --- a/faraday_agent_dispatcher/static/executors/official/codeql.py +++ b/faraday_agent_dispatcher/static/executors/official/codeql.py @@ -10,9 +10,9 @@ logger = logging.getLogger(__name__) -CVE_PATTERN = r'.*(CVE-\d{4}-\d{4,7}).*' -CWE_PATTERN = r'.*(cwe-\d+)' -REFS_PATTERN = r'.*?\((https://.*?)\).*' +CVE_PATTERN = r".*(CVE-\d{4}-\d{4,7}).*" +CWE_PATTERN = r".*(cwe-\d+)" +REFS_PATTERN = r".*?\((https://.*?)\).*" token = os.getenv("GITHUB_TOKEN") owner = os.getenv("GITHUB_OWNER") @@ -34,26 +34,30 @@ class SecurityEvent: def get_custom_description(vulnerability_data): custom_description = f'{vulnerability_data["rule"]["full_description"]}' - most_recent_instance_data = vulnerability_data['most_recent_instance'] + most_recent_instance_data = vulnerability_data["most_recent_instance"] - if 'location' in most_recent_instance_data: + if "location" in most_recent_instance_data: location_data = most_recent_instance_data["location"] if location_data: commit_sha = None path = None github_link = "" - if 'commit_sha' in most_recent_instance_data: - commit_sha = most_recent_instance_data['commit_sha'] - if 'path' in most_recent_instance_data['location']: - path = most_recent_instance_data['location']['path'] + if "commit_sha" in most_recent_instance_data: + commit_sha = most_recent_instance_data["commit_sha"] + if "path" in most_recent_instance_data["location"]: + path = most_recent_instance_data["location"]["path"] if commit_sha and path: - github_link = f"[View it on Github](https://github.com/{owner}/{repository}/blob/{commit_sha}/{path}" \ - f"#L{location_data['start_line']}-{location_data['end_line']})" - - custom_description = f'{custom_description}\n\n' \ - f'Column: {location_data["start_column"]} - {location_data["end_column"]}\n' \ - f'Line: {location_data["start_line"]} - {location_data["end_line"]}\n\n' \ - f'{github_link}' + github_link = ( + f"[View it on Github](https://github.com/{owner}/{repository}/blob/{commit_sha}/{path}" + f"#L{location_data['start_line']}-{location_data['end_line']})" + ) + + custom_description = ( + f"{custom_description}\n\n" + f'Column: {location_data["start_column"]} - {location_data["end_column"]}\n' + f'Line: {location_data["start_line"]} - {location_data["end_line"]}\n\n' + f"{github_link}" + ) return custom_description @@ -62,12 +66,15 @@ def get_security_event_obj(event_id): auth = {"Authorization": f"Bearer {token}"} response = requests.get(url, headers=auth) if response.status_code != http.HTTPStatus.OK: - print(f"Response from server {response.status_code} / repo {repository} / owner {owner}", file=sys.stderr) + print( + f"Response from server {response.status_code} / repo {repository} / owner {owner}", + file=sys.stderr, + ) return None security_event_obj = None event = response.json() if event: - name = event['rule']['description'] + name = event["rule"]["description"] description = get_custom_description(event) tags = get_tags(event) cwe = get_cwe(event) @@ -86,11 +93,11 @@ def get_security_event_obj(event_id): description=description, tags=tags, cwe=cwe, - severity=event['rule']['security_severity_level'], - data=event['most_recent_instance']['message']['text'], + severity=event["rule"]["security_severity_level"], + data=event["most_recent_instance"]["message"]["text"], refs=refs, cve=cve, - resolution=resolution + resolution=resolution, ) return security_event_obj @@ -98,8 +105,8 @@ def get_security_event_obj(event_id): def get_cwe(alert): cwe_set = set() - for cwe in alert['rule']['tags']: - if 'cwe-' in cwe: + for cwe in alert["rule"]["tags"]: + if "cwe-" in cwe: try: parsed_cwe = re.search(CWE_PATTERN, cwe)[1] cwe_set.add(parsed_cwe) @@ -111,29 +118,29 @@ def get_cwe(alert): def get_tags(alert): tags = set() - for tag in alert['rule']['tags']: - if 'cwe-' in tag: + for tag in alert["rule"]["tags"]: + if "cwe-" in tag: continue else: tags.add(tag) - if alert["most_recent_instance"]['category'].startswith("/language"): - tags.add(alert["most_recent_instance"]['category'].split(":")[1]) + if alert["most_recent_instance"]["category"].startswith("/language"): + tags.add(alert["most_recent_instance"]["category"].split(":")[1]) return list(tags) def get_resolution(alert): try: - resolution, _ = alert['rule']['help'].split('## References') + resolution, _ = alert["rule"]["help"].split("## References") except ValueError: - resolution = alert['rule']['help'] + resolution = alert["rule"]["help"] return resolution def parse_resolution(alert): try: - resolution, unparsed_data = alert['rule']['help'].split('## References') + resolution, unparsed_data = alert["rule"]["help"].split("## References") except ValueError: - resolution = alert['rule']['help'] + resolution = alert["rule"]["help"] unparsed_data = "" return resolution, unparsed_data @@ -143,7 +150,7 @@ def parse_references(unparsed_data): for ref in re.findall(REFS_PATTERN, unparsed_data): if "cwe.mitre" in ref: continue - refs.append({'type': 'other', 'name': ref}) + refs.append({"type": "other", "name": ref}) return refs @@ -159,26 +166,35 @@ def get_security_events(): data = {"state": "open"} response = requests.get(url, headers=auth, data=data) if response.status_code != http.HTTPStatus.OK: - print(f"Could not get {owner} alerts " - f"from {repository} repository. " - f"Response code was {response.status_code}", file=sys.stderr) + print( + f"Could not get {owner} alerts " + f"from {repository} repository. " + f"Response code was {response.status_code}", + file=sys.stderr, + ) return [] return response.json() def get_assets_to_create(vulnerability_tags: list, asset_tags: list) -> list: security_events = get_security_events() - assets = list({security_event["most_recent_instance"]["location"]['path'] - for security_event in security_events}) + assets = list( + { + security_event["most_recent_instance"]["location"]["path"] + for security_event in security_events + } + ) assets_to_create = [] for asset in assets: asset_vulnerabilities = [] for security_event in security_events: - if security_event["most_recent_instance"]["location"]['path'] == asset: - security_event_obj = get_security_event_obj(security_event['number']) + if security_event["most_recent_instance"]["location"]["path"] == asset: + security_event_obj = get_security_event_obj(security_event["number"]) if not security_event_obj: - print(f"Could not get details of event with id {security_event['number']}") + print( + f"Could not get details of event with id {security_event['number']}" + ) continue vulnerability = { "name": f"{security_event_obj.name}", @@ -190,7 +206,7 @@ def get_assets_to_create(vulnerability_tags: list, asset_tags: list) -> list: "cve": security_event_obj.cve, "refs": security_event_obj.refs, "tags": vulnerability_tags + list(security_event_obj.tags), - "resolution": security_event_obj.resolution + "resolution": security_event_obj.resolution, } asset_vulnerabilities.append(vulnerability) assets_to_create.append( From 3fa0bd96250a344df067ae6b185387b5ae5ea195 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Tue, 5 Mar 2024 12:00:11 -0300 Subject: [PATCH 5/6] Modify format with black --- .../static/executors/official/codeql.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/codeql.py b/faraday_agent_dispatcher/static/executors/official/codeql.py index 5a6d4fb8..cc39aabe 100644 --- a/faraday_agent_dispatcher/static/executors/official/codeql.py +++ b/faraday_agent_dispatcher/static/executors/official/codeql.py @@ -178,12 +178,7 @@ def get_security_events(): def get_assets_to_create(vulnerability_tags: list, asset_tags: list) -> list: security_events = get_security_events() - assets = list( - { - security_event["most_recent_instance"]["location"]["path"] - for security_event in security_events - } - ) + assets = list({security_event["most_recent_instance"]["location"]["path"] for security_event in security_events}) assets_to_create = [] for asset in assets: @@ -192,9 +187,7 @@ def get_assets_to_create(vulnerability_tags: list, asset_tags: list) -> list: if security_event["most_recent_instance"]["location"]["path"] == asset: security_event_obj = get_security_event_obj(security_event["number"]) if not security_event_obj: - print( - f"Could not get details of event with id {security_event['number']}" - ) + print(f"Could not get details of event with id {security_event['number']}") continue vulnerability = { "name": f"{security_event_obj.name}", From b99b920a96103ff21d837241fddde48640535fe1 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Wed, 6 Mar 2024 18:08:26 +0000 Subject: [PATCH 6/6] Update 208.md --- CHANGELOG/current/208.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG/current/208.md b/CHANGELOG/current/208.md index 9c7269a1..6b9bc43f 100644 --- a/CHANGELOG/current/208.md +++ b/CHANGELOG/current/208.md @@ -1,2 +1 @@ [ADD] New Github CodeQL agent. #208 -`` \ No newline at end of file