From e294dede76f924a9f78b0cb53f141dcfb0e9cc62 Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Mon, 24 Jun 2024 15:43:04 -0300 Subject: [PATCH 1/9] ms defender test version --- .../executors/official/microsoft_defender.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 faraday_agent_dispatcher/static/executors/official/microsoft_defender.py diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py new file mode 100644 index 00000000..14e2e791 --- /dev/null +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +import os +import sys +import json +import time +from datetime import datetime +import requests + + +def log(msg="", end="\n"): + print(msg, file=sys.stderr, flush=True, end=end) + + +def dt_ms_patch(date_str): + """ + This patch fixes an issue where datetime cannot process milliseconds + with more than 6 decimal places, which raises an exception. + Also removes anything beyond the milliseconds + """ + patch = (date_str[:-1][: date_str.index(".") + 7]) if "." in date_str else date_str[:-1] + return datetime.strptime(patch, "%Y-%m-%dT%H:%M:%S.%f") + + +def description_maker(machine): + ips = "" + for ip in machine["ipAddresses"]: + if ip["type"] not in ["SoftwareLoopback", "Tunnel"]: + # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability + mac = ( + (":".join(ip["macAddress"][i : i + 2 :] for i in range(0, len(ip["macAddress"]), 2))) + if ip["macAddress"] is not None + else "N/A" + ) + ips += f" IP: {ip['ipAddress']}\n MAC: {mac}\n {'-'*(len(mac)+5)}\n" + + last_seen = dt_ms_patch(machine["lastSeen"]).strftime("%d/%m/%Y at %H:%M:%S UTC") + first_seen = dt_ms_patch(machine["firstSeen"]).strftime("%d/%m/%Y at %H:%M:%S UTC") + + desc = f"OS: {machine['osPlatform']} " + desc += f"{machine['osArchitecture'] if machine['osArchitecture'] is not None else ''} " + desc += f"{machine['version'] if machine['version'] not in [None, 'Other'] else ''} " + desc += f"{'(build ' + str(machine['osBuild']) + ')' if machine['osBuild'] not in [None, 'Other'] else ''}\n\n" + desc += f"Device Timestamps:\n First Seen: {first_seen}\n Last Seen: {last_seen}\n\n" + desc += f"Known IP's & associated MAC address:\n{ips}" + return desc + + +def token_gen(tenant_id, client_id, client_secret): + app_id_url = "https://api.securitycenter.microsoft.com" + app_auth_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/token" + r_body = { + "client_id": client_id, + "client_secret": client_secret, + "grant_type": "client_credentials", + "resource": app_id_url, + } + resp = requests.post(app_auth_url, data=r_body).json() + if "error" in resp.keys(): + log(f"Error at token generation: {resp['error']}") + log(resp["error_description"]) + exit(1) + access_token = resp["access_token"] + log("Token generated successfully") + return access_token + + +def get_machines(token): + headers = {"Authorization": f"Bearer {token}"} + resp = requests.get("https://api.security.microsoft.com/api/machines", headers=headers).json() + if "error" in resp.keys(): + log(f"Error at retrieving machines: {resp['error']['code']}") + log(resp["error"]["message"]) + exit(1) + return resp["value"] + + +def get_machine_vulns(token, machine_id): + headers = {"Authorization": f"Bearer {token}"} + resp = requests.get( + f"https://api.security.microsoft.com/api/machines/{machine_id}/vulnerabilities", headers=headers + ).json() + if "error" in resp.keys(): + log(f"Error at retrieving machine vulns: {resp['error']['code']}") + log(resp["error"]["message"]) + exit(1) + return resp["value"] + + +def generate_report(token, vuln_tags, host_tags, days_old): + hosts = [] + log("Fetching machines...", "\r") + machines = get_machines(token) + log(f"Retrieved {len(machines)} machines") + log("Processing assets ...", "\r") + avr_time = [] + for machine in machines: + _start = time.time() + # check if machine is younger than threshold, measured in days (1 day = 86400 secs) + if dt_ms_patch(machine["lastSeen"]).timestamp() < (datetime.now().timestamp() - (days_old * 86400)): + continue + host = { + "ip": machine["id"], + "os": machine["osPlatform"], + "hostnames": [machine["computerDnsName"] if machine["computerDnsName"] is not None else "N/A"], + "mac": "", + "tags": [tag for tag in machine["machineTags"]] + host_tags, + "description": description_maker(machine), + } + vuln_list = [] + for vuln in get_machine_vulns(token, machine["id"]): + vuln_list.append( + { + "name": vuln["name"], + "desc": vuln["description"], + "severity": vuln["severity"].lower(), + "external_id": vuln["id"], + "type": "Vulnerability", + "status": "open", + "cve": [vuln["id"]], + "cvss3": {"base_score": str(vuln["cvssV3"]), "vector_string": vuln["cvssVector"]}, + "tags": vuln["tags"] + vuln_tags, + "confirmed": vuln["exploitVerified"], + "refs": [{"name": url, "type": "other"} for url in vuln["exploitUris"]], + "resolution": f"https://security.microsoft.com/vulnerabilities/vulnerability/{vuln['id']}" + + "/recommendation", + } + ) + host["vulnerabilities"] = vuln_list + hosts.append(host) + avr_time.append(time.time() - _start) + log( + f"Processing assets ... {len(hosts)} / {len(machines)} ({len(hosts)*100/len(machines):.2f}%)" + + f" ETA: {((len(machines)-len(hosts))*(sum(avr_time)/len(avr_time)))/60:.2f} min", + "\r", + ) + log() + print(json.dumps(hosts)) + + +def main(): + params_tenant_id = "" # os.getenv("TENANT_ID") + params_client_id = "" # os.getenv("CLIENT_ID") + params_client_secret = "" # os.getenv("CLIENT_SECRET") + params_days_old = "1" # os.getenv("EXECUTOR_CONFIG_DAYS_OLD") + params_days_old = ( + int(params_days_old) if params_days_old.isnumeric() else log("DAYS_OLD Variable must be an integer") or exit() + ) + + params_vuln_tags = os.getenv("AGENT_CONFIG_VULN_TAG") + params_vuln_tags = (params_vuln_tags.split(",") if params_vuln_tags != "" else []) if params_vuln_tags else [] + + params_host_tags = os.getenv("AGENT_CONFIG_HOSTNAME_TAG") + params_host_tags = (params_host_tags.split(",") if params_host_tags != "" else []) if params_host_tags else [] + + token = token_gen(params_tenant_id, params_client_id, params_client_secret) + generate_report( + token, params_vuln_tags, params_host_tags, int(params_days_old) if int(params_days_old) >= 1 else 1 + ) + + +if __name__ == "__main__": + main() From cb303755ff50623d7845c6193100cf232c32aa9f Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Fri, 28 Jun 2024 13:13:46 -0300 Subject: [PATCH 2/9] MS Defender finished --- .../executors/official/microsoft_defender.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index 14e2e791..d0a57794 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -21,27 +21,36 @@ def dt_ms_patch(date_str): return datetime.strptime(patch, "%Y-%m-%dT%H:%M:%S.%f") +def severity_patch(severity): + if severity not in ["critical", "high", "medium", "low", "info", "unclassified"]: + return "unclassified" + return severity + + def description_maker(machine): ips = "" for ip in machine["ipAddresses"]: - if ip["type"] not in ["SoftwareLoopback", "Tunnel"]: - # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability - mac = ( - (":".join(ip["macAddress"][i : i + 2 :] for i in range(0, len(ip["macAddress"]), 2))) - if ip["macAddress"] is not None - else "N/A" - ) - ips += f" IP: {ip['ipAddress']}\n MAC: {mac}\n {'-'*(len(mac)+5)}\n" + if ip["type"] in ["SoftwareLoopback", "Tunnel"]: + continue + if len(ip["ipAddress"]) < 7 or ip["ipAddress"] in ["127.0.0.1"]: + continue + # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability + mac = ( + (":".join(ip["macAddress"][i : i + 2] for i in range(0, len(ip["macAddress"]), 2))) + if ip["macAddress"] is not None + else "N/A" + ) + ips += f" IP: {ip['ipAddress']}\n MAC: {mac}\n\n" last_seen = dt_ms_patch(machine["lastSeen"]).strftime("%d/%m/%Y at %H:%M:%S UTC") first_seen = dt_ms_patch(machine["firstSeen"]).strftime("%d/%m/%Y at %H:%M:%S UTC") - desc = f"OS: {machine['osPlatform']} " + desc = f"## Operating System\n{machine['osPlatform']} " desc += f"{machine['osArchitecture'] if machine['osArchitecture'] is not None else ''} " desc += f"{machine['version'] if machine['version'] not in [None, 'Other'] else ''} " desc += f"{'(build ' + str(machine['osBuild']) + ')' if machine['osBuild'] not in [None, 'Other'] else ''}\n\n" - desc += f"Device Timestamps:\n First Seen: {first_seen}\n Last Seen: {last_seen}\n\n" - desc += f"Known IP's & associated MAC address:\n{ips}" + desc += f"## Device Timestamps\nFirst Seen: {first_seen}\n Last Seen: {last_seen}\n\n" + desc += f"## Known IP's & associated MAC address\n{ips}" return desc @@ -112,7 +121,7 @@ def generate_report(token, vuln_tags, host_tags, days_old): { "name": vuln["name"], "desc": vuln["description"], - "severity": vuln["severity"].lower(), + "severity": severity_patch(vuln["severity"].lower()), "external_id": vuln["id"], "type": "Vulnerability", "status": "open", @@ -123,6 +132,7 @@ def generate_report(token, vuln_tags, host_tags, days_old): "refs": [{"name": url, "type": "other"} for url in vuln["exploitUris"]], "resolution": f"https://security.microsoft.com/vulnerabilities/vulnerability/{vuln['id']}" + "/recommendation", + "cwe": [], } ) host["vulnerabilities"] = vuln_list @@ -131,10 +141,9 @@ def generate_report(token, vuln_tags, host_tags, days_old): log( f"Processing assets ... {len(hosts)} / {len(machines)} ({len(hosts)*100/len(machines):.2f}%)" + f" ETA: {((len(machines)-len(hosts))*(sum(avr_time)/len(avr_time)))/60:.2f} min", - "\r", + "\n", ) - log() - print(json.dumps(hosts)) + print(json.dumps({"hosts": hosts})) def main(): From b75c4349d81713faaffeda048cea7eb1ecc305b4 Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Fri, 28 Jun 2024 13:17:17 -0300 Subject: [PATCH 3/9] added changelog for ms defender --- CHANGELOG/current/218.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGELOG/current/218.md diff --git a/CHANGELOG/current/218.md b/CHANGELOG/current/218.md new file mode 100644 index 00000000..35472baa --- /dev/null +++ b/CHANGELOG/current/218.md @@ -0,0 +1 @@ +[ADD] Added agent for Microsoft Defender for Endpoint. #218 From d16a9896fd214ff61c0603e773318714f7c750c1 Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Mon, 1 Jul 2024 12:04:25 -0300 Subject: [PATCH 4/9] fixed minor errors --- .../executors/official/microsoft_defender.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index d0a57794..d6bbe74e 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -100,13 +100,16 @@ def generate_report(token, vuln_tags, host_tags, days_old): log("Fetching machines...", "\r") machines = get_machines(token) log(f"Retrieved {len(machines)} machines") - log("Processing assets ...", "\r") - avr_time = [] + log("Filtering assets ...") + matched_machines = [] for machine in machines: - _start = time.time() # check if machine is younger than threshold, measured in days (1 day = 86400 secs) - if dt_ms_patch(machine["lastSeen"]).timestamp() < (datetime.now().timestamp() - (days_old * 86400)): - continue + if dt_ms_patch(machine["lastSeen"]).timestamp() >= (datetime.now().timestamp() - (days_old * 86400)): + matched_machines.append(machine) + log(f"{len(matched_machines)} machines matched time filter") + avr_time = [] + for machine in matched_machines: + _start = time.time() host = { "ip": machine["id"], "os": machine["osPlatform"], @@ -139,18 +142,17 @@ def generate_report(token, vuln_tags, host_tags, days_old): hosts.append(host) avr_time.append(time.time() - _start) log( - f"Processing assets ... {len(hosts)} / {len(machines)} ({len(hosts)*100/len(machines):.2f}%)" - + f" ETA: {((len(machines)-len(hosts))*(sum(avr_time)/len(avr_time)))/60:.2f} min", - "\n", + f"Processing assets: {len(hosts)} / {len(matched_machines)} ({len(hosts)*100/len(matched_machines):.2f}%)" + + f" ETA: {((len(matched_machines)-len(hosts))*(sum(avr_time)/len(avr_time)))/60:.2f} min" ) print(json.dumps({"hosts": hosts})) def main(): - params_tenant_id = "" # os.getenv("TENANT_ID") - params_client_id = "" # os.getenv("CLIENT_ID") - params_client_secret = "" # os.getenv("CLIENT_SECRET") - params_days_old = "1" # os.getenv("EXECUTOR_CONFIG_DAYS_OLD") + params_tenant_id = os.getenv("TENANT_ID") + params_client_id = os.getenv("CLIENT_ID") + params_client_secret = os.getenv("CLIENT_SECRET") + params_days_old = os.getenv("EXECUTOR_CONFIG_DAYS_OLD") params_days_old = ( int(params_days_old) if params_days_old.isnumeric() else log("DAYS_OLD Variable must be an integer") or exit() ) From 22f25935182dc37e66f7343dd6e7d847030b90c1 Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Mon, 1 Jul 2024 12:16:16 -0300 Subject: [PATCH 5/9] fixed minor errors --- .../static/executors/official/microsoft_defender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index d6bbe74e..67318f30 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -36,7 +36,7 @@ def description_maker(machine): continue # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability mac = ( - (":".join(ip["macAddress"][i : i + 2] for i in range(0, len(ip["macAddress"]), 2))) + (":".join(ip["macAddress"][i:i+2] for i in range(0, len(ip["macAddress"]), 2))) if ip["macAddress"] is not None else "N/A" ) From 68ed286e54a70c520bc71279ed74a3e0ec51fbef Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Mon, 1 Jul 2024 12:23:38 -0300 Subject: [PATCH 6/9] fixed minor errors --- .../static/executors/official/microsoft_defender.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index 67318f30..2734cb7c 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -36,10 +36,13 @@ def description_maker(machine): continue # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability mac = ( + # fmt: off (":".join(ip["macAddress"][i:i+2] for i in range(0, len(ip["macAddress"]), 2))) + # fmt: on if ip["macAddress"] is not None else "N/A" ) + ips += f" IP: {ip['ipAddress']}\n MAC: {mac}\n\n" last_seen = dt_ms_patch(machine["lastSeen"]).strftime("%d/%m/%Y at %H:%M:%S UTC") From 364501d5defbae511ea0aa833175d25965a5614f Mon Sep 17 00:00:00 2001 From: Dante Acosta Date: Mon, 1 Jul 2024 12:25:45 -0300 Subject: [PATCH 7/9] fixed minor errors --- .../static/executors/official/microsoft_defender.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index 2734cb7c..a20e04f7 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -36,9 +36,7 @@ def description_maker(machine): continue # converts '7CDB98C877F1' into '7C:DB:98:C8:77:F1' for better readability mac = ( - # fmt: off - (":".join(ip["macAddress"][i:i+2] for i in range(0, len(ip["macAddress"]), 2))) - # fmt: on + (":".join(ip["macAddress"][i : i + 2] for i in range(0, len(ip["macAddress"]), 2))) # noqa: E203 if ip["macAddress"] is not None else "N/A" ) From 82b178b75fe0462a039dc659fde97054f89fa646 Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Wed, 3 Jul 2024 10:58:04 -0300 Subject: [PATCH 8/9] Add rate limit --- .../executors/official/microsoft_defender.py | 15 +++++++++++---- setup.py | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index a20e04f7..5eaff97d 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -4,7 +4,14 @@ import json import time from datetime import datetime -import requests + +from requests import Session +from requests_ratelimiter import LimiterAdapter + +session = Session() +adapter = LimiterAdapter(per_minute=40) +session.mount("http://", adapter) +session.mount("https://", adapter) def log(msg="", end="\n"): @@ -64,7 +71,7 @@ def token_gen(tenant_id, client_id, client_secret): "grant_type": "client_credentials", "resource": app_id_url, } - resp = requests.post(app_auth_url, data=r_body).json() + resp = session.post(app_auth_url, data=r_body).json() if "error" in resp.keys(): log(f"Error at token generation: {resp['error']}") log(resp["error_description"]) @@ -76,7 +83,7 @@ def token_gen(tenant_id, client_id, client_secret): def get_machines(token): headers = {"Authorization": f"Bearer {token}"} - resp = requests.get("https://api.security.microsoft.com/api/machines", headers=headers).json() + resp = session.get("https://api.security.microsoft.com/api/machines", headers=headers).json() if "error" in resp.keys(): log(f"Error at retrieving machines: {resp['error']['code']}") log(resp["error"]["message"]) @@ -86,7 +93,7 @@ def get_machines(token): def get_machine_vulns(token, machine_id): headers = {"Authorization": f"Bearer {token}"} - resp = requests.get( + resp = session.get( f"https://api.security.microsoft.com/api/machines/{machine_id}/vulnerabilities", headers=headers ).json() if "error" in resp.keys(): diff --git a/setup.py b/setup.py index 2ebacce6..074900dc 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ "psutil", "pytenable", "python-socketio==5.8.0", + "requests-ratelimiter", ] setup_requirements = ["pytest-runner", "click", "setuptools_scm"] From d6003e071e7518844e25b5df021f187b020543be Mon Sep 17 00:00:00 2001 From: Diego Nadares Date: Wed, 3 Jul 2024 12:09:58 -0300 Subject: [PATCH 9/9] Add too many requests error handling. --- .../executors/official/microsoft_defender.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py index 5eaff97d..73aa6739 100644 --- a/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py +++ b/faraday_agent_dispatcher/static/executors/official/microsoft_defender.py @@ -8,6 +8,8 @@ from requests import Session from requests_ratelimiter import LimiterAdapter +SECONDS_AFTER_TOO_MANY_REQUESTS = 30 + session = Session() adapter = LimiterAdapter(per_minute=40) session.mount("http://", adapter) @@ -99,7 +101,18 @@ def get_machine_vulns(token, machine_id): if "error" in resp.keys(): log(f"Error at retrieving machine vulns: {resp['error']['code']}") log(resp["error"]["message"]) - exit(1) + # In case limiter is not suffice. We'll try a second time after SECONDS_AFTER_TOO_MANY_REQUESTS seconds. + if resp["error"]["code"] == "TooManyRequests": + log(f"Waiting for {SECONDS_AFTER_TOO_MANY_REQUESTS} seconds") + time.sleep(SECONDS_AFTER_TOO_MANY_REQUESTS) + resp = session.get( + f"https://api.security.microsoft.com/api/machines/{machine_id}/vulnerabilities", headers=headers + ).json() + if "error" in resp.keys(): + log(f"Second error at retrieving machine vulns: {resp['error']['code']}. Exiting...") + exit(1) + else: + exit(1) return resp["value"]