From d0ca20df5761bcf9dea63a9652195d090a74e3b8 Mon Sep 17 00:00:00 2001 From: devplayer55221 Date: Fri, 4 Oct 2024 08:24:53 +0530 Subject: [PATCH 1/5] Adding namecheap-1 --- mantis/modules/dns/Namecheap.py | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 mantis/modules/dns/Namecheap.py diff --git a/mantis/modules/dns/Namecheap.py b/mantis/modules/dns/Namecheap.py new file mode 100644 index 0000000..a00a482 --- /dev/null +++ b/mantis/modules/dns/Namecheap.py @@ -0,0 +1,136 @@ +import os +import logging +import argparse +from itertools import count +import json +from lxml import etree +import requests +import sys +import yaml +from mantis.config_parsers.config_client import ConfigProvider +from mantis.tool_base_classes.baseScanner import BaseScanner +from mantis.models.args_model import ArgsModel +from mantis.utils.asset_type import AssetType +from mantis.constants import ASSET_TYPE_TLD, ASSET_TYPE_SUBDOMAIN +from mantis.utils.crud_utils import CrudUtils + +''' +namecheap-export-dns-records.py - Extract DNS records from Namecheap + +This script can export DNS records from a domain in Namecheap and print the domains detected. +python namecheap-export-dns-records.py + +To use the script you need to enable the Namecheap API on your account and +whitelist the public IPv4 address of the host you're running the script on. See +https://www.namecheap.com/support/api/intro/ for details. + +You need to provide valid Namecheap API details in the global variables. +''' + +class Namecheap(BaseScanner): + + async def init(self, args: ArgsModel): + self.args = args + return [(self, "Namecheap")] + + async def execute(self, tooltuple): + #logging.info(f"Using credentials from {os.environ['AWS_SHARED_CREDENTIALS_FILE']}") + logging.info(f"Reading zone files from Namecheap nameservers") + return await self.main() + + async def main(self): + """This script dumps all the domain names (hostnames) for all hosted zones on a Namecheap instance \n + + Prerequisite for this script - Namecheap API details need to be present in global variables \n + + """ + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + namecheap_api_url = 'https://api.namecheap.com/xml.response' + + #api_user = "" + #user_name = "" + #api_key = "" + #client_ip = "" # Client IP white-listing is mandatory in Namecheap + + output_dict_list = [] + results = {} + results["success"] = 0 + results["failure"] = 0 + + try: + + domain = "example.org" + (sld, tld) = domain.split('.', 1) + logging.info(f'Enumerating domain: {domain}') + records = do_export(sld,tld) + + for subdomain in records: + domain_dict = {} + domain_dict['_id'] = subdomain + domain_dict['asset'] = subdomain + if AssetType.check_tld(subdomain): + domain_dict['asset_type'] = ASSET_TYPE_TLD + else: + domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN + domain_dict['org'] = self.args.org + output_dict_list.append(domain_dict) + await CrudUtils.insert_assets(output_dict_list, source='internal') + results["success"] = 1 + return results + + + except Exception as e: + results["failure"] = 1 + results['exception'] = str(e) + return results + + def make_namecheap_request(data): + request = data.copy() + request.update({ + 'ApiUser': api_user, + 'UserName': user_name, + 'ApiKey': api_key, + 'ClientIP': client_ip, + }) + response = requests.post(namecheap_api_url, request) + response.raise_for_status() + response_xml = etree.XML(response.content) + if response_xml.get('Status') != 'OK': + raise Exception('Bad response: {}'.format(response.content)) + return response_xml + + def get_records(sld, tld): + response = make_namecheap_request({ + 'Command': 'namecheap.domains.dns.getHosts', + 'SLD': sld, + 'TLD': tld}) + host_elements = response.xpath( + '/x:ApiResponse/x:CommandResponse/x:DomainDNSGetHostsResult/x:host', + namespaces={'x': 'http://api.namecheap.com/xml.response'}) + records = [dict(h.attrib) for h in host_elements] + for record in records: + record.pop('AssociatedAppTitle', None) + record.pop('FriendlyName', None) + record.pop('HostId', None) + record['HostName'] = record.pop('Name')+"."+sld+"."+tld + record.pop('IsActive', None) + record.pop('IsDDNSEnabled', None) + if record['Type'] != 'MX': + record.pop('MXPref', None) + record['RecordType'] = record.pop('Type') + if record['TTL'] == '1800': + record.pop('TTL') + return records + + def do_export(sld,tld): + records = get_records(sld,tld) + return records + # for record in records: + # print(record['HostName']) + + def dict_hash(d): + d = d.copy() + name = d.pop('HostName') + type_ = d.pop('RecordType') + return (type_, name, json.dumps(d, sort_keys=True)) + \ No newline at end of file From 6352016da74ce885a809f2411481bdc384378d47 Mon Sep 17 00:00:00 2001 From: devplayer55221 Date: Sun, 6 Oct 2024 21:04:04 +0530 Subject: [PATCH 2/5] Modified Cloudflare.py for exception handling --- mantis/modules/dns/Cloudflare.py | 16 ++-- mantis/modules/dns/Namecheap.py | 136 ------------------------------- 2 files changed, 11 insertions(+), 141 deletions(-) delete mode 100644 mantis/modules/dns/Namecheap.py diff --git a/mantis/modules/dns/Cloudflare.py b/mantis/modules/dns/Cloudflare.py index 4582939..2bcd2a7 100644 --- a/mantis/modules/dns/Cloudflare.py +++ b/mantis/modules/dns/Cloudflare.py @@ -32,17 +32,23 @@ async def main(self): Prerequisite for this script - A CLoudflare DNS Zone Read only API key \n """ - token = None - - cf = CloudFlare.CloudFlare(token, raw=True) + token = None per_page = 100 - - zones = cf.zones.get(params={'per_page': per_page, 'page': 0}) output_dict_list = [] results = {} results["success"] = 0 results["failure"] = 0 try: + try: + logging.info("[+] Using Cloudflare token - {}".format(token)) + cf = CloudFlare.CloudFlare(token, raw=True) + zones = cf.zones.get(params={'per_page': per_page, 'page': 0}) + results["success"] += 1 + except Exception as e: + results["failure"] += 1 + results["exception"] = str(e) + logging.error("[!] Error in accessing Cloudflare token - {}".format(token)) + for zone_page in range(zones['result_info']['total_pages']): zones = cf.zones.get(params={'per_page': per_page, 'page': zone_page}) for zone in zones['result']: diff --git a/mantis/modules/dns/Namecheap.py b/mantis/modules/dns/Namecheap.py deleted file mode 100644 index a00a482..0000000 --- a/mantis/modules/dns/Namecheap.py +++ /dev/null @@ -1,136 +0,0 @@ -import os -import logging -import argparse -from itertools import count -import json -from lxml import etree -import requests -import sys -import yaml -from mantis.config_parsers.config_client import ConfigProvider -from mantis.tool_base_classes.baseScanner import BaseScanner -from mantis.models.args_model import ArgsModel -from mantis.utils.asset_type import AssetType -from mantis.constants import ASSET_TYPE_TLD, ASSET_TYPE_SUBDOMAIN -from mantis.utils.crud_utils import CrudUtils - -''' -namecheap-export-dns-records.py - Extract DNS records from Namecheap - -This script can export DNS records from a domain in Namecheap and print the domains detected. -python namecheap-export-dns-records.py - -To use the script you need to enable the Namecheap API on your account and -whitelist the public IPv4 address of the host you're running the script on. See -https://www.namecheap.com/support/api/intro/ for details. - -You need to provide valid Namecheap API details in the global variables. -''' - -class Namecheap(BaseScanner): - - async def init(self, args: ArgsModel): - self.args = args - return [(self, "Namecheap")] - - async def execute(self, tooltuple): - #logging.info(f"Using credentials from {os.environ['AWS_SHARED_CREDENTIALS_FILE']}") - logging.info(f"Reading zone files from Namecheap nameservers") - return await self.main() - - async def main(self): - """This script dumps all the domain names (hostnames) for all hosted zones on a Namecheap instance \n - - Prerequisite for this script - Namecheap API details need to be present in global variables \n - - """ - logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') - namecheap_api_url = 'https://api.namecheap.com/xml.response' - - #api_user = "" - #user_name = "" - #api_key = "" - #client_ip = "" # Client IP white-listing is mandatory in Namecheap - - output_dict_list = [] - results = {} - results["success"] = 0 - results["failure"] = 0 - - try: - - domain = "example.org" - (sld, tld) = domain.split('.', 1) - logging.info(f'Enumerating domain: {domain}') - records = do_export(sld,tld) - - for subdomain in records: - domain_dict = {} - domain_dict['_id'] = subdomain - domain_dict['asset'] = subdomain - if AssetType.check_tld(subdomain): - domain_dict['asset_type'] = ASSET_TYPE_TLD - else: - domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN - domain_dict['org'] = self.args.org - output_dict_list.append(domain_dict) - await CrudUtils.insert_assets(output_dict_list, source='internal') - results["success"] = 1 - return results - - - except Exception as e: - results["failure"] = 1 - results['exception'] = str(e) - return results - - def make_namecheap_request(data): - request = data.copy() - request.update({ - 'ApiUser': api_user, - 'UserName': user_name, - 'ApiKey': api_key, - 'ClientIP': client_ip, - }) - response = requests.post(namecheap_api_url, request) - response.raise_for_status() - response_xml = etree.XML(response.content) - if response_xml.get('Status') != 'OK': - raise Exception('Bad response: {}'.format(response.content)) - return response_xml - - def get_records(sld, tld): - response = make_namecheap_request({ - 'Command': 'namecheap.domains.dns.getHosts', - 'SLD': sld, - 'TLD': tld}) - host_elements = response.xpath( - '/x:ApiResponse/x:CommandResponse/x:DomainDNSGetHostsResult/x:host', - namespaces={'x': 'http://api.namecheap.com/xml.response'}) - records = [dict(h.attrib) for h in host_elements] - for record in records: - record.pop('AssociatedAppTitle', None) - record.pop('FriendlyName', None) - record.pop('HostId', None) - record['HostName'] = record.pop('Name')+"."+sld+"."+tld - record.pop('IsActive', None) - record.pop('IsDDNSEnabled', None) - if record['Type'] != 'MX': - record.pop('MXPref', None) - record['RecordType'] = record.pop('Type') - if record['TTL'] == '1800': - record.pop('TTL') - return records - - def do_export(sld,tld): - records = get_records(sld,tld) - return records - # for record in records: - # print(record['HostName']) - - def dict_hash(d): - d = d.copy() - name = d.pop('HostName') - type_ = d.pop('RecordType') - return (type_, name, json.dumps(d, sort_keys=True)) - \ No newline at end of file From 6de7423357dd91aa7ab2e0338adaae9ee68fd321 Mon Sep 17 00:00:00 2001 From: devplayer55221 Date: Tue, 8 Oct 2024 08:55:42 +0530 Subject: [PATCH 3/5] Modified the CloudFlare.CloudFlare to include email parameter --- mantis/modules/dns/Cloudflare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mantis/modules/dns/Cloudflare.py b/mantis/modules/dns/Cloudflare.py index 2bcd2a7..5f9c80a 100644 --- a/mantis/modules/dns/Cloudflare.py +++ b/mantis/modules/dns/Cloudflare.py @@ -32,7 +32,7 @@ async def main(self): Prerequisite for this script - A CLoudflare DNS Zone Read only API key \n """ - token = None + token = None #Edit the value with actual token per_page = 100 output_dict_list = [] results = {} @@ -41,13 +41,13 @@ async def main(self): try: try: logging.info("[+] Using Cloudflare token - {}".format(token)) - cf = CloudFlare.CloudFlare(token, raw=True) + cf = CloudFlare.CloudFlare("", token, raw=True) zones = cf.zones.get(params={'per_page': per_page, 'page': 0}) results["success"] += 1 except Exception as e: results["failure"] += 1 results["exception"] = str(e) - logging.error("[!] Error in accessing Cloudflare token - {}".format(token)) + logging.error("[!] Error - {}".format(str(e))) for zone_page in range(zones['result_info']['total_pages']): zones = cf.zones.get(params={'per_page': per_page, 'page': zone_page}) From 3f378cf7e401b55b9fbd0f8e441c42421250a299 Mon Sep 17 00:00:00 2001 From: devplayer55221 Date: Sat, 12 Oct 2024 12:19:20 +0530 Subject: [PATCH 4/5] Filtering the records in Cloudflare nameserver matching the assets --- mantis/modules/dns/Cloudflare.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/mantis/modules/dns/Cloudflare.py b/mantis/modules/dns/Cloudflare.py index 5f9c80a..c1f9109 100644 --- a/mantis/modules/dns/Cloudflare.py +++ b/mantis/modules/dns/Cloudflare.py @@ -1,5 +1,6 @@ import os import logging +from mantis.utils.tool_utils import get_assets_grouped_by_type from mantis.config_parsers.config_client import ConfigProvider from mantis.tool_base_classes.baseScanner import BaseScanner from mantis.models.args_model import ArgsModel @@ -19,6 +20,7 @@ class Cloudflare(BaseScanner): async def init(self, args: ArgsModel): self.args = args + self.db_assets = await get_assets_grouped_by_type(self, args, ASSET_TYPE_TLD) return [(self, "Cloudflare")] async def execute(self, tooltuple): @@ -40,7 +42,6 @@ async def main(self): results["failure"] = 0 try: try: - logging.info("[+] Using Cloudflare token - {}".format(token)) cf = CloudFlare.CloudFlare("", token, raw=True) zones = cf.zones.get(params={'per_page': per_page, 'page': 0}) results["success"] += 1 @@ -59,14 +60,28 @@ async def main(self): records = cf.zones.dns_records.get(zone['id'], params={'per_page': per_page, 'page': record_page})['result'] for record in records: domain_dict = {} - domain_dict['_id'] = record['name'] - domain_dict['asset'] = record['name'] - if AssetType.check_tld(record['name']): - domain_dict['asset_type'] = ASSET_TYPE_TLD + if(self.args.ignore_stale == True): + print(self.db_assets) + for asset in self.db_assets: + if(asset in record['name']): + domain_dict['_id'] = record['name'] + domain_dict['asset'] = record['name'] + if AssetType.check_tld(record['name']): + domain_dict['asset_type'] = ASSET_TYPE_TLD + else: + domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN + domain_dict['org'] = self.args.org + output_dict_list.append(domain_dict) + break else: - domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN - domain_dict['org'] = self.args.org - output_dict_list.append(domain_dict) + domain_dict['_id'] = record['name'] + domain_dict['asset'] = record['name'] + if AssetType.check_tld(record['name']): + domain_dict['asset_type'] = ASSET_TYPE_TLD + else: + domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN + domain_dict['org'] = self.args.org + output_dict_list.append(domain_dict) await CrudUtils.insert_assets(output_dict_list, source='internal') results["success"] = 1 return results From b9f7bab5f45f0611181c4151a71f87cf34677717 Mon Sep 17 00:00:00 2001 From: devplayer55221 Date: Mon, 14 Oct 2024 21:53:56 +0530 Subject: [PATCH 5/5] Added a flag for filtering Cloudflare records based on scope --- mantis/models/args_model.py | 1 + mantis/modules/dns/Cloudflare.py | 2 +- mantis/utils/args_parse.py | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mantis/models/args_model.py b/mantis/models/args_model.py index a843f94..ceb59bc 100644 --- a/mantis/models/args_model.py +++ b/mantis/models/args_model.py @@ -21,4 +21,5 @@ class ArgsModel(BaseModel): subdomain: str = Field(None) list_: bool = False list_orgs: bool = False + in_scope: bool = False \ No newline at end of file diff --git a/mantis/modules/dns/Cloudflare.py b/mantis/modules/dns/Cloudflare.py index c1f9109..3e89b22 100644 --- a/mantis/modules/dns/Cloudflare.py +++ b/mantis/modules/dns/Cloudflare.py @@ -60,7 +60,7 @@ async def main(self): records = cf.zones.dns_records.get(zone['id'], params={'per_page': per_page, 'page': record_page})['result'] for record in records: domain_dict = {} - if(self.args.ignore_stale == True): + if(self.args.in_scope == True): print(self.db_assets) for asset in self.db_assets: if(asset in record['name']): diff --git a/mantis/utils/args_parse.py b/mantis/utils/args_parse.py index 43916c2..061cb32 100644 --- a/mantis/utils/args_parse.py +++ b/mantis/utils/args_parse.py @@ -229,6 +229,12 @@ def args_parse() -> ArgsModel: scan_parser.add_argument('--sub', dest = 'subdomain', help='Subdomain to scan') + + scan_parser.add_argument('-is', '--in_scope', + dest = 'in_scope', + help = 'List only the records from nameserver that are in scope', + action = 'store_true' + ) list_parser = subparser.add_parser("list", help="List entities present in db", usage=ArgsParse.list_msg())