From 18ca85cacd4bb8c8cb6b957c6332045c10174bec Mon Sep 17 00:00:00 2001 From: Naboot42 Date: Wed, 2 Aug 2023 09:47:58 +0200 Subject: [PATCH 1/4] add theHarvester Former-commit-id: 1757ed54c51862d02de1bf183c9fcdb097ae05d5 --- secator/definitions.py | 2 + secator/tasks/theHarvester.py | 78 +++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 secator/tasks/theHarvester.py diff --git a/secator/definitions.py b/secator/definitions.py index 12bb61d5..72dee15b 100644 --- a/secator/definitions.py +++ b/secator/definitions.py @@ -82,6 +82,7 @@ INPUT = 'input' IP = 'ip' JSON = 'json' +LIMIT = 'limit' LINES = 'lines' METHOD = 'method' MATCH_CODES = 'match_codes' @@ -96,6 +97,7 @@ QUIET = 'quiet' RATE_LIMIT = 'rate_limit' RETRIES = 'retries' +SCREENSHOT = 'screenshot' TAGS = 'tags' THREADS = 'threads' TIME = 'time' diff --git a/secator/tasks/theHarvester.py b/secator/tasks/theHarvester.py new file mode 100644 index 00000000..fbe24c12 --- /dev/null +++ b/secator/tasks/theHarvester.py @@ -0,0 +1,78 @@ +import os +import json + +from secator.decorators import task +from secator.definitions import (DELAY, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, PROXY, LIMIT, SCREENSHOT, OPT_NOT_SUPPORTED, TEMP_FOLDER) +from secator.tasks._categories import ReconUser +from secator.utils import get_file_timestamp +from secator.output_types import UserAccount + +@task() +class theHarvester(ReconUser): + """theHarvester is a simple to use, yet powerful tool designed to be used during the reconnaissance stage of a red +team assessment or penetration test.""" + cmd = 'theHarvester ' + file_flag = None + input_flag = '--domain' + json_flag = '--filename' + opt_prefix = '--' + + opts = { + 'source': {'type': str, 'short': 'b', 'help': 'bevigil, censys, fullhunt, securityTrails'}, + } + + opt_key_map = { + PROXY: 'proxies', + LIMIT: 'limit', + SCREENSHOT: 'screenshot', + THREADS: OPT_NOT_SUPPORTED, + DELAY: OPT_NOT_SUPPORTED, + RATE_LIMIT: OPT_NOT_SUPPORTED, + RETRIES: OPT_NOT_SUPPORTED, + TIMEOUT: OPT_NOT_SUPPORTED + } + + @staticmethod + def on_start(self): + output_path = self.get_opt_value('output_path') + if not output_path: + timestr = get_file_timestamp() + output_path = f'{TEMP_FOLDER}/theHarvester_{timestr}.json' + self.output_path = output_path + self.cmd = self.cmd.replace('--filename', f'--filename {self.output_path}') + +def yielder(self): + prev = self.print_item_count + self.print_item_count = False + list(super().yielder()) + if self.return_code != 0: + return + self.results = [] + if os.path.exists(self.output_path): + with open(self.output_path, 'r') as f: + data = json.load(f) + if self.print_orig: # original h8mail output + yield data + return + targets = data['hosts'] + for target in targets: + email = target['target'] + target_data = target['data'] + if not len(target_data) > 0: + continue + entries = target_data[0] + for entry in entries: + source, site_name = tuple(entry.split(':')) + yield UserAccount(**{ + "site_name": site_name, + "username": email.split('@')[0], + "email": email, + "extra_data": { + 'source': source + }, + }) + + + install_cmd = ('git clone https://github.com/laramies/theHarvester || True &&' + 'cd theHarvester || python3 -m pip install -r requirements/base.txt') + socks5_proxy = True \ No newline at end of file From 6b6699ace0d57bd38bc463c6e06b30893354ca03 Mon Sep 17 00:00:00 2001 From: Naboot42 Date: Fri, 4 Aug 2023 11:48:00 +0200 Subject: [PATCH 2/4] add theHarvester Former-commit-id: 270202c40c9ead078e3a854b7c8a9931cf63b276 --- secator/tasks/theHarvester.py | 72 ++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/secator/tasks/theHarvester.py b/secator/tasks/theHarvester.py index fbe24c12..4a3931fc 100644 --- a/secator/tasks/theHarvester.py +++ b/secator/tasks/theHarvester.py @@ -1,26 +1,30 @@ -import os import json +import os + +import validators from secator.decorators import task -from secator.definitions import (DELAY, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, PROXY, LIMIT, SCREENSHOT, OPT_NOT_SUPPORTED, TEMP_FOLDER) +from secator.definitions import (DELAY, LIMIT, OPT_NOT_SUPPORTED, PROXY, + RATE_LIMIT, RETRIES, SCREENSHOT, TEMP_FOLDER, + THREADS, TIMEOUT) +from secator.output_types import Ip, Subdomain, Url, UserAccount from secator.tasks._categories import ReconUser from secator.utils import get_file_timestamp -from secator.output_types import UserAccount + @task() class theHarvester(ReconUser): - """theHarvester is a simple to use, yet powerful tool designed to be used during the reconnaissance stage of a red -team assessment or penetration test.""" - cmd = 'theHarvester ' + """theHarvester is a tool designed to be used during the reconnaissance stage.""" + cmd = 'theHarvester' file_flag = None input_flag = '--domain' json_flag = '--filename' opt_prefix = '--' - + opts = { 'source': {'type': str, 'short': 'b', 'help': 'bevigil, censys, fullhunt, securityTrails'}, - } - + } + opt_key_map = { PROXY: 'proxies', LIMIT: 'limit', @@ -31,7 +35,7 @@ class theHarvester(ReconUser): RETRIES: OPT_NOT_SUPPORTED, TIMEOUT: OPT_NOT_SUPPORTED } - + @staticmethod def on_start(self): output_path = self.get_opt_value('output_path') @@ -41,8 +45,8 @@ def on_start(self): self.output_path = output_path self.cmd = self.cmd.replace('--filename', f'--filename {self.output_path}') -def yielder(self): - prev = self.print_item_count + def yielder(self): + #prev = self.print_item_count self.print_item_count = False list(super().yielder()) if self.return_code != 0: @@ -54,25 +58,33 @@ def yielder(self): if self.print_orig: # original h8mail output yield data return - targets = data['hosts'] - for target in targets: - email = target['target'] - target_data = target['data'] - if not len(target_data) > 0: + hosts = data.get('hosts', []) + #asns = data.get('asns', []) + interesting_urls = data.get('interesting_urls', []) + ips = data.get('ips', []) + emails = data.get('emails', []) + target = self.targets[0] + + for ip in ips: + yield Ip(ip=ip, host=target) + for host in hosts: + parts = host.split(':') + if len(parts) == 1: continue - entries = target_data[0] - for entry in entries: - source, site_name = tuple(entry.split(':')) - yield UserAccount(**{ - "site_name": site_name, - "username": email.split('@')[0], - "email": email, - "extra_data": { - 'source': source - }, - }) + if len(parts) > 2: + host = parts[0] + ip = ':'.join(parts[1:-1]) + else: + host, ip = tuple(parts) + if validators.ip_address.ipv4(ip) or validators.ip_address.ipv6(ip): + yield Ip(ip=ip, host=host) + yield Subdomain(host=host, domain=target) + for interesting_url in interesting_urls: + yield Url(url=interesting_url) + for email in emails: + yield UserAccount(email=emails) - install_cmd = ('git clone https://github.com/laramies/theHarvester || True &&' +install_cmd = ('git clone https://github.com/laramies/theHarvester || True &&' 'cd theHarvester || python3 -m pip install -r requirements/base.txt') - socks5_proxy = True \ No newline at end of file +socks5_proxy = True From a1b85b5f0d0682de43dfb84514cbe1741c43926a Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 31 Jul 2024 09:46:21 +0200 Subject: [PATCH 3/4] Update theHarvester.py --- secator/tasks/theHarvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secator/tasks/theHarvester.py b/secator/tasks/theHarvester.py index 4a3931fc..df2bdd48 100644 --- a/secator/tasks/theHarvester.py +++ b/secator/tasks/theHarvester.py @@ -22,7 +22,7 @@ class theHarvester(ReconUser): opt_prefix = '--' opts = { - 'source': {'type': str, 'short': 'b', 'help': 'bevigil, censys, fullhunt, securityTrails'}, + 'source': {'type': str, 'short': 'b', 'help': 'Modules to use (cf https://github.com/laramies/theHarvester)', 'default': 'all'}, } opt_key_map = { From 0c35973338989324e8894a2e6441ef650eb6aa1f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Oct 2024 07:13:36 -0400 Subject: [PATCH 4/4] update --- secator/definitions.py | 1 - secator/tasks/theHarvester.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/secator/definitions.py b/secator/definitions.py index 8e54e0dc..96066bf3 100644 --- a/secator/definitions.py +++ b/secator/definitions.py @@ -54,7 +54,6 @@ HEADER = 'header' HOST = 'host' IP = 'ip' -JSON = 'json' LIMIT = 'limit' PROTOCOL = 'protocol' LINES = 'lines' diff --git a/secator/tasks/theHarvester.py b/secator/tasks/theHarvester.py index df2bdd48..378dd3ae 100644 --- a/secator/tasks/theHarvester.py +++ b/secator/tasks/theHarvester.py @@ -5,7 +5,7 @@ from secator.decorators import task from secator.definitions import (DELAY, LIMIT, OPT_NOT_SUPPORTED, PROXY, - RATE_LIMIT, RETRIES, SCREENSHOT, TEMP_FOLDER, + RATE_LIMIT, RETRIES, SCREENSHOT, THREADS, TIMEOUT) from secator.output_types import Ip, Subdomain, Url, UserAccount from secator.tasks._categories import ReconUser @@ -41,7 +41,7 @@ def on_start(self): output_path = self.get_opt_value('output_path') if not output_path: timestr = get_file_timestamp() - output_path = f'{TEMP_FOLDER}/theHarvester_{timestr}.json' + output_path = f'{self.reports_folder}/theHarvester_{timestr}.json' self.output_path = output_path self.cmd = self.cmd.replace('--filename', f'--filename {self.output_path}')