From d975a9e51c6442d0285e28e30989c6541327a67e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 29 Nov 2024 05:29:11 -0500 Subject: [PATCH] feat: load wordlists from URLs and config values --- secator/config.py | 127 ++++++++++++++++++++--------------- secator/decorators.py | 1 + secator/runners/command.py | 5 ++ secator/tasks/_categories.py | 4 +- secator/utils.py | 24 ++++++- 5 files changed, 103 insertions(+), 58 deletions(-) diff --git a/secator/config.py b/secator/config.py index d21b2845..8fd2b506 100644 --- a/secator/config.py +++ b/secator/config.py @@ -496,56 +496,78 @@ def download_files(data: dict, target_folder: Path, offline_mode: bool, type: st offline_mode (bool): Offline mode. """ for name, url_or_path in data.items(): - if url_or_path.startswith('git+'): - # Clone Git repository - git_url = url_or_path[4:] # remove 'git+' prefix - repo_name = git_url.split('/')[-1] - if repo_name.endswith('.git'): - repo_name = repo_name[:-4] - target_path = target_folder / repo_name - if not target_path.exists(): - console.print(f'[bold turquoise4]Cloning git {type} [bold magenta]{repo_name}[/] ...[/] ', end='') + target_path = download_file(url_or_path, target_folder, offline_mode, type, name=name) + if target_path: + data[name] = target_path + + +def download_file(url_or_path, target_folder: Path, offline_mode: bool, type: str, name: str = None): + """Download remote file to target folder, clone git repos, or symlink local files. + + Args: + data (dict): Dict of name to url or local path prefixed with 'git+' for Git repos. + target_folder (Path): Target folder for storing files or repos. + offline_mode (bool): Offline mode. + type (str): Type of files to handle. + name (str, Optional): Name of object. + + Returns: + path (Path): Path to downloaded file / folder. + """ + if url_or_path.startswith('git+'): + # Clone Git repository + git_url = url_or_path[4:] # remove 'git+' prefix + repo_name = git_url.split('/')[-1] + if repo_name.endswith('.git'): + repo_name = repo_name[:-4] + target_path = target_folder / repo_name + if not target_path.exists(): + console.print(f'[bold turquoise4]Cloning git {type} [bold magenta]{repo_name}[/] ...[/] ', end='') + if offline_mode: + console.print('[bold orange1]skipped [dim][offline[/].[/]') + return + try: + call(['git', 'clone', git_url, str(target_path)], stderr=DEVNULL, stdout=DEVNULL) + console.print('[bold green]ok.[/]') + except Exception as e: + console.print(f'[bold red]failed ({str(e)}).[/]') + return target_path.resolve() + elif Path(url_or_path).exists(): + # Create a symbolic link for a local file + local_path = Path(url_or_path) + target_path = target_folder / local_path.name + if not name: + name = url_or_path.split('/')[-1] + if not target_path.exists(): + console.print(f'[bold turquoise4]Symlinking {type} [bold magenta]{name}[/] ...[/] ', end='') + try: + target_path.symlink_to(local_path) + console.print('[bold green]ok.[/]') + except Exception as e: + console.print(f'[bold red]failed ({str(e)}).[/]') + return target_path.resolve() + else: + # Download file from URL + ext = url_or_path.split('.')[-1] + if not name: + name = url_or_path.split('/')[-1] + filename = f'{name}.{ext}' if not name.endswith(ext) else name + target_path = target_folder / filename + if not target_path.exists(): + try: + console.print(f'[bold turquoise4]Downloading {type} [bold magenta]{filename}[/] ...[/] ', end='') if offline_mode: - console.print('[bold orange1]skipped [dim][offline[/].[/]') - continue - try: - call(['git', 'clone', git_url, str(target_path)], stderr=DEVNULL, stdout=DEVNULL) - console.print('[bold green]ok.[/]') - except Exception as e: - console.print(f'[bold red]failed ({str(e)}).[/]') - data[name] = target_path.resolve() - elif Path(url_or_path).exists(): - # Create a symbolic link for a local file - local_path = Path(url_or_path) - target_path = target_folder / local_path.name - if not target_path.exists(): - console.print(f'[bold turquoise4]Symlinking {type} [bold magenta]{name}[/] ...[/] ', end='') - try: - target_path.symlink_to(local_path) - console.print('[bold green]ok.[/]') - except Exception as e: - console.print(f'[bold red]failed ({str(e)}).[/]') - data[name] = target_path.resolve() - else: - # Download file from URL - ext = url_or_path.split('.')[-1] - filename = f'{name}.{ext}' if not name.endswith(ext) else name - target_path = target_folder / filename - if not target_path.exists(): - try: - console.print(f'[bold turquoise4]Downloading {type} [bold magenta]{filename}[/] ...[/] ', end='') - if offline_mode: - console.print('[bold orange1]skipped [dim](offline)[/].[/]') - continue - resp = requests.get(url_or_path, timeout=3) - resp.raise_for_status() - with open(target_path, 'wb') as f: - f.write(resp.content) - console.print('[bold green]ok.[/]') - except requests.RequestException as e: - console.print(f'[bold red]failed ({str(e)}).[/]') - continue - data[name] = target_path.resolve() + console.print('[bold orange1]skipped [dim](offline)[/].[/]') + return + resp = requests.get(url_or_path, timeout=3) + resp.raise_for_status() + with open(target_path, 'wb') as f: + f.write(resp.content) + console.print('[bold green]ok.[/]') + except requests.RequestException as e: + console.print(f'[bold red]failed ({str(e)}).[/]') + return + return target_path.resolve() # Load default_config @@ -577,13 +599,8 @@ def download_files(data: dict, target_folder: Path, offline_mode: bool, type: st dir.mkdir(parents=False) console.print('[bold green]ok.[/]') -# Download wordlists and set defaults +# Download wordlists and payloads download_files(CONFIG.wordlists.templates, CONFIG.dirs.wordlists, CONFIG.offline_mode, 'wordlist') -for category, name in CONFIG.wordlists.defaults.items(): - if name in CONFIG.wordlists.templates.keys(): - CONFIG.wordlists.defaults[category] = str(CONFIG.wordlists.templates[name]) - -# Download payloads download_files(CONFIG.payloads.templates, CONFIG.dirs.payloads, CONFIG.offline_mode, 'payload') # Print config diff --git a/secator/decorators.py b/secator/decorators.py index 5a84e392..08b5d1ab 100644 --- a/secator/decorators.py +++ b/secator/decorators.py @@ -228,6 +228,7 @@ def decorator(f): conf.pop('shlex', None) conf.pop('meta', None) conf.pop('supported', None) + conf.pop('process', None) reverse = conf.pop('reverse', False) long = f'--{opt_name}' short = f'-{short_opt}' if short_opt else f'-{opt_name}' diff --git a/secator/runners/command.py b/secator/runners/command.py index 75e00130..ecfd56b5 100644 --- a/secator/runners/command.py +++ b/secator/runners/command.py @@ -666,6 +666,11 @@ def _process_opts( debug('skipped (falsy)', obj={'name': opt_name, 'value': opt_val}, obj_after=False, sub='command.options', verbose=True) # noqa: E501 continue + # Apply process function on opt value + if 'process' in opt_conf: + func = opt_conf['process'] + opt_val = func(opt_val) + # Convert opt value to expected command opt value mapped_opt_val = opt_value_map.get(opt_name) if mapped_opt_val: diff --git a/secator/tasks/_categories.py b/secator/tasks/_categories.py index 2ad1065e..4d40a0d0 100644 --- a/secator/tasks/_categories.py +++ b/secator/tasks/_categories.py @@ -16,7 +16,7 @@ from secator.output_types import Ip, Port, Subdomain, Tag, Url, UserAccount, Vulnerability from secator.config import CONFIG from secator.runners import Command -from secator.utils import debug +from secator.utils import debug, process_wordlist OPTS = { @@ -39,7 +39,7 @@ THREADS: {'type': int, 'help': 'Number of threads to run', 'default': 50}, TIMEOUT: {'type': int, 'help': 'Request timeout'}, USER_AGENT: {'type': str, 'short': 'ua', 'help': 'User agent, e.g "Mozilla Firefox 1.0"'}, - WORDLIST: {'type': str, 'short': 'w', 'default': CONFIG.wordlists.defaults.http, 'help': 'Wordlist to use'} + WORDLIST: {'type': str, 'short': 'w', 'default': 'http', 'process': process_wordlist, 'help': 'Wordlist to use'} } OPTS_HTTP = [ diff --git a/secator/utils.py b/secator/utils.py index 0b03ecf3..491d52da 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -26,7 +26,7 @@ import yaml from secator.definitions import (DEBUG_COMPONENT, VERSION, DEV_PACKAGE) -from secator.config import CONFIG, ROOT_FOLDER, LIB_FOLDER +from secator.config import CONFIG, ROOT_FOLDER, LIB_FOLDER, download_file from secator.rich import console logger = logging.getLogger(__name__) @@ -689,3 +689,25 @@ def merge_two_dicts(dict1, dict2): # Use reduce to apply merge_two_dicts to all dictionaries in dicts return reduce(merge_two_dicts, dicts, {}) + + +def process_wordlist(val): + """Pre-process wordlist option value to allow referencing wordlists from remote URLs or from config keys. + + Args: + val (str): Can be a config value in CONFIG.wordlists.defaults or CONFIG.wordlists.templates, or a local path, + or a URL. + """ + default_wordlist = getattr(CONFIG.wordlists.defaults, val) + if default_wordlist: + val = default_wordlist + template_wordlist = getattr(CONFIG.wordlists.templates, val) + if template_wordlist: + return template_wordlist + else: + return download_file( + val, + target_folder=CONFIG.dirs.wordlists, + offline_mode=CONFIG.offline_mode, + type='wordlist' + )