diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 61d4b634..588aa3ac 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -2,6 +2,7 @@ name: Install secator description: Installs secator inputs: python-version: + description: "Python version" required: true runs: using: "composite" @@ -15,3 +16,7 @@ runs: - name: Install secator with pipx shell: bash run: pipx install -e .[dev] + + - name: Add secator to $PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml new file mode 100644 index 00000000..53856c9c --- /dev/null +++ b/.github/workflows/release-test.yml @@ -0,0 +1,38 @@ +name: release-test + +on: + push: + branches: + - release-please* + +permissions: + contents: write + pull-requests: write + +jobs: + test-docker: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install secator + uses: ./.github/actions/install + with: + python-version: ${{ matrix.python-version }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + run: secator u build docker + + - name: Run secator health check (strict) + run: docker run -it --privileged freelabz/secator:dev health --strict diff --git a/secator/cli.py b/secator/cli.py index acf6e15f..332d317d 100644 --- a/secator/cli.py +++ b/secator/cli.py @@ -823,7 +823,8 @@ def report_export(json_path, output_folder, output): @cli.command(name='health') @click.option('--json', '-json', is_flag=True, default=False, help='JSON lines output') @click.option('--debug', '-debug', is_flag=True, default=False, help='Debug health output') -def health(json, debug): +@click.option('--strict', '-strict', is_flag=True, default=False, help='Fail if missing tools') +def health(json, debug, strict): """[dim]Get health status.[/]""" tools = ALL_TASKS status = {'secator': {}, 'languages': {}, 'tools': {}, 'addons': {}} @@ -870,10 +871,7 @@ def health(json, debug): table = get_health_table() with Live(table, console=console): for tool in tools: - cmd = tool.cmd.split(' ')[0] - version_flag = tool.version_flag or f'{tool.opt_prefix}version' - version_flag = None if tool.version_flag == OPT_NOT_SUPPORTED else version_flag - info = get_version_info(cmd, version_flag, tool.install_github_handle) + info = get_version_info(tool.cmd.split(' ')[0], tool.version_flag, tool.install_github_handle, tool.install_cmd) row = fmt_health_table_row(info, 'tools') table.add_row(*row) status['tools'][tool.__name__] = info @@ -883,6 +881,18 @@ def health(json, debug): import json as _json print(_json.dumps(status)) + # Strict mode + if strict: + error = False + for tool, info in status['tools'].items(): + if not info['installed']: + console.print(f'[bold red]{tool} not installed and strict mode is enabled.[/]') + error = True + if error: + sys.exit(1) + console.print('[bold green]Strict healthcheck passed ![/]') + + #---------# # INSTALL # #---------# @@ -1066,11 +1076,15 @@ def install_tools(cmds): tools = [cls for cls in ALL_TASKS if cls.__name__ in cmds] else: tools = ALL_TASKS - + return_code = 0 for ix, cls in enumerate(tools): with console.status(f'[bold yellow][{ix}/{len(tools)}] Installing {cls.__name__} ...'): - ToolInstaller.install(cls) + status = ToolInstaller.install(cls) + if not status.is_ok(): + console.print(f'[bold red]Failed installing {cls.__name__}[/]') + return_code = 1 console.print() + sys.exit(return_code) @install.command('cves') @@ -1103,22 +1117,52 @@ def install_cves(force): #--------# @cli.command('update') -def update(): +@click.option('--all', '-a', is_flag=True, help='Update all secator dependencies (addons, tools, ...)') +def update(all): """[dim]Update to latest version.[/]""" if CONFIG.offline_mode: console.print('[bold red]Cannot run this command in offline mode.[/]') - return + sys.exit(1) + + # Check current and latest version info = get_version_info('secator', github_handle='freelabz/secator', version=VERSION) latest_version = info['latest_version'] + do_update = True + + # Skip update if latest if info['status'] == 'latest': console.print(f'[bold green]secator is already at the newest version {latest_version}[/] !') - sys.exit(0) - console.print(f'[bold gold3]:wrench: Updating secator from {VERSION} to {latest_version} ...[/]') - if 'pipx' in sys.executable: - Command.execute(f'pipx install secator=={latest_version} --force') - else: - Command.execute(f'pip install secator=={latest_version}') + do_update = False + + # Fail if unknown latest + if not latest_version: + console.print('[bold red]Could not fetch latest secator version.[/]') + sys.exit(1) + + # Update secator + if do_update: + console.print(f'[bold gold3]:wrench: Updating secator from {VERSION} to {latest_version} ...[/]') + if 'pipx' in sys.executable: + ret = Command.execute(f'pipx install secator=={latest_version} --force') + else: + ret = Command.execute(f'pip install secator=={latest_version}') + if not ret.return_code == 0: + sys.exit(1) + # Update tools + if all: + return_code = 0 + for cls in ALL_TASKS: + cmd = cls.cmd.split(' ')[0] + version_flag = cls.version_flag or f'{cls.opt_prefix}version' + version_flag = None if cls.version_flag == OPT_NOT_SUPPORTED else version_flag + info = get_version_info(cmd, version_flag, cls.install_github_handle) + if not info['installed'] or info['status'] == 'outdated' or not info['latest_version']: + with console.status(f'[bold yellow]Installing {cls.__name__} ...'): + status = ToolInstaller.install(cls) + if not status.is_ok(): + return_code = 1 + sys.exit(return_code) #-------# # ALIAS # diff --git a/secator/config.py b/secator/config.py index a10973cc..d21b2845 100644 --- a/secator/config.py +++ b/secator/config.py @@ -67,7 +67,7 @@ class Celery(StrictModel): class Cli(StrictModel): - github_token: str = '' + github_token: str = os.environ.get('GITHUB_TOKEN', '') record: bool = False stdin_timeout: int = 1000 diff --git a/secator/installer.py b/secator/installer.py index ec81afac..4dc7b95a 100644 --- a/secator/installer.py +++ b/secator/installer.py @@ -1,21 +1,42 @@ - -import requests import os import platform +import re import shutil import tarfile import zipfile import io +from enum import Enum + +import json +import requests + from rich.table import Table +from secator.definitions import OPT_NOT_SUPPORTED from secator.rich import console from secator.runners import Command from secator.config import CONFIG +class InstallerStatus(Enum): + SUCCESS = 'SUCCESS' + INSTALL_NOT_SUPPORTED = 'INSTALL_NOT_SUPPORTED' + GITHUB_LATEST_RELEASE_NOT_FOUND = 'GITHUB_LATEST_RELEASE_NOT_FOUND' + GITHUB_RELEASE_NOT_FOUND = 'RELEASE_NOT_FOUND' + GITHUB_RELEASE_FAILED_DOWNLOAD = 'GITHUB_RELEASE_FAILED_DOWNLOAD' + GITHUB_BINARY_NOT_FOUND_IN_ARCHIVE = 'GITHUB_BINARY_NOT_FOUND_IN_ARCHIVE' + SOURCE_INSTALL_FAILED = 'SOURCE_INSTALL_FAILED' + UNKNOWN = 'UNKNOWN' + + def is_ok(self): + return self.value in ['SUCCESS', 'INSTALL_NOT_SUPPORTED'] + + class ToolInstaller: + status = InstallerStatus + @classmethod def install(cls, tool_cls): """Install a tool. @@ -25,29 +46,29 @@ def install(cls, tool_cls): tool_cls: Tool class (derived from secator.runners.Command). Returns: - bool: True if install is successful, False otherwise. + InstallerStatus: Install status. """ console.print(f'[bold gold3]:wrench: Installing {tool_cls.__name__}') - success = False + status = InstallerStatus.UNKNOWN if not tool_cls.install_github_handle and not tool_cls.install_cmd: console.print( f'[bold red]{tool_cls.__name__} install is not supported yet. Please install it manually.[/]') - return False + status = InstallerStatus.INSTALL_NOT_SUPPORTED if tool_cls.install_github_handle: - success = GithubInstaller.install(tool_cls.install_github_handle) + status = GithubInstaller.install(tool_cls.install_github_handle) - if tool_cls.install_cmd and not success: - success = SourceInstaller.install(tool_cls.install_cmd) + if tool_cls.install_cmd and not status.is_ok(): + status = SourceInstaller.install(tool_cls.install_cmd) - if success: + if status == InstallerStatus.SUCCESS: console.print( f'[bold green]:tada: {tool_cls.__name__} installed successfully[/] !') else: console.print( - f'[bold red]:exclamation_mark: Failed to install {tool_cls.__name__}.[/]') - return success + f'[bold red]:exclamation_mark: Failed to install {tool_cls.__name__}: {status}.[/]') + return status class SourceInstaller: @@ -62,10 +83,10 @@ def install(cls, install_cmd): install_cmd (str): Install command. Returns: - bool: True if install is successful, False otherwise. + Status: install status. """ ret = Command.execute(install_cmd, cls_attributes={'shell': True}) - return ret.return_code == 0 + return InstallerStatus.SUCCESS if ret.return_code == 0 else InstallerStatus.SOURCE_INSTALL_FAILED class GithubInstaller: @@ -79,24 +100,23 @@ def install(cls, github_handle): github_handle (str): A GitHub handle {user}/{repo} Returns: - bool: True if install is successful, False otherwise. + InstallerStatus: status. """ _, repo = tuple(github_handle.split('/')) latest_release = cls.get_latest_release(github_handle) if not latest_release: - return False + return InstallerStatus.GITHUB_LATEST_RELEASE_NOT_FOUND # Find the right asset to download os_identifiers, arch_identifiers = cls._get_platform_identifier() download_url = cls._find_matching_asset(latest_release['assets'], os_identifiers, arch_identifiers) if not download_url: console.print('[dim red]Could not find a GitHub release matching distribution.[/]') - return False + return InstallerStatus.GITHUB_RELEASE_NOT_FOUND # Download and unpack asset console.print(f'Found release URL: {download_url}') - cls._download_and_unpack(download_url, CONFIG.dirs.bin, repo) - return True + return cls._download_and_unpack(download_url, CONFIG.dirs.bin, repo) @classmethod def get_latest_release(cls, github_handle): @@ -181,10 +201,21 @@ def _find_matching_asset(cls, assets, os_identifiers, arch_identifiers): @classmethod def _download_and_unpack(cls, url, destination, repo_name): - """Download and unpack a release asset.""" + """Download and unpack a release asset. + + Args: + cls (Runner): Task class. + url (str): GitHub release URL. + destination (str): Local destination. + repo_name (str): GitHub repository name. + + Returns: + InstallerStatus: install status. + """ console.print(f'Downloading and unpacking to {destination}...') response = requests.get(url, timeout=5) - response.raise_for_status() + if not response.status_code == 200: + return InstallerStatus.GITHUB_RELEASE_FAILED_DOWNLOAD # Create a temporary directory to extract the archive temp_dir = os.path.join("/tmp", repo_name) @@ -202,8 +233,10 @@ def _download_and_unpack(cls, url, destination, repo_name): if binary_path: os.chmod(binary_path, 0o755) # Make it executable shutil.move(binary_path, os.path.join(destination, repo_name)) # Move the binary + return InstallerStatus.SUCCESS else: console.print('[bold red]Binary matching the repository name was not found in the archive.[/]') + return InstallerStatus.GITHUB_BINARY_NOT_FOUND_IN_ARCHIVE @classmethod def _find_binary_in_directory(cls, directory, binary_name): @@ -235,31 +268,47 @@ def get_version(version_cmd): version_cmd (str): Command to get the version. Returns: - str: Version string. + tuple[str]: Version string, return code. """ from secator.runners import Command import re regex = r'[0-9]+\.[0-9]+\.?[0-9]*\.?[a-zA-Z]*' ret = Command.execute(version_cmd, quiet=True, print_errors=False) + return_code = ret.return_code + if not return_code == 0: + return '', ret.return_code match = re.findall(regex, ret.output) if not match: - return '' - return match[0] + return '', return_code + return match[0], return_code + + +def parse_version(ver): + from packaging import version as _version + try: + return _version.parse(ver) + except _version.InvalidVersion: + version_regex = re.compile(r'(\d+\.\d+(?:\.\d+)?)') + match = version_regex.search(ver) + if match: + return _version.parse(match.group(1)) + return None -def get_version_info(name, version_flag=None, github_handle=None, version=None): +def get_version_info(name, version_flag, install_github_handle=None, install_cmd=None, version=None, opt_prefix='--'): """Get version info for a command. Args: name (str): Command name. version_flag (str): Version flag. - github_handle (str): Github handle. + install_github_handle (str): Github handle. + install_cmd (str): Install command. version (str): Existing version. + opt_prefix (str, default: '--'): Option prefix. Return: dict: Version info. """ - from packaging import version as _version from secator.installer import GithubInstaller info = { 'name': name, @@ -274,22 +323,50 @@ def get_version_info(name, version_flag=None, github_handle=None, version=None): location = which(name).output info['location'] = location + # Get latest version + latest_version = None + if not CONFIG.offline_mode: + if install_github_handle: + latest_version = GithubInstaller.get_latest_version(install_github_handle) + info['latest_version'] = latest_version + elif install_cmd and install_cmd.startswith('pip'): + req = requests.get(f'https://pypi.python.org/pypi/{name}/json') + version = parse_version('0') + if req.status_code == requests.codes.ok: + j = json.loads(req.text.encode(req.encoding)) + releases = j.get('releases', []) + for release in releases: + ver = parse_version(release) + if ver and not ver.is_prerelease: + version = max(version, ver) + latest_version = str(version) + info['latest_version'] = latest_version + elif install_cmd and install_cmd.startswith('sudo apt install'): + ret = Command.execute(f'apt-cache madison {name}', quiet=True) + if ret.return_code == 0: + output = ret.output.split(' | ') + if len(output) > 1: + ver = parse_version(output[1].strip()) + if ver: + latest_version = str(ver) + info['latest_version'] = latest_version + # Get current version + version_ret = 1 + version_flag = None if version_flag == OPT_NOT_SUPPORTED else version_flag or f'{opt_prefix}version' if version_flag: version_cmd = f'{name} {version_flag}' - version = get_version(version_cmd) + version, version_ret = get_version(version_cmd) info['version'] = version - - # Get latest version - latest_version = None - if not CONFIG.offline_mode: - latest_version = GithubInstaller.get_latest_version(github_handle) - info['latest_version'] = latest_version + if version_ret != 0: # version command error + info['installed'] = False + info['status'] = 'missing' + return info if location: info['installed'] = True if version and latest_version: - if _version.parse(version) < _version.parse(latest_version): + if parse_version(version) < parse_version(latest_version): info['status'] = 'outdated' else: info['status'] = 'latest' @@ -310,6 +387,7 @@ def fmt_health_table_row(version_info, category=None): version = version_info['version'] status = version_info['status'] installed = version_info['installed'] + latest_version = version_info['latest_version'] name_str = f'[magenta]{name:<13}[/]' # Format version row @@ -319,6 +397,8 @@ def fmt_health_table_row(version_info, category=None): _version += ' [bold green](latest)[/]' elif status == 'outdated': _version += ' [bold red](outdated)[/]' + if latest_version: + _version += f' [dim](<{latest_version})' elif status == 'missing': _version = '[bold red]missing[/]' elif status == 'ok': diff --git a/secator/runners/_base.py b/secator/runners/_base.py index 84ac04fc..13a3e3ba 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -557,7 +557,7 @@ def run_validators(self, validator_type, *args, error=True): name = f'{self.__class__.__name__}.{validator_type}' fun = self.get_func_path(validator) if not validator(self, *args): - self.debug('', obj={name + ' [dim yellow]->[/] ' + fun: 'failed'}, id=_id, sub='validators') + self.debug('', obj={name + ' [dim yellow]->[/] ' + fun: '[dim red]failed[/]'}, id=_id, verbose=True, sub='validators') # noqa: E501 doc = validator.__doc__ if error: message = 'Validator failed' @@ -570,7 +570,7 @@ def run_validators(self, validator_type, *args, error=True): ) self.add_result(error, print=True) return False - self.debug('', obj={name + ' [dim yellow]->[/] ' + fun: 'success'}, id=_id, sub='validators') + self.debug('', obj={name + ' [dim yellow]->[/] ' + fun: '[dim green]success[/]'}, id=_id, verbose=True, sub='validators') # noqa: E501 return True def register_hooks(self, hooks): diff --git a/secator/runners/command.py b/secator/runners/command.py index ed8fe2dc..75e00130 100644 --- a/secator/runners/command.py +++ b/secator/runners/command.py @@ -578,9 +578,10 @@ def _prompt_sudo(self, command): return -1, error # If not, prompt the user for a password - self._print('[bold red]Please enter sudo password to continue.[/]') + self._print('[bold red]Please enter sudo password to continue.[/]', rich=True) for _ in range(3): - self._print('\[sudo] password: ') + user = getpass.getuser() + self._print(f'\[sudo] password for {user}: ▌', rich=True) sudo_password = getpass.getpass() result = subprocess.run( ['sudo', '-S', '-p', '', 'true'], diff --git a/secator/tasks/bbot.py b/secator/tasks/bbot.py index fbb32712..464fcfc0 100644 --- a/secator/tasks/bbot.py +++ b/secator/tasks/bbot.py @@ -155,6 +155,7 @@ class bbot(Command): json_flag = '--json' input_flag = '-t' file_flag = None + version_flag = '--help' opts = { 'modules': {'type': str, 'short': 'm', 'default': '', 'help': ','.join(BBOT_MODULES)}, 'presets': {'type': str, 'short': 'ps', 'default': 'kitchen-sink', 'help': ','.join(BBOT_PRESETS), 'shlex': False}, diff --git a/secator/tasks/dirsearch.py b/secator/tasks/dirsearch.py index f8bebdd7..ccd478e4 100644 --- a/secator/tasks/dirsearch.py +++ b/secator/tasks/dirsearch.py @@ -20,7 +20,7 @@ class dirsearch(HttpFuzzer): cmd = 'dirsearch' input_flag = '-u' file_flag = '-l' - json_flag = '--format json' + json_flag = '-O json' opt_prefix = '--' encoding = 'ansi' opt_key_map = { @@ -52,7 +52,7 @@ class dirsearch(HttpFuzzer): STATUS_CODE: 'status' } } - install_cmd = 'pipx install dirsearch' + install_cmd = 'pipx install git+https://github.com/maurosoria/dirsearch' proxychains = True proxy_socks5 = True proxy_http = True diff --git a/secator/tasks/maigret.py b/secator/tasks/maigret.py index 319412e7..0bb7d6e0 100644 --- a/secator/tasks/maigret.py +++ b/secator/tasks/maigret.py @@ -41,7 +41,7 @@ class maigret(ReconUser): EXTRA_DATA: lambda x: x['status'].get('ids', {}) } } - install_cmd = 'pipx install git+https://github.com/soxoj/maigret@6be2f409e58056b1ca8571a8151e53bef107dedc' + install_cmd = 'pipx install git+https://github.com/soxoj/maigret' socks5_proxy = True profile = 'io' diff --git a/secator/tasks/nuclei.py b/secator/tasks/nuclei.py index ef86484b..7c5d786e 100644 --- a/secator/tasks/nuclei.py +++ b/secator/tasks/nuclei.py @@ -74,7 +74,7 @@ class nuclei(VulnMulti): } } ignore_return_code = True - install_cmd = 'go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest && nuclei update-templates' + install_cmd = 'go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest' install_github_handle = 'projectdiscovery/nuclei' proxychains = False proxy_socks5 = True # kind of, leaks data when running network / dns templates diff --git a/secator/tasks/wpscan.py b/secator/tasks/wpscan.py index f86d4913..a2535fcd 100644 --- a/secator/tasks/wpscan.py +++ b/secator/tasks/wpscan.py @@ -66,7 +66,7 @@ class wpscan(VulnHttp): }, } output_types = [Vulnerability, Tag] - install_cmd = 'sudo apt install -y build-essential ruby-dev rubygems && sudo gem install wpscan' + install_cmd = 'sudo apt install -y build-essential ruby-dev rubygems && sudo apt install -y libcurl4t64 || true && sudo gem install wpscan' # noqa: E501 proxychains = False proxy_http = True proxy_socks5 = False diff --git a/secator/utils.py b/secator/utils.py index d1cb71fb..0b03ecf3 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -414,7 +414,7 @@ def print_version(): """Print secator version information.""" from secator.installer import get_version_info console.print(f'[bold gold3]Current version[/]: {VERSION}', highlight=False, end='') - info = get_version_info('secator', github_handle='freelabz/secator', version=VERSION) + info = get_version_info('secator', install_github_handle='freelabz/secator', version=VERSION) latest_version = info['latest_version'] status = info['status'] location = info['location']