feat: rework install to use pre-packaged binaries when possible (#253)

.github/workflows/tests.yml

@@ -82,6 +82,8 @@ jobs:
         secator install langs go
         secator install langs ruby
         secator install tools
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # to avoid being rate-limited when fetching GitHub releases
 
     - name: Run integration tests (${{ matrix.test_type }})
       run: |

secator/cli.py

@@ -18,6 +18,7 @@
 								 OPT_NOT_SUPPORTED, PAYLOADS_FOLDER,
 								 REDIS_ADDON_ENABLED, REVSHELLS_FOLDER, ROOT_FOLDER,
 								 TRACE_ADDON_ENABLED, VERSION, WORKER_ADDON_ENABLED)
+from secator.installer import ToolInstaller
 from secator.rich import console
 from secator.runners import Command
 from secator.serializers.dataclass import loads_dataclass
@@ -476,7 +477,7 @@ def install_tools(cmds):
 
 	for ix, cls in enumerate(tools):
 		with console.status(f'[bold yellow][{ix}/{len(tools)}] Installing {cls.__name__} ...'):
-			cls.install()
+			ToolInstaller.install(cls)
 		console.print()
 
 

secator/definitions.py

@@ -24,6 +24,7 @@
 LIB_FOLDER = ROOT_FOLDER + '/secator'
 CONFIGS_FOLDER = LIB_FOLDER + '/configs'
 EXTRA_CONFIGS_FOLDER = os.environ.get('SECATOR_EXTRA_CONFIGS_FOLDER')
+BIN_FOLDER = os.environ.get('SECATOR_BIN_FOLDER', f'{os.path.expanduser("~")}/.local/bin')
 DATA_FOLDER = os.environ.get('SECATOR_DATA_FOLDER', f'{os.path.expanduser("~")}/.secator')
 REPORTS_FOLDER = os.environ.get('SECATOR_REPORTS_FOLDER', f'{DATA_FOLDER}/reports')
 WORDLISTS_FOLDER = os.environ.get('SECATOR_WORDLISTS_FOLDER', f'{DATA_FOLDER}/wordlists')
@@ -32,6 +33,7 @@
 PAYLOADS_FOLDER = f'{DATA_FOLDER}/payloads'
 REVSHELLS_FOLDER = f'{DATA_FOLDER}/revshells'
 TESTS_FOLDER = f'{ROOT_FOLDER}/tests'
+os.makedirs(BIN_FOLDER, exist_ok=True)
 os.makedirs(DATA_FOLDER, exist_ok=True)
 os.makedirs(REPORTS_FOLDER, exist_ok=True)
 os.makedirs(WORDLISTS_FOLDER, exist_ok=True)
@@ -58,6 +60,7 @@
 CELERY_OVERRIDE_DEFAULT_LOGGING = bool(int(os.environ.get('CELERY_OVERRIDE_DEFAULT_LOGGING', 1)))
 GOOGLE_DRIVE_PARENT_FOLDER_ID = os.environ.get('GOOGLE_DRIVE_PARENT_FOLDER_ID')
 GOOGLE_CREDENTIALS_PATH = os.environ.get('GOOGLE_CREDENTIALS_PATH')
+GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
 
 # Defaults HTTP and Proxy settings
 DEFAULT_SOCKS5_PROXY = os.environ.get('SOCKS5_PROXY', "socks5://127.0.0.1:9050")

secator/installer.py

@@ -0,0 +1,192 @@
+
+import requests
+import os
+import platform
+import shutil
+import tarfile
+import zipfile
+import io
+
+from secator.rich import console
+from secator.runners import Command
+from secator.definitions import BIN_FOLDER, GITHUB_TOKEN
+
+
+class ToolInstaller:
+
+	@classmethod
+	def install(cls, tool_cls):
+		"""Install a tool.
+
+		Args:
+			cls: ToolInstaller class.
+			tool_cls: Tool class (derived from secator.runners.Command).
+
+		Returns:
+			bool: True if install is successful, False otherwise.
+		"""
+		console.print(f'[bold gold3]:wrench: Installing {tool_cls.__name__}')
+		success = False
+
+		if not tool_cls.install_github_handle and not tool_cls.install_cmd:
+			console.print(
+				f'[bold red]{tool_cls.__name__} install is not supported yet. Please install it manually.[/]')
+			return False
+
+		if tool_cls.install_github_handle:
+			success = GithubInstaller.install(tool_cls.install_github_handle)
+
+		if tool_cls.install_cmd and not success:
+			success = SourceInstaller.install(tool_cls.install_cmd)
+
+		if success:
+			console.print(
+				f'[bold green]:tada: {tool_cls.__name__} installed successfully[/] !')
+		else:
+			console.print(
+				f'[bold red]:exclamation_mark: Failed to install {tool_cls.__name__}.[/]')
+		return success
+
+
+class SourceInstaller:
+	"""Install a tool from source."""
+
+	@classmethod
+	def install(cls, install_cmd):
+		"""Install from source.
+
+		Args:
+			cls: ToolInstaller class.
+			install_cmd (str): Install command.
+
+		Returns:
+			bool: True if install is successful, False otherwise.
+		"""
+		ret = Command.execute(install_cmd, cls_attributes={'shell': True})
+		return ret.return_code == 0
+
+
+class GithubInstaller:
+	"""Install a tool from GitHub releases."""
+
+	@classmethod
+	def install(cls, github_handle):
+		"""Find and install a release from a GitHub handle {user}/{repo}.
+
+		Args:
+			github_handle (str): A GitHub handle {user}/{repo}
+
+		Returns:
+			bool: True if install is successful,, False otherwise.
+		"""
+		owner, repo = tuple(github_handle.split('/'))
+		releases_url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest"
+
+		# Query latest release endpoint
+		headers = {}
+		if GITHUB_TOKEN:
+			headers['Authorization'] = f'Bearer {GITHUB_TOKEN}'
+		response = requests.get(releases_url, headers=headers)
+		if response.status_code == 403:
+			console.print('[bold red]Rate-limited by GitHub API. Retry later or set a GITHUB_TOKEN.')
+			return False
+		elif response.status_code == 404:
+			console.print('[dim red]No GitHub releases found.')
+			return False
+
+		# Find the right asset to download
+		latest_release = response.json()
+		os_identifiers, arch_identifiers = cls._get_platform_identifier()
+		download_url = cls._find_matching_asset(latest_release['assets'], os_identifiers, arch_identifiers)
+		if not download_url:
+			console.print('[dim red]Could not find a GitHub release matching distribution.[/]')
+			return False
+
+		# Download and unpack asset
+		console.print(f'Found release URL: {download_url}')
+		cls._download_and_unpack(download_url, BIN_FOLDER, repo)
+		return True
+
+	@classmethod
+	def _get_platform_identifier(cls):
+		"""Generate lists of possible identifiers for the current platform."""
+		system = platform.system().lower()
+		arch = platform.machine().lower()
+
+		# Mapping common platform.system() values to those found in release names
+		os_mapping = {
+			'linux': ['linux'],
+			'windows': ['windows', 'win'],
+			'darwin': ['darwin', 'macos', 'osx', 'mac']
+		}
+
+		# Enhanced architecture mapping to avoid conflicts
+		arch_mapping = {
+			'x86_64': ['amd64', 'x86_64'],
+			'amd64': ['amd64', 'x86_64'],
+			'aarch64': ['arm64', 'aarch64'],
+			'armv7l': ['armv7', 'arm'],
+			'386': ['386', 'x86', 'i386'],
+		}
+
+		os_identifiers = os_mapping.get(system, [])
+		arch_identifiers = arch_mapping.get(arch, [])
+		return os_identifiers, arch_identifiers
+
+	@classmethod
+	def _find_matching_asset(cls, assets, os_identifiers, arch_identifiers):
+		"""Find a release asset matching the current platform more precisely."""
+		potential_matches = []
+
+		for asset in assets:
+			asset_name = asset['name'].lower()
+			if any(os_id in asset_name for os_id in os_identifiers) and \
+			   any(arch_id in asset_name for arch_id in arch_identifiers):
+				potential_matches.append(asset['browser_download_url'])
+
+		# Preference ordering for file formats, if needed
+		preferred_formats = ['.tar.gz', '.zip']
+
+		for format in preferred_formats:
+			for match in potential_matches:
+				if match.endswith(format):
+					return match
+
+		if potential_matches:
+			return potential_matches[0]
+
+	@classmethod
+	def _download_and_unpack(cls, url, destination, repo_name):
+		"""Download and unpack a release asset."""
+		console.print(f'Downloading and unpacking to {destination}...')
+		response = requests.get(url)
+		response.raise_for_status()
+
+		# Create a temporary directory to extract the archive
+		temp_dir = os.path.join("/tmp", repo_name)
+		os.makedirs(temp_dir, exist_ok=True)
+
+		if url.endswith('.zip'):
+			with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:
+				zip_ref.extractall(temp_dir)
+		elif url.endswith('.tar.gz'):
+			with tarfile.open(fileobj=io.BytesIO(response.content), mode='r:gz') as tar:
+				tar.extractall(path=temp_dir)
+
+		# For archives, find and move the binary that matches the repo name
+		binary_path = cls._find_binary_in_directory(temp_dir, repo_name)
+		if binary_path:
+			os.chmod(binary_path, 0o755)  # Make it executable
+			shutil.move(binary_path, os.path.join(destination, repo_name))  # Move the binary
+		else:
+			console.print('[bold red]Binary matching the repository name was not found in the archive.[/]')
+
+	@classmethod
+	def _find_binary_in_directory(cls, directory, binary_name):
+		"""Search for the binary in the given directory that matches the repository name."""
+		for root, _, files in os.walk(directory):
+			for file in files:
+				# Match the file name exactly with the repository name
+				if file == binary_name:
+					return os.path.join(root, file)
+		return None

secator/runners/command.py

@@ -16,7 +16,6 @@
 							   DEFAULT_PROXYCHAINS_COMMAND,
 							   DEFAULT_SOCKS5_PROXY, OPT_NOT_SUPPORTED,
 							   OPT_PIPE_INPUT, DEFAULT_INPUT_CHUNK_SIZE)
-from secator.rich import console
 from secator.runners import Runner
 from secator.serializers import JSONSerializer
 from secator.utils import debug
@@ -81,8 +80,9 @@ class Command(Runner):
 	# Flag to show version
 	version_flag = None
 
-	# Install command
+	# Install
 	install_cmd = None
+	install_github_handle = None
 
 	# Serializer
 	item_loader = None
@@ -252,20 +252,6 @@ def convert(d):
 	# Class methods #
 	#---------------#
 
-	@classmethod
-	def install(cls):
-		"""Install command by running the content of cls.install_cmd."""
-		console.print(f':heavy_check_mark: Installing {cls.__name__}...', style='bold yellow')
-		if not cls.install_cmd:
-			console.print(f'{cls.__name__} install is not supported yet. Please install it manually.', style='bold red')
-			return
-		ret = cls.execute(cls.install_cmd, name=cls.__name__, cls_attributes={'shell': True})
-		if ret.return_code != 0:
-			console.print(f':exclamation_mark: Failed to install {cls.__name__}.', style='bold red')
-		else:
-			console.print(f':tada: {cls.__name__} installed successfully !', style='bold green')
-		return ret
-
 	@classmethod
 	def execute(cls, cmd, name=None, cls_attributes={}, **kwargs):
 		"""Execute an ad-hoc command.

secator/tasks/dalfox.py

@@ -53,6 +53,7 @@ class dalfox(VulnHttp):
 		}
 	}
 	install_cmd = 'go install -v github.com/hahwul/dalfox/v2@latest'
+	install_github_handle = 'hahwul/dalfox'
 	encoding = 'ansi'
 	proxychains = False
 	proxychains_flavor = 'proxychains4'

secator/tasks/dnsx.py

@@ -25,6 +25,7 @@ class dnsx(ReconDns):
 	}
 
 	install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest'
+	install_github_handle = 'projectdiscovery/dnsx'
 	profile = 'io'
 
 	@staticmethod

secator/tasks/dnsxbrute.py

@@ -31,4 +31,5 @@ class dnsxbrute(ReconDns):
         }
     }
     install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest'
+    install_github_handle = 'projectdiscovery/dnsx'
     profile = 'cpu'

secator/tasks/feroxbuster.py

@@ -64,6 +64,7 @@ class feroxbuster(HttpFuzzer):
 		'curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | '
 		'bash && sudo mv feroxbuster /usr/local/bin'
 	)
+	install_github_handle = 'epi052/feroxbuster'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True

secator/tasks/ffuf.py

@@ -71,6 +71,7 @@ class ffuf(HttpFuzzer):
 	}
 	encoding = 'ansi'
 	install_cmd = f'go install -v github.com/ffuf/ffuf@latest && sudo git clone https://github.com/danielmiessler/SecLists {WORDLISTS_FOLDER}/seclists || true'  # noqa: E501
+	install_github_handle = 'ffuf/ffuf'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True

secator/tasks/gau.py

@@ -37,6 +37,7 @@ class gau(HttpCrawler):
 		USER_AGENT: OPT_NOT_SUPPORTED,
 	}
 	install_cmd = 'go install -v github.com/lc/gau/v2/cmd/gau@latest'
+	install_github_handle = 'lc/gau'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True

secator/tasks/gospider.py

@@ -52,6 +52,7 @@ class gospider(HttpCrawler):
 		}
 	}
 	install_cmd = 'go install -v github.com/jaeles-project/gospider@latest'
+	install_github_handle = 'jaeles-project/gospider'
 	ignore_return_code = True
 	proxychains = False
 	proxy_socks5 = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61

secator/tasks/grype.py

@@ -30,6 +30,7 @@ class grype(VulnCode):
 	install_cmd = (
 		'curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin'
 	)
+	install_github_handle = 'anchore/grype'
 
 	@staticmethod
 	def item_loader(self, line):

secator/tasks/httpx.py

@@ -60,6 +60,7 @@ class httpx(Http):
 		DELAY: lambda x: str(x) + 's' if x else None,
 	}
 	install_cmd = 'go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest'
+	install_github_handle = 'projectdiscovery/httpx'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True

secator/tasks/katana.py

@@ -71,6 +71,7 @@ class katana(HttpCrawler):
 	}
 	item_loaders = []
 	install_cmd = 'sudo apt install build-essential && go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
+	install_github_handle = 'projectdiscovery/katana'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True

secator/tasks/mapcidr.py

@@ -14,6 +14,7 @@ class mapcidr(ReconIp):
 	input_flag = '-cidr'
 	file_flag = '-cl'
 	install_cmd = 'go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidr@latest'
+	install_github_handle = 'projectdiscovery/mapcidr'
 	input_type = CIDR_RANGE
 	output_types = [Ip]
 	opt_key_map = {

secator/tasks/naabu.py

@@ -46,6 +46,7 @@ class naabu(ReconPort):
 	}
 	output_types = [Port]
 	install_cmd = 'sudo apt install -y build-essential libpcap-dev && go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest'  # noqa: E501
+	install_github_handle = 'projectdiscovery/naabu'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = False

secator/tasks/nuclei.py

@@ -68,6 +68,7 @@ class nuclei(VulnMulti):
 	}
 	ignore_return_code = True
 	install_cmd = 'go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest && nuclei update-templates'
+	install_github_handle = 'projectdiscovery/nuclei'
 	proxychains = False
 	proxy_socks5 = True  # kind of, leaks data when running network / dns templates
 	proxy_http = True  # same

secator/tasks/subfinder.py

@@ -30,6 +30,7 @@ class subfinder(ReconDns):
 	}
 	output_types = [Subdomain]
 	install_cmd = 'go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest'
+	install_github_handle = 'projectdiscovery/subfinder'
 	proxychains = False
 	proxy_http = True
 	proxy_socks5 = False