Merge remote-tracking branch 'origin/master' into unblob-2

.github/workflows/build_ci.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   build-ci:
-    runs-on: [ self-hosted, linux, x64, focal ]
+    runs-on: [ self-hosted, linux, x64, jammy ]
     timeout-minutes: 45
     steps:
       - name: Add Masks

Dockerfile

@@ -1,4 +1,4 @@
-FROM phusion/baseimage:jammy-1.0.1
+FROM phusion/baseimage:jammy-1.0.3
 
 WORKDIR /opt/app
 

README.md

@@ -30,7 +30,7 @@ fact_extractor/install/pre_install.sh
 fact_extractor/install.py
 ```
 
-:warning: **We no longer support Ubuntu 16.04 and Python <3.7** 
+:warning: **We no longer support Ubuntu 18.04 and Python <3.8** 
 (It may still work with a bit of tinkering, though)
 
 :warning: For the `generic_fs` unpacker plugin to work with all file system types, you may need to install extra kernel modules

extract.py

@@ -82,7 +82,7 @@ def call_docker(input_file, container, target, report_file, memory_limit, tmpdir
 
         shutil.copy(input_file, str(Path(tmpdir.name, 'input', Path(input_file).name)))
 
-        command = f'docker run --rm -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}'
+        command = f'docker run --rm --ulimit nofile=20000:50000 -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}'
         subprocess.run(command, shell=True)
 
         with suppress(shutil.Error):

fact_extractor/docker_extraction.py

@@ -22,7 +22,7 @@
 
 from helperFunctions.config import get_config_dir
 from helperFunctions.file_system import change_owner_of_output_files
-from helperFunctions.program_setup import load_config, setup_logging
+from helperFunctions.program_setup import check_ulimits, load_config, setup_logging
 from unpacker.unpack import unpack
 
 
@@ -43,6 +43,7 @@ def _parse_args():
 def main(args):
     config = load_config(f'{get_config_dir()}/main.cfg')
     setup_logging(debug=False)
+    check_ulimits()
 
     input_dir = Path(config.get('unpack', 'data_folder'), 'input')
     input_file = list(input_dir.iterdir())[0]

fact_extractor/helperFunctions/install.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import configparser
 import logging
 import os
@@ -104,7 +106,9 @@ def pip_install_packages(*packages):
     for packet in packages:
         try:
             run_shell_command_raise_on_return_code(
-                f'{pip_command} install --upgrade {packet}', f'Error in installation of python package {packet}', True
+                f'{pip_command} install --upgrade "{packet}"',
+                f'Error in installation of python package {packet}',
+                True
             )
         except InstallationError as installation_error:
             if 'is a distutils installed project' in str(installation_error):
@@ -113,6 +117,13 @@ def pip_install_packages(*packages):
                 raise installation_error
 
 
+def load_requirements_file(path: Path) -> list[str]:
+    return [
+        line for line in path.read_text().splitlines()
+        if line and not line.startswith('#')
+    ]
+
+
 def check_if_command_in_path(command):
     _, return_code = execute_shell_command_get_return_code(f'command -v {command}')
     if return_code != 0:

fact_extractor/helperFunctions/program_setup.py

@@ -1,6 +1,7 @@
 import argparse
 import configparser
 import logging
+import resource
 
 from common_helper_files import create_dir_for_file
 
@@ -39,6 +40,16 @@ def setup_logging(debug, log_file=None, log_level=None):
     logger.addHandler(console_log)
 
 
+def check_ulimits():
+    # Get number of openable files
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if soft < 1024:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (min(1024, hard), hard))
+        logging.info(f'The number of openable files has been raised from {soft} to {min(1024, hard)}.')
+    elif soft == resource.RLIM_INFINITY or soft > 100000:
+        logging.warning('Warning: A very high (or no) nofile limit will slow down fakeroot and cause other problems.')
+
+
 def load_config(config_file):
     config = configparser.ConfigParser()
     config.read(config_file)

fact_extractor/install/common.py

@@ -4,56 +4,46 @@
 from pathlib import Path
 
 from helperFunctions.config import load_config
-from helperFunctions.install import apt_install_packages, apt_update_sources, pip_install_packages
+from helperFunctions.install import (
+    apt_install_packages, apt_update_sources, pip_install_packages, load_requirements_file
+)
 
-
-DEPENDENCIES = {
+APT_DEPENDENCIES = {
     # Ubuntu
-    'bionic': {},
-    'focal': {},
-    'jammy': {},
+    'bionic': [],
+    'focal': [],
+    'jammy': [],
     # Debian
-    'buster': {},
-    'bullseye': {},
+    'buster': [],
+    'bullseye': [],
     # Packages common to all platforms
-    'common': {
-        'apt': [
-            # Non python dependencies
-            'build-essential',
-            'automake',
-            'autoconf',
-            'libtool',
-            # Python dependencies
-            'python3',
-            'python3-dev',
-            'python-wheel-common',
-        ],
-        'pip3': [
-            'flask',
-            'flask_restful',
-            'gunicorn',
-            'pytest',
-            'pytest-cov',
-            'testresources',
-        ],
-    },
+    'common': [
+        # Non python dependencies
+        'build-essential',
+        'automake',
+        'autoconf',
+        'libtool',
+        # Python dependencies
+        'python3',
+        'python3-dev',
+        'python-wheel-common',
+    ],
 }
+PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-common.txt'
 
 
-def install_dependencies(dependencies):
-    apt = dependencies.get('apt', [])
-    pip3 = dependencies.get('pip3', [])
-    apt_install_packages(*apt)
-    pip_install_packages(*pip3)
+def install_apt_dependencies(distribution: str):
+    apt_install_packages(*APT_DEPENDENCIES['common'])
+    apt_install_packages(*APT_DEPENDENCIES[distribution])
 
 
 def main(distribution):
     logging.info('Updating package lists')
     apt_update_sources()
 
     # install dependencies
-    install_dependencies(DEPENDENCIES['common'])
-    install_dependencies(DEPENDENCIES[distribution])
+    install_apt_dependencies(distribution)
+    pip_install_packages(*load_requirements_file(PIP_DEPENDENCY_FILE))
 
     # make bin dir
     with suppress(FileExistsError):

fact_extractor/install/unpacker.py

@@ -1,18 +1,22 @@
+import hashlib
 import logging
 import os
 from getpass import getuser
 from pathlib import Path
+from shlex import split
+from subprocess import CalledProcessError, run
 from tempfile import TemporaryDirectory
 
 from common_helper_process import execute_shell_command_get_return_code
 
 from helperFunctions.install import (
-    apt_install_packages,
-    install_github_project,
     InstallationError,
     OperateInDirectory,
-    pip_install_packages,
+    apt_install_packages,
     apt_remove_packages,
+    install_github_project,
+    pip_install_packages,
+    load_requirements_file,
 )
 
 BIN_DIR = Path(__file__).parent.parent / 'bin'
@@ -102,31 +106,32 @@
             'liblzo2-dev',
             'xvfb',
             'libcapstone-dev',
-            # patool and unpacking backends
-            'lrzip',
+            # patool
+            'arj',
+            'cabextract',
             'cpio',
-            'unadf',
-            'rpm2cpio',
-            'lzop',
+            'flac',
+            'gzip',
             'lhasa',
-            'cabextract',
-            'zpaq',
             'libchm-dev',
-            'arj',
-            'xdms',
-            'rzip',
+            'liblz4-tool',
+            'lrzip',
             'lzip',
-            'unalz',
-            'unrar',
-            'gzip',
+            'lzop',
+            'ncompress',
             'nomarch',
-            'flac',
-            'unace',
+            'p7zip-full',
+            'rpm2cpio',
+            'rzip',
             'sharutils',
+            'unace',
+            'unadf',
+            'unalz',
             'unar',
+            'unrar',
+            'xdms',
+            'zpaq',
             'zstd',
-            'liblz4-tool',
-            'p7zip-full',
             # Freetz
             'autoconf',
             'automake',
@@ -153,51 +158,18 @@
             # 7z
             'yasm',
         ],
-        'pip3': [
-            'pluginbase',
-            'git+https://github.com/armbues/python-entropy',  # To be checked. Original dependency was deleted.
-            'git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git',
-            'git+https://github.com/fkie-cad/fact_helper_file.git',
-            'git+https://github.com/wummel/patool.git',
-            'archmage',
-            # jefferson + deps
-            'git+https://github.com/sviehb/jefferson.git',
-            'cstruct==2.1',
-            'python-lzo',
-            'git+https://github.com/jrspruitt/[email protected]',  # pinned as broken currently
-            # dji / dlink_shrs
-            'pycryptodome',
-            # hp / raw
-            'git+https://github.com/fkie-cad/common_helper_extraction.git',
-            # intel_hex
-            'intelhex',
-            # linuxkernel
-            'lz4',
-            'git+https://github.com/marin-m/vmlinux-to-elf',
-            # mikrotik
-            'npkPy',
-            # sevenz
-            'git+https://github.com/fkie-cad/common_helper_passwords.git',
-            # srec
-            'bincopy',
-            # uboot
-            'extract-dtb',
-            # uefi
-            'git+https://github.com/theopolis/[email protected]',
-            # unblob
-            'unblob',
-        ],
         'github': [
             (
                 'rampageX/firmware-mod-kit',
                 [
                     '(cd src && make untrx && make -C tpl-tool/src && make -C yaffs2utils)',
-                    'cp src/untrx src/yaffs2utils/unyaffs2 src/tpl-tool/src/tpl-tool ../../bin/'
+                    'cp src/untrx src/yaffs2utils/unyaffs2 src/tpl-tool/src/tpl-tool ../../bin/',
                 ],
             ),
         ],
     },
 }
+PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-unpackers.txt'
 
 
 def check_mod_kit_installed() -> bool:
@@ -209,10 +181,9 @@ def check_mod_kit_installed() -> bool:
 
 def install_dependencies(dependencies):
     apt = dependencies.get('apt', [])
-    pip3 = dependencies.get('pip3', [])
     github = dependencies.get('github', [])
     apt_install_packages(*apt)
-    pip_install_packages(*pip3)
+    pip_install_packages(*load_requirements_file(PIP_DEPENDENCY_FILE))
     for repo in github:
         if repo[0].endswith('firmware-mod-kit') and check_mod_kit_installed():
             logging.info('Skipping firmware-mod-kit since it is already installed')
@@ -236,6 +207,7 @@ def main(distribution):
 
     # install plug-in dependencies
     _install_plugins()
+    _install_patool_deps()
 
     # configure environment
     _edit_sudoers()
@@ -270,6 +242,25 @@ def _edit_sudoers():
         raise InstallationError('Editing sudoers file did not succeed\n{chown_output}\n{mv_output}')
 
 
+def _install_patool_deps():
+    '''install additional dependencies of patool'''
+    with TemporaryDirectory(prefix='patool') as build_directory:
+        with OperateInDirectory(build_directory):
+            # install zoo unpacker
+            file_name = 'zoo_2.10-28_amd64.deb'
+            try:
+                run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), capture_output=True, check=True)
+                expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a'
+                assert _sha256_hash_file(Path(file_name)) == expected_sha
+                run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True)
+            except (AssertionError, CalledProcessError) as error:
+                raise InstallationError('Error during zoo unpacker installation') from error
+
+
+def _sha256_hash_file(file_path: Path) -> str:
+    return hashlib.sha256(file_path.read_bytes()).hexdigest()
+
+
 def _install_freetz():
     if all(
         (Path(__file__).parent.parent / 'bin' / tool).exists()