Merge remote-tracking branch 'origin/master' into sfx-fix

fact_extractor/install/unpacker.py

@@ -1,6 +1,7 @@
 import hashlib
 import logging
 import os
+import platform
 from getpass import getuser
 from pathlib import Path
 from shlex import split
@@ -156,7 +157,6 @@
             'yasm',
         ],
         'github': [
-            ('threadexio/sasquatch', ['./build.sh']),
             (
                 'rampageX/firmware-mod-kit',
                 [
@@ -168,7 +168,36 @@
     },
 }
 PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-unpackers.txt'
-
+if platform.machine() == 'x86_64':
+    EXTERNAL_DEB_DEPS = [
+        # zoo
+        (
+            'zoo_2.10-28_amd64.deb',
+            'http://launchpadlibrarian.net/230277773',
+            '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a',
+        ),
+        # sasquatch
+        (
+            'sasquatch_1.0_amd64.deb',
+            'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4',
+            'bb211daf90069a43b7d5e76f136766a8542a5328015773e9b8be87541b307b60',
+        ),
+    ]
+elif platform.machine() == 'aarch64':
+    EXTERNAL_DEB_DEPS = [
+        # zoo
+        (
+            'zoo_2.10-28_arm64.deb',
+            'http://ports.ubuntu.com/pool/universe/z/zoo',
+            'e6600d4e878eddd18d1353664fae9bee015a8f9206aa62d2c9bfa070fe4cb7b3',
+        ),
+        # sasquatch
+        (
+            'sasquatch_1.0_arm64.deb',
+            'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4',
+            'fb281906a25667414e8b6aff96b49ceb227519122a7844bbc8166f2b6a59554a',
+        ),
+    ]
 
 def install_dependencies(dependencies):
     apt = dependencies.get('apt', [])
@@ -191,11 +220,12 @@ def main(distribution):
     install_dependencies(DEPENDENCIES[distribution])
 
     # installing freetz
-    _install_freetz()
+    if platform.machine() == 'x86_64':
+        _install_freetz()
 
     # install plug-in dependencies
     _install_plugins()
-    _install_patool_deps()
+    _install_external_deb_deps()
 
     # configure environment
     _edit_sudoers()
@@ -215,7 +245,7 @@ def _edit_sudoers():
                 '/bin/mount',
                 '/bin/umount',
                 '/bin/mknod',
-                '/usr/local/bin/sasquatch',
+                '/usr/bin/sasquatch',
                 '/bin/rm',
                 '/bin/cp',
                 '/bin/dd',
@@ -230,19 +260,20 @@ def _edit_sudoers():
         raise InstallationError('Editing sudoers file did not succeed\n{chown_output}\n{mv_output}')
 
 
-def _install_patool_deps():
-    '''install additional dependencies of patool'''
+def _install_external_deb_deps():
+    '''
+    install deb packages that aren't available through Debian/Ubuntu package sources
+    '''
     with TemporaryDirectory(prefix='patool') as build_directory:
         with OperateInDirectory(build_directory):
-            # install zoo unpacker
-            file_name = 'zoo_2.10-28_amd64.deb'
-            try:
-                run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), capture_output=True, check=True)
-                expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a'
-                assert _sha256_hash_file(Path(file_name)) == expected_sha
-                run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True)
-            except (AssertionError, CalledProcessError) as error:
-                raise InstallationError('Error during zoo unpacker installation') from error
+            for file_name, url, sha256 in EXTERNAL_DEB_DEPS:
+                try:
+                    run(split(f'wget {url}/{file_name}'), check=True, env=os.environ)
+                    if not _sha256_hash_file(Path(file_name)) == sha256:
+                        raise InstallationError(f'Wrong file hash: {file_name}')
+                    run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True)
+                except CalledProcessError as error:
+                    raise InstallationError(f'Error during {file_name} unpacker installation') from error
 
 
 def _sha256_hash_file(file_path: Path) -> str:
@@ -268,7 +299,7 @@ def _install_freetz():
                     'sudo su makeuser -c "make -j$(nproc) tools"',
                     f'sudo chmod -R 777 {build_directory}',
                     f'sudo chown -R {current_user} {build_directory}',
-                    'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma '
+                    'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk '
                     f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}',
                     'sudo userdel makeuser',
                 ],

fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py

@@ -62,7 +62,7 @@ def remove_false_positive_archives(self) -> str:
                 'application/zip',
                 'application/zlib',
             ]:
-                self._remove_invalid_archives(file_path, '7zzs l {}', 'ERROR')
+                self._remove_invalid_archives(file_path, '7z l {}', 'ERROR')
 
             if file_path.is_file():
                 self._remove_trailing_data(file_type, file_path)
@@ -116,7 +116,7 @@ def _output_is_empty(output):
 
 def _find_trailing_data_index_zip(file_path: Path) -> int | None:
     '''Archives carved by binwalk often have trailing data at the end. 7z can determine the actual file size.'''
-    output = execute_shell_command(f'7zzs l {file_path}')
+    output = execute_shell_command(f'7z l {file_path}')
     if 'There are data after the end of archive' in output:
         match = REAL_SIZE_REGEX.search(output)
         if match:

fact_extractor/plugins/unpacking/patool/code/patool.py

@@ -24,6 +24,7 @@
     'application/x-gzip',
     'application/x-lha',
     'application/x-lrzip',
+    'application/x-lz4',
     'application/x-lzh',
     'application/x-lzh-compressed',
     'application/x-lzip',

fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py

@@ -48,6 +48,7 @@ def test_archive_extraction(self, in_file, ignore):
             'test.gz',
             'test.lrz',
             'test.lz',
+            'test.lz4',
             'test.lzo',
             'test.rz',
             'test.xz',

fact_extractor/plugins/unpacking/sevenz/code/sevenz.py

@@ -34,7 +34,7 @@
 ]
 VERSION = '0.9.0'
 
-UNPACKER_EXECUTABLE = '7zzs'
+UNPACKER_EXECUTABLE = '7z'
 
 # Empty password must be first in list to correctly detect if archive has no password
 PW_LIST = [""]

fact_extractor/plugins/unpacking/sevenz/install.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -euo pipefail
 
 cd "$( dirname "${BASH_SOURCE[0]}" )"
 
@@ -8,14 +8,22 @@ echo "             install 7z             "
 echo "------------------------------------"
 
 VERSION="2407"
-FILE="7z${VERSION}-linux-x64.tar.xz"
+ARCH=$(uname -m)
+if [[ $ARCH == "x86_64" ]]; then
+    ARCH_SUFFIX="x64"
+elif [[ $ARCH == "aarch64" ]]; then
+    ARCH_SUFFIX="arm64"
+else
+    echo "unsupported architecture ${ARCH}"
+    exit 1
+fi
+FILE="7z${VERSION}-linux-${ARCH_SUFFIX}.tar.xz"
 
-# install newest version of 7z
 mkdir -p /tmp/fact_build
 cd /tmp/fact_build
 wget "https://www.7-zip.org/a/${FILE}"
 tar xvf "${FILE}" 7zzs
-sudo mv 7zzs /usr/local/bin/
+sudo mv 7zzs /usr/local/bin/7z
 rm "${FILE}"
 
 exit 0

fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py

@@ -7,16 +7,18 @@
 
 from helperFunctions.file_system import get_fact_bin_dir
 
-SASQUATCH = Path('/usr/local/bin/sasquatch')
+SASQUATCH = Path('/usr/bin/sasquatch')
+SASQUATCH_BE = Path('/usr/bin/sasquatch-v4be')
 UNSQUASHFS4_AVM_BE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-be'
 UNSQUASHFS4_AVM_LE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-le'
 UNSQUASHFS3_MULTI = Path(get_fact_bin_dir()) / 'unsquashfs3-multi'
 
 NAME = 'SquashFS'
 MIME_PATTERNS = ['filesystem/squashfs']
-VERSION = '0.10'
+VERSION = '0.11.0'
 SQUASH_UNPACKER = [
-    (SASQUATCH, '-c lzma-adaptive'),
+    (SASQUATCH, ''),
+    (SASQUATCH_BE, ''),
     (UNSQUASHFS4_AVM_BE, '-scan'),
     (UNSQUASHFS4_AVM_LE, '-scan'),
     (UNSQUASHFS3_MULTI, '-scan'),

fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+
 import pytest
 from tempfile import TemporaryDirectory
 
@@ -37,7 +38,23 @@ class TestSquashUnpacker(TestUnpackerBase):
     def test_unpacker_selection_generic(self):
         self.check_unpacker_selection('filesystem/squashfs', 'SquashFS')
 
-    def test_extraction_sqfs(self):
-        self.check_unpacking_of_standard_unpack_set(
-            TEST_DATA_DIR / 'sqfs.img',
-        )
+    @pytest.mark.parametrize(('file', 'expected'), [
+        ('avm_be.sqfs4', 'sasquatch-v4be'),
+        ('avm_le.sqfs4', 'sasquatch'),
+        ('gzip.sqfs', 'sasquatch'),
+        ('lz4.sqfs', 'sasquatch'),
+        ('lzma.sqfs', 'sasquatch'),
+        ('lzma1_be.sqfs3', 'sasquatch'),
+        ('lzma1_le.sqfs3', 'sasquatch'),
+        ('lzma_be.sqfs2', 'unsquashfs4-avm-be'),
+        ('lzma_le.sqfs2', 'unsquashfs4-avm-be'),
+        ('lzo.sqfs', 'sasquatch'),
+        ('xz.sqfs', 'sasquatch'),
+        ('zlib_be.sqfs3', 'sasquatch'),
+        ('zlib_le.sqfs3', 'sasquatch'),
+        ('zstd.sqfs', 'sasquatch'),
+    ])
+    def test_extraction_sqfs(self, file, expected):
+        meta_data = self.check_unpacking_of_standard_unpack_set(TEST_DATA_DIR / file)
+        assert meta_data['plugin_used'] == 'SquashFS'
+        assert meta_data['unpacking_tool'] == expected