diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index fe85e5f6..9304ddba 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -1,6 +1,7 @@ import hashlib import logging import os +import platform from getpass import getuser from pathlib import Path from shlex import split @@ -156,7 +157,6 @@ 'yasm', ], 'github': [ - ('threadexio/sasquatch', ['./build.sh']), ( 'rampageX/firmware-mod-kit', [ @@ -168,7 +168,36 @@ }, } PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-unpackers.txt' - +if platform.machine() == 'x86_64': + EXTERNAL_DEB_DEPS = [ + # zoo + ( + 'zoo_2.10-28_amd64.deb', + 'http://launchpadlibrarian.net/230277773', + '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a', + ), + # sasquatch + ( + 'sasquatch_1.0_amd64.deb', + 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', + 'bb211daf90069a43b7d5e76f136766a8542a5328015773e9b8be87541b307b60', + ), + ] +elif platform.machine() == 'aarch64': + EXTERNAL_DEB_DEPS = [ + # zoo + ( + 'zoo_2.10-28_arm64.deb', + 'http://ports.ubuntu.com/pool/universe/z/zoo', + 'e6600d4e878eddd18d1353664fae9bee015a8f9206aa62d2c9bfa070fe4cb7b3', + ), + # sasquatch + ( + 'sasquatch_1.0_arm64.deb', + 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', + 'fb281906a25667414e8b6aff96b49ceb227519122a7844bbc8166f2b6a59554a', + ), + ] def install_dependencies(dependencies): apt = dependencies.get('apt', []) @@ -191,11 +220,12 @@ def main(distribution): install_dependencies(DEPENDENCIES[distribution]) # installing freetz - _install_freetz() + if platform.machine() == 'x86_64': + _install_freetz() # install plug-in dependencies _install_plugins() - _install_patool_deps() + _install_external_deb_deps() # configure environment _edit_sudoers() @@ -215,7 +245,7 @@ def _edit_sudoers(): '/bin/mount', '/bin/umount', '/bin/mknod', - '/usr/local/bin/sasquatch', + '/usr/bin/sasquatch', '/bin/rm', '/bin/cp', '/bin/dd', @@ -230,19 +260,20 @@ def _edit_sudoers(): raise InstallationError('Editing sudoers file did not succeed\n{chown_output}\n{mv_output}') -def _install_patool_deps(): - '''install additional dependencies of patool''' +def _install_external_deb_deps(): + ''' + install deb packages that aren't available through Debian/Ubuntu package sources + ''' with TemporaryDirectory(prefix='patool') as build_directory: with OperateInDirectory(build_directory): - # install zoo unpacker - file_name = 'zoo_2.10-28_amd64.deb' - try: - run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), capture_output=True, check=True) - expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a' - assert _sha256_hash_file(Path(file_name)) == expected_sha - run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) - except (AssertionError, CalledProcessError) as error: - raise InstallationError('Error during zoo unpacker installation') from error + for file_name, url, sha256 in EXTERNAL_DEB_DEPS: + try: + run(split(f'wget {url}/{file_name}'), check=True, env=os.environ) + if not _sha256_hash_file(Path(file_name)) == sha256: + raise InstallationError(f'Wrong file hash: {file_name}') + run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) + except CalledProcessError as error: + raise InstallationError(f'Error during {file_name} unpacker installation') from error def _sha256_hash_file(file_path: Path) -> str: @@ -268,7 +299,7 @@ def _install_freetz(): 'sudo su makeuser -c "make -j$(nproc) tools"', f'sudo chmod -R 777 {build_directory}', f'sudo chown -R {current_user} {build_directory}', - 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma ' + 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk ' f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}', 'sudo userdel makeuser', ], diff --git a/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py b/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py index 80653f6d..a11b7eee 100644 --- a/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py +++ b/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py @@ -62,7 +62,7 @@ def remove_false_positive_archives(self) -> str: 'application/zip', 'application/zlib', ]: - self._remove_invalid_archives(file_path, '7zzs l {}', 'ERROR') + self._remove_invalid_archives(file_path, '7z l {}', 'ERROR') if file_path.is_file(): self._remove_trailing_data(file_type, file_path) @@ -116,7 +116,7 @@ def _output_is_empty(output): def _find_trailing_data_index_zip(file_path: Path) -> int | None: '''Archives carved by binwalk often have trailing data at the end. 7z can determine the actual file size.''' - output = execute_shell_command(f'7zzs l {file_path}') + output = execute_shell_command(f'7z l {file_path}') if 'There are data after the end of archive' in output: match = REAL_SIZE_REGEX.search(output) if match: diff --git a/fact_extractor/plugins/unpacking/patool/code/patool.py b/fact_extractor/plugins/unpacking/patool/code/patool.py index cd74564a..961edf81 100644 --- a/fact_extractor/plugins/unpacking/patool/code/patool.py +++ b/fact_extractor/plugins/unpacking/patool/code/patool.py @@ -24,6 +24,7 @@ 'application/x-gzip', 'application/x-lha', 'application/x-lrzip', + 'application/x-lz4', 'application/x-lzh', 'application/x-lzh-compressed', 'application/x-lzip', diff --git a/fact_extractor/plugins/unpacking/patool/test/data/test.lz4 b/fact_extractor/plugins/unpacking/patool/test/data/test.lz4 new file mode 100644 index 00000000..68458a0e Binary files /dev/null and b/fact_extractor/plugins/unpacking/patool/test/data/test.lz4 differ diff --git a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py index 5fb3d12a..5600db7e 100644 --- a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py +++ b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py @@ -48,6 +48,7 @@ def test_archive_extraction(self, in_file, ignore): 'test.gz', 'test.lrz', 'test.lz', + 'test.lz4', 'test.lzo', 'test.rz', 'test.xz', diff --git a/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py b/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py index 6181d813..110f7439 100644 --- a/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py +++ b/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py @@ -34,7 +34,7 @@ ] VERSION = '0.9.0' -UNPACKER_EXECUTABLE = '7zzs' +UNPACKER_EXECUTABLE = '7z' # Empty password must be first in list to correctly detect if archive has no password PW_LIST = [""] diff --git a/fact_extractor/plugins/unpacking/sevenz/install.sh b/fact_extractor/plugins/unpacking/sevenz/install.sh index 9c0d47dd..21f39c89 100755 --- a/fact_extractor/plugins/unpacking/sevenz/install.sh +++ b/fact_extractor/plugins/unpacking/sevenz/install.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -euo pipefail cd "$( dirname "${BASH_SOURCE[0]}" )" @@ -8,14 +8,22 @@ echo " install 7z " echo "------------------------------------" VERSION="2407" -FILE="7z${VERSION}-linux-x64.tar.xz" +ARCH=$(uname -m) +if [[ $ARCH == "x86_64" ]]; then + ARCH_SUFFIX="x64" +elif [[ $ARCH == "aarch64" ]]; then + ARCH_SUFFIX="arm64" +else + echo "unsupported architecture ${ARCH}" + exit 1 +fi +FILE="7z${VERSION}-linux-${ARCH_SUFFIX}.tar.xz" -# install newest version of 7z mkdir -p /tmp/fact_build cd /tmp/fact_build wget "https://www.7-zip.org/a/${FILE}" tar xvf "${FILE}" 7zzs -sudo mv 7zzs /usr/local/bin/ +sudo mv 7zzs /usr/local/bin/7z rm "${FILE}" exit 0 diff --git a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py index 7d141d83..1286b212 100755 --- a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py +++ b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py @@ -7,16 +7,18 @@ from helperFunctions.file_system import get_fact_bin_dir -SASQUATCH = Path('/usr/local/bin/sasquatch') +SASQUATCH = Path('/usr/bin/sasquatch') +SASQUATCH_BE = Path('/usr/bin/sasquatch-v4be') UNSQUASHFS4_AVM_BE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-be' UNSQUASHFS4_AVM_LE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-le' UNSQUASHFS3_MULTI = Path(get_fact_bin_dir()) / 'unsquashfs3-multi' NAME = 'SquashFS' MIME_PATTERNS = ['filesystem/squashfs'] -VERSION = '0.10' +VERSION = '0.11.0' SQUASH_UNPACKER = [ - (SASQUATCH, '-c lzma-adaptive'), + (SASQUATCH, ''), + (SASQUATCH_BE, ''), (UNSQUASHFS4_AVM_BE, '-scan'), (UNSQUASHFS4_AVM_LE, '-scan'), (UNSQUASHFS3_MULTI, '-scan'), diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 new file mode 100644 index 00000000..434bd929 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 new file mode 100644 index 00000000..604ba49d Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/sqfs.img b/fact_extractor/plugins/unpacking/squashFS/test/data/gzip.sqfs similarity index 95% rename from fact_extractor/plugins/unpacking/squashFS/test/data/sqfs.img rename to fact_extractor/plugins/unpacking/squashFS/test/data/gzip.sqfs index 2e926e56..eecf149e 100644 Binary files a/fact_extractor/plugins/unpacking/squashFS/test/data/sqfs.img and b/fact_extractor/plugins/unpacking/squashFS/test/data/gzip.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lz4.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/lz4.sqfs new file mode 100644 index 00000000..64e05c8f Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lz4.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs new file mode 100644 index 00000000..cb442891 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_be.sqfs3 b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_be.sqfs3 new file mode 100644 index 00000000..b765ba4f Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_be.sqfs3 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_le.sqfs3 b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_le.sqfs3 new file mode 100644 index 00000000..a08047c3 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_le.sqfs3 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 new file mode 100644 index 00000000..9fb2716c Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_le.sqfs2 b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_le.sqfs2 new file mode 100644 index 00000000..2b46f433 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_le.sqfs2 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs new file mode 100644 index 00000000..694a20b9 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs new file mode 100644 index 00000000..a7c82b31 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 b/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 new file mode 100644 index 00000000..eadb98c8 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_le.sqfs3 b/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_le.sqfs3 new file mode 100644 index 00000000..409c28b3 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_le.sqfs3 differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/zstd.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/zstd.sqfs new file mode 100644 index 00000000..e0e330f2 Binary files /dev/null and b/fact_extractor/plugins/unpacking/squashFS/test/data/zstd.sqfs differ diff --git a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py index 75e277dd..111c5cae 100755 --- a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py +++ b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py @@ -1,4 +1,5 @@ from pathlib import Path + import pytest from tempfile import TemporaryDirectory @@ -37,7 +38,23 @@ class TestSquashUnpacker(TestUnpackerBase): def test_unpacker_selection_generic(self): self.check_unpacker_selection('filesystem/squashfs', 'SquashFS') - def test_extraction_sqfs(self): - self.check_unpacking_of_standard_unpack_set( - TEST_DATA_DIR / 'sqfs.img', - ) + @pytest.mark.parametrize(('file', 'expected'), [ + ('avm_be.sqfs4', 'sasquatch-v4be'), + ('avm_le.sqfs4', 'sasquatch'), + ('gzip.sqfs', 'sasquatch'), + ('lz4.sqfs', 'sasquatch'), + ('lzma.sqfs', 'sasquatch'), + ('lzma1_be.sqfs3', 'sasquatch'), + ('lzma1_le.sqfs3', 'sasquatch'), + ('lzma_be.sqfs2', 'unsquashfs4-avm-be'), + ('lzma_le.sqfs2', 'unsquashfs4-avm-be'), + ('lzo.sqfs', 'sasquatch'), + ('xz.sqfs', 'sasquatch'), + ('zlib_be.sqfs3', 'sasquatch'), + ('zlib_le.sqfs3', 'sasquatch'), + ('zstd.sqfs', 'sasquatch'), + ]) + def test_extraction_sqfs(self, file, expected): + meta_data = self.check_unpacking_of_standard_unpack_set(TEST_DATA_DIR / file) + assert meta_data['plugin_used'] == 'SquashFS' + assert meta_data['unpacking_tool'] == expected