From 05844cd6c0578b9e5a6dbab3b40960ce58903541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 2 Jul 2024 15:07:34 +0200 Subject: [PATCH 01/12] patool unpacker: added lz4 MIME type which would have been already supported by patool and the lz4 tool was also already installed --- .../plugins/unpacking/patool/code/patool.py | 1 + .../plugins/unpacking/patool/test/data/test.lz4 | Bin 0 -> 147 bytes .../unpacking/patool/test/test_plugin_patool.py | 1 + 3 files changed, 2 insertions(+) create mode 100644 fact_extractor/plugins/unpacking/patool/test/data/test.lz4 diff --git a/fact_extractor/plugins/unpacking/patool/code/patool.py b/fact_extractor/plugins/unpacking/patool/code/patool.py index cd74564a..961edf81 100644 --- a/fact_extractor/plugins/unpacking/patool/code/patool.py +++ b/fact_extractor/plugins/unpacking/patool/code/patool.py @@ -24,6 +24,7 @@ 'application/x-gzip', 'application/x-lha', 'application/x-lrzip', + 'application/x-lz4', 'application/x-lzh', 'application/x-lzh-compressed', 'application/x-lzip', diff --git a/fact_extractor/plugins/unpacking/patool/test/data/test.lz4 b/fact_extractor/plugins/unpacking/patool/test/data/test.lz4 new file mode 100644 index 0000000000000000000000000000000000000000..68458a0ec59229b436744708bcf0b87bf24f419d GIT binary patch literal 147 zcmWN^IT1oZ3;@tyAc7hN$>!KihXF+7+5#1S1__8j1Mc8;?{a%w>;5^9<1FF`Ry9xU z5hww34_@YD+Km`bfOPjbNIT=k@2Vx8T7p>kZlWdxp_X>9DkzkS% R;nWc6v^Ip`@%MbczCV!9C~W`$ literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py index 5fb3d12a..5600db7e 100644 --- a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py +++ b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py @@ -48,6 +48,7 @@ def test_archive_extraction(self, in_file, ignore): 'test.gz', 'test.lrz', 'test.lz', + 'test.lz4', 'test.lzo', 'test.rz', 'test.xz', From d5b80f0dd699f85051746b65765b21e273b7f640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 17 Jul 2024 17:57:13 +0200 Subject: [PATCH 02/12] unpacker installation: fixed bug in zoo installation --- fact_extractor/install/unpacker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index fe85e5f6..4abf2b71 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -237,7 +237,7 @@ def _install_patool_deps(): # install zoo unpacker file_name = 'zoo_2.10-28_amd64.deb' try: - run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), capture_output=True, check=True) + run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), check=True, env=os.environ) expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a' assert _sha256_hash_file(Path(file_name)) == expected_sha run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) From 7eb129f51bfbf72b4755d83fbe4a66d66485b798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 17 Jul 2024 17:57:34 +0200 Subject: [PATCH 03/12] unpacker installation: fixed bug in freetz installation --- fact_extractor/install/unpacker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index 4abf2b71..847b23d0 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -268,7 +268,7 @@ def _install_freetz(): 'sudo su makeuser -c "make -j$(nproc) tools"', f'sudo chmod -R 777 {build_directory}', f'sudo chown -R {current_user} {build_directory}', - 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma ' + 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk ' f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}', 'sudo userdel makeuser', ], From 27e381c3258b5c54dfefd7c46bb5c886f1673860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 23 Jul 2024 13:19:05 +0200 Subject: [PATCH 04/12] squashfs unpacker: switch to newer sasquatch fork --- fact_extractor/install/unpacker.py | 41 ++++++++++++------- .../unpacking/squashFS/code/squash_fs.py | 8 ++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index 847b23d0..449a3730 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -156,7 +156,6 @@ 'yasm', ], 'github': [ - ('threadexio/sasquatch', ['./build.sh']), ( 'rampageX/firmware-mod-kit', [ @@ -168,6 +167,20 @@ }, } PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-unpackers.txt' +EXTERNAL_DEB_DEPS = [ + # zoo + ( + 'zoo_2.10-28_amd64.deb', + 'http://launchpadlibrarian.net/230277773', + '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a', + ), + # sasquatch + ( + 'sasquatch_1.0_amd64.deb', + 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', + 'bb211daf90069a43b7d5e76f136766a8542a5328015773e9b8be87541b307b60', + ), +] def install_dependencies(dependencies): @@ -195,7 +208,7 @@ def main(distribution): # install plug-in dependencies _install_plugins() - _install_patool_deps() + _install_external_deb_deps() # configure environment _edit_sudoers() @@ -215,7 +228,7 @@ def _edit_sudoers(): '/bin/mount', '/bin/umount', '/bin/mknod', - '/usr/local/bin/sasquatch', + '/usr/bin/sasquatch', '/bin/rm', '/bin/cp', '/bin/dd', @@ -230,19 +243,19 @@ def _edit_sudoers(): raise InstallationError('Editing sudoers file did not succeed\n{chown_output}\n{mv_output}') -def _install_patool_deps(): - '''install additional dependencies of patool''' +def _install_external_deb_deps(): + ''' + install deb packages that aren't available through Debian/Ubuntu package sources + ''' with TemporaryDirectory(prefix='patool') as build_directory: with OperateInDirectory(build_directory): - # install zoo unpacker - file_name = 'zoo_2.10-28_amd64.deb' - try: - run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), check=True, env=os.environ) - expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a' - assert _sha256_hash_file(Path(file_name)) == expected_sha - run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) - except (AssertionError, CalledProcessError) as error: - raise InstallationError('Error during zoo unpacker installation') from error + for file_name, url, sha256 in EXTERNAL_DEB_DEPS: + try: + run(split(f'wget {url}/{file_name}'), check=True, env=os.environ) + assert _sha256_hash_file(Path(file_name)) == sha256 + run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) + except (AssertionError, CalledProcessError) as error: + raise InstallationError(f'Error during {file_name} unpacker installation') from error def _sha256_hash_file(file_path: Path) -> str: diff --git a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py index 7d141d83..1286b212 100755 --- a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py +++ b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py @@ -7,16 +7,18 @@ from helperFunctions.file_system import get_fact_bin_dir -SASQUATCH = Path('/usr/local/bin/sasquatch') +SASQUATCH = Path('/usr/bin/sasquatch') +SASQUATCH_BE = Path('/usr/bin/sasquatch-v4be') UNSQUASHFS4_AVM_BE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-be' UNSQUASHFS4_AVM_LE = Path(get_fact_bin_dir()) / 'unsquashfs4-avm-le' UNSQUASHFS3_MULTI = Path(get_fact_bin_dir()) / 'unsquashfs3-multi' NAME = 'SquashFS' MIME_PATTERNS = ['filesystem/squashfs'] -VERSION = '0.10' +VERSION = '0.11.0' SQUASH_UNPACKER = [ - (SASQUATCH, '-c lzma-adaptive'), + (SASQUATCH, ''), + (SASQUATCH_BE, ''), (UNSQUASHFS4_AVM_BE, '-scan'), (UNSQUASHFS4_AVM_LE, '-scan'), (UNSQUASHFS3_MULTI, '-scan'), From 9932b5d3a05cffe16e6c483466a418c7bedbb01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 23 Jul 2024 13:20:25 +0200 Subject: [PATCH 05/12] squashfs unpacker: add more test cases --- .../unpacking/squashFS/test/data/avm_be.sqfs4 | Bin 0 -> 4096 bytes .../unpacking/squashFS/test/data/avm_le.sqfs4 | Bin 0 -> 4096 bytes .../test/data/{sqfs.img => gzip.sqfs} | Bin 4096 -> 4096 bytes .../unpacking/squashFS/test/data/lz4.sqfs | Bin 0 -> 4096 bytes .../unpacking/squashFS/test/data/lzma.sqfs | Bin 0 -> 4096 bytes .../squashFS/test/data/lzma1_be.sqfs3 | Bin 0 -> 4096 bytes .../squashFS/test/data/lzma1_le.sqfs3 | Bin 0 -> 4096 bytes .../squashFS/test/data/lzma_be.sqfs2 | Bin 0 -> 4096 bytes .../squashFS/test/data/lzma_le.sqfs2 | Bin 0 -> 4096 bytes .../unpacking/squashFS/test/data/lzo.sqfs | Bin 0 -> 4096 bytes .../unpacking/squashFS/test/data/xz.sqfs | Bin 0 -> 4096 bytes .../squashFS/test/data/zlib_be.sqfs3 | Bin 0 -> 4096 bytes .../squashFS/test/data/zlib_le.sqfs3 | Bin 0 -> 4096 bytes .../unpacking/squashFS/test/data/zstd.sqfs | Bin 0 -> 4096 bytes .../squashFS/test/test_plugin_squashfs.py | 25 +++++++++++++++--- 15 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 rename fact_extractor/plugins/unpacking/squashFS/test/data/{sqfs.img => gzip.sqfs} (95%) create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lz4.sqfs create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_be.sqfs3 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzma1_le.sqfs3 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzma_le.sqfs2 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/zlib_le.sqfs3 create mode 100644 fact_extractor/plugins/unpacking/squashFS/test/data/zstd.sqfs diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_be.sqfs4 new file mode 100644 index 0000000000000000000000000000000000000000..434bd929b0a814032493e52d72fcc5dfc91e424f GIT binary patch literal 4096 zcmXReEY4tHU|?lnU=mne={#VV&tMq~gn(*;nLwLANY=3pZn0rL0ixCeP{7K*=cso zFLuw)y?bLvZQAq)_fCcP#WU^<5MbWMD)S8NB{TQ)P7I8p3qV1YwY4|g&VaWYYyo3r zR7L|>34{O(GB7dl!+on@9IsbWQ38~P$bmvy0mx%y;7w1>OD)PwR!GawNl7hYZ~&@c zV&DMj1O=EOLjnUcvVf6515h6bBmmu!2VsIp#%&r6P@V#mhEWbsMgo*>fY6K=Sb!|% v7Z4s3%w-VnC^;GeqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?Y(fA4z7%?y literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 b/fact_extractor/plugins/unpacking/squashFS/test/data/avm_le.sqfs4 new file mode 100644 index 0000000000000000000000000000000000000000..604ba49d52250d77d94af03331cfacb1a1669827 GIT binary patch literal 4096 zcmc~OE-YqcU|`^30ul_2K*+$r!XUtKfSD0UGcYs&F$h59fpTDV3=IFF;2cDn!2_fb z2#!GMZ%{fUBUPa!BeN()p(M4qL?JCRCsh&7vf@h4&nrpIE3s1WtqcLm2bJceDj4b+ z=^1l{r4|)u=I2={1Qg|`7bWH@80di^S4c)?u>ug408K7V1sZ^4bOpm-^Qwp_phq&5 z?wK(ka9F^o$Ov}v1%8IZ4;bb%Si}OoD6;jX+QO_8OKY`82Rqa!|p*(_vF-h)$T z6n<1VIJfTUJF5Sr?ydjkvb^<{H+{BxB+eB-tIzOY%8~{K#*hUdubof3ILFR_w;K{{ zkx>~9KtV8o1`q=i13v@YrwYdLdLssG`wJD tVE6~+K_y42(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S``80su&mdyN19 literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/sqfs.img b/fact_extractor/plugins/unpacking/squashFS/test/data/gzip.sqfs similarity index 95% rename from fact_extractor/plugins/unpacking/squashFS/test/data/sqfs.img rename to fact_extractor/plugins/unpacking/squashFS/test/data/gzip.sqfs index 2e926e56562a94c5686ee520159a04bbb4bce2ba..eecf149e4c71ac6b38229bc7a71fda1cee672be4 100644 GIT binary patch delta 52 zcmV-40L%Y?Ab=nYXmfFM1poj5gKwW^kqo#JSi;Bv1iAV1)0h}gfd~*YGQ5%2Dg@S| K4!*Ml0sId0@)RNf delta 52 zcmV-40L%Y?Ab=nYXmfFM1poj5CgdS6opSFX-$h(YSm4Pj3Ov5N`FAGh;$>m5!zDRxG>UA+mtrJBn91N>m#@l z!IyBSE0-$x5N-quy4IUIh`z&ICY+mlm@_k9&p-8J0G{2e4#1#MA43{X)pSpD{M&{8H+zR5O$fK4ZaC8eA{&@yym%G?_|ged4~5?ve~nJPAi~HU(o(Pp%$JPZYJjr=~*m+ zDTdKJ-BiOHhn7Hr`gPRTNMh57SCQ72z!<-7Ak%h)^)F3}+0kwz>_7U!36>%EY zk)f;wX5vGi#qVo)`8Xe(1v%8C!?}sEIfJfnglD&S#~rkVSEVEdvjS{kw`f66qZWZE zrLfYc%&<>8qq>mX7O_EBMy5K8jAbGWCoXU;eItC;fqVHUO(lweBA^H;0*Zhlpa>`e Pihv@Z2q*%Iz<&bYle1Y% literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma.sqfs new file mode 100644 index 0000000000000000000000000000000000000000..cb442891206a7fc2d3980e329529f2ace4d43a0a GIT binary patch literal 4096 zcmc~OE-YqcU|?90KR=Cufr)_;$Yx>?WHjc(Knwy$K@vc)7fSz!g0m1|25G2x z3zS|DrDH+HEZ`zu^p7u-Mb$K9J| z+GPc=S9I#nm{zp?l9W$SN(I;Pl`8A5Gn&Xw-Bv$Adt+6?8{fUg(+?_s%HQLg+OsS^ zI=TM;wJQz`0u2dJ6LX<-5j3n6Ar@-DEEHfcYTMF-wIT)su~DH}tW--Q4F%I`NY z*f9vK_{0Bs;}O-2J&(-}FZ)^3sAJ?kTbDsEbJpIwZ9x~T4_h^P9k|qC_R!Wv`{lF; z3T6|v&u=yssQ>Tb0(7MZ14CB5=wEy1)y^7cq6E$e_~mmbPCduh&v53z&h4M=9gjb$ zODJ2~-aqr}>01|_#FZ=MZ!Y}AwU@_K*q}IwE%-$D?En8Cohe{w5P?QZGikkf&ZCTR)6`h zthj!E`xy(jPx=fKr>%LuLOvqS@2NEFVvB^GrrH1hf7)HbAka__4U7aRodgX8QBdF- z@;C4<{uHp0rQ|`R(EmT~2@Ew*=@-mU^`q2i2#kinXb6mkz-S1JhQMeDjE2By2#kin OXb6mkz-S0i8Ug?YIE4iO literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 b/fact_extractor/plugins/unpacking/squashFS/test/data/lzma_be.sqfs2 new file mode 100644 index 0000000000000000000000000000000000000000..9fb2716cd5dc6811ca567a81774afbc39d1eb70a GIT binary patch literal 4096 zcmXReEY4tHU|?lnVAKR+RUjJ#<^%CH1||j{AT8j)$dER_>jFp|2*Mc{!8DL$RAkVS z;#kCN`!vu=j_-(%M(vBE6Oyxk#C*5yoaVM(sBFRsi_nrIDUaB)rDY`)A`)9GS*I+^ zh_2OGHLv2HNf7tTwy1@-c6nXo(nvpd?9I36z?+B5w`e|PV+iw`@c&=={RRe827wiS z_&;wvqMEViu^TIoW6il=PWvw(vdxVW(cAg0YG>2c-+_AO@2@lDoQm7GgQLFw$Fds? z4E78R8UOC76<_1Lc|YZSi+>_#_4SkMP6VBN-#YK%|LhvSi~C;}eCvLbc_wu3?i=z{gvwy^Vx9yzfwqB@g!U>Dek|QaP*s`T%B@`kOTPsfSx9Gr|hs(EUK4oJF^PBMhU-|t869xuFwi^Bq zKI!h0g6$oR<8l%$mc?k48C1Sm;_Y|s?eTvOO$%AY&AahQMeDjE2By2#kinXb6mkz-S1JhQMeD PjE2By2#kin5DozV>z{2r literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/lzo.sqfs new file mode 100644 index 0000000000000000000000000000000000000000..694a20b9185e806dd574330c7b63c1f94294756c GIT binary patch literal 4096 zcmeIuK}!Nb6bJDC?5?`3q@>cx!vuy&DXBLPVV9yv3K0(xXy~X5x=Xto6to~wm*~*3 zLr{l4N7%iOp!Y6*gQ8|*=o56D2g7?a4<2uR1NYPo0eH2adVm7LYZZ%_@Ji$Ln4cgX zf(DO!k-tSueqVu@c<+gPCGtvRz^FGcoi6p5>(QP$WXstMq7|cKTOPB#3awo6>^lR2(wTNQff*nQ_@L^ldIFd`|_rMvL9qb{rJxdB-O_OYbkg`f&`#J`|m zogj6dL{A%ksKBkb309e;fE17dQa}nw0VyB_q<|EV0#ZN<{ENU3 Dgf~;s literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs b/fact_extractor/plugins/unpacking/squashFS/test/data/xz.sqfs new file mode 100644 index 0000000000000000000000000000000000000000..a7c82b31600205aeac8b1ce9263eba9be74c7a64 GIT binary patch literal 4096 zcmc~OE-YqcU|@KjKR=Cufr)_;$Yx;>WHjc(KnwyrOc0s{O8G`TnxXaJJY6%2pPt0JNp7#K5^ z?wK(ka9F^o$Ov}v1%8IZ4;bb%Si}N7CbIRV+QO_8OKY`82Rqa!|p*(_vF-h)$T z6n<1VIJfTUJF5Sr?ydjkvb^<{H+{BxB+eB-tIzOY%8~{K#*hUdubof3ILFR_w;OCY zV`NlD15gkQpaBF7OMV8pPZf;g^-3yAU~(W66o8nKftMjYH7~U&Gg%=mKPM%%$N{L2 ziGc&G6BJm62@K3A0!9K22|%47kOwjV2sVL91_q4=1rQerI6&D6P`Uw1pMmONX?VfR t!0->sgG!E4qaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8Un*G1OWApec1p2 literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 b/fact_extractor/plugins/unpacking/squashFS/test/data/zlib_be.sqfs3 new file mode 100644 index 0000000000000000000000000000000000000000..eadb98c85edc8f4c82a357ce9f4220e7241280b5 GIT binary patch literal 4096 zcmXReEY4tHU|*2ow}Jz{rp`zuN^WUjr6ofRbQA#)S|$#<@^6FzPB)Tmnj$ zLg}d$x42Jwgle2Tq2m>(;i;<^=yCdtXK2gjH9IDK{_GX>*=vnRjka^cBFCkcE4kXL z+^UYnB(8KlI`d~`7#E}K&zG*9JGt1Z5>ALtni#o;!9~?!a(8een}Q@SgW7IYJ_g5% zTfs4hKG^5h)#))aGaECj2?*A3Pfuq_QezWiogp(>$w{Akmz}~k=Sk_VllhsFBvm%g zJn(I^`U%IUHyQemiMTR&R@_QXXkbWVQ>$ZZn{?%oQ0ta6b3#}jSL~hCC8V}OM_E}} zpV_%8D=sTAZIe?}ou5~h;A3SzwzLUs6RMom6#7|@88b2LnIn?eAOH=Hav&WCWdkY3 zWCk&iB@E2097h8VH*n=FDB)ocy2EG+QNvivz`*g z45ce>ai8=E)i`-V$1707Q&%t0c1-&G*(>O?*BX%;ZRdtXj!P|9a(!qi>kxq?%+l?1xa28wcVd ziYF!gm_OUzzIP_0a0kOjSIZ6?9u9yos>U{X}fq6yPxPF|HDvpzg1@DR(S1ePRL z);PAdFIVjc(Knwz#KoUT(3QGTnf&*Yt1_pkp zcrlco1En>#>i<>9+snid)G8#=pq;~C+CVH?fJNWE^Nxwpikp`2!CEsfM{-(3{z6%I4S8Qqx-ygQ*>aS8JW`;cp4|lHK zs!{IC0JdR)DiZ?>6BEM{MuuFVPnZ})7#ZF&s{Gmx-+N%J;lHvuppo)BF8}e z9bIqM}{5ISSAL80LZ3m<0y#JCJwCI%H|1}1)nlGNf7g|y6^R0ZRB zy^@L&Mg?Am^whl6qReE4wEUcu)FKC9s4+2c7$z_<8*wn0^xtFHy^HPb&W4|!Z=zM0 z!vz`=pg{zq6QMyP53x&QD+5D<0)s-T(jq literal 0 HcmV?d00001 diff --git a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py index 75e277dd..111c5cae 100755 --- a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py +++ b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py @@ -1,4 +1,5 @@ from pathlib import Path + import pytest from tempfile import TemporaryDirectory @@ -37,7 +38,23 @@ class TestSquashUnpacker(TestUnpackerBase): def test_unpacker_selection_generic(self): self.check_unpacker_selection('filesystem/squashfs', 'SquashFS') - def test_extraction_sqfs(self): - self.check_unpacking_of_standard_unpack_set( - TEST_DATA_DIR / 'sqfs.img', - ) + @pytest.mark.parametrize(('file', 'expected'), [ + ('avm_be.sqfs4', 'sasquatch-v4be'), + ('avm_le.sqfs4', 'sasquatch'), + ('gzip.sqfs', 'sasquatch'), + ('lz4.sqfs', 'sasquatch'), + ('lzma.sqfs', 'sasquatch'), + ('lzma1_be.sqfs3', 'sasquatch'), + ('lzma1_le.sqfs3', 'sasquatch'), + ('lzma_be.sqfs2', 'unsquashfs4-avm-be'), + ('lzma_le.sqfs2', 'unsquashfs4-avm-be'), + ('lzo.sqfs', 'sasquatch'), + ('xz.sqfs', 'sasquatch'), + ('zlib_be.sqfs3', 'sasquatch'), + ('zlib_le.sqfs3', 'sasquatch'), + ('zstd.sqfs', 'sasquatch'), + ]) + def test_extraction_sqfs(self, file, expected): + meta_data = self.check_unpacking_of_standard_unpack_set(TEST_DATA_DIR / file) + assert meta_data['plugin_used'] == 'SquashFS' + assert meta_data['unpacking_tool'] == expected From 2c91286c6d38bc944c8867e395c5017b4b87ded2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 23 Jul 2024 15:19:05 +0200 Subject: [PATCH 06/12] Freetz zoo bugfixes (#140) * unpacker installation: fixed bugs in zoo and freetz installation --- fact_extractor/install/unpacker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index fe85e5f6..847b23d0 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -237,7 +237,7 @@ def _install_patool_deps(): # install zoo unpacker file_name = 'zoo_2.10-28_amd64.deb' try: - run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), capture_output=True, check=True) + run(split(f'wget http://launchpadlibrarian.net/230277773/{file_name}'), check=True, env=os.environ) expected_sha = '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a' assert _sha256_hash_file(Path(file_name)) == expected_sha run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) @@ -268,7 +268,7 @@ def _install_freetz(): 'sudo su makeuser -c "make -j$(nproc) tools"', f'sudo chmod -R 777 {build_directory}', f'sudo chown -R {current_user} {build_directory}', - 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma ' + 'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk ' f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}', 'sudo userdel makeuser', ], From 5eb039ceb31d4cb85e283747d15d83e60ac15815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 23 Jul 2024 15:49:36 +0200 Subject: [PATCH 07/12] ext deb installation: replace assert with if->raise --- fact_extractor/install/unpacker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index 449a3730..a0eee7af 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -252,7 +252,8 @@ def _install_external_deb_deps(): for file_name, url, sha256 in EXTERNAL_DEB_DEPS: try: run(split(f'wget {url}/{file_name}'), check=True, env=os.environ) - assert _sha256_hash_file(Path(file_name)) == sha256 + if not _sha256_hash_file(Path(file_name)) == sha256: + raise InstallationError(f'Wrong file hash: {file_name}') run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) except (AssertionError, CalledProcessError) as error: raise InstallationError(f'Error during {file_name} unpacker installation') from error From 0bd81b4f2c498dfc5ed62cdd1eecc08d27f4a64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 23 Jul 2024 15:54:24 +0200 Subject: [PATCH 08/12] ext deb installation: remove unused exception type in try except block --- fact_extractor/install/unpacker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index a0eee7af..f16f28cf 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -255,7 +255,7 @@ def _install_external_deb_deps(): if not _sha256_hash_file(Path(file_name)) == sha256: raise InstallationError(f'Wrong file hash: {file_name}') run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True) - except (AssertionError, CalledProcessError) as error: + except CalledProcessError as error: raise InstallationError(f'Error during {file_name} unpacker installation') from error From 4efb79967f78399edb44b7a861b7b09b1e2a3b33 Mon Sep 17 00:00:00 2001 From: soxrok2212 Date: Thu, 25 Jul 2024 03:33:57 +0000 Subject: [PATCH 09/12] 7zip: allow building for any cpu --- fact_extractor/plugins/unpacking/sevenz/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fact_extractor/plugins/unpacking/sevenz/install.sh b/fact_extractor/plugins/unpacking/sevenz/install.sh index 736e3ed2..6e7d71c7 100755 --- a/fact_extractor/plugins/unpacking/sevenz/install.sh +++ b/fact_extractor/plugins/unpacking/sevenz/install.sh @@ -22,7 +22,7 @@ cd p7zip* # gcc >= 11 has -Wnarrowing as default flag which leads to an error during compilation # g++ will try to use standard C++17 but the code is not compatible -> use C++14 sed -i 's/CXXFLAGS=-c -I. \\/CXXFLAGS=-c -I. -Wno-narrowing -std=c++14 \\/g' makefile.glb || echo "Warning: Could not apply makefile patch" -cp makefile.linux_amd64_asm makefile.machine +cp makefile.linux_any_cpu makefile.machine make -j"$(nproc)" all3 sudo ./install.sh cd .. From 426387c95006c6be21ab08239a8c1dc37e5e5326 Mon Sep 17 00:00:00 2001 From: soxrok2212 Date: Thu, 25 Jul 2024 04:08:44 +0000 Subject: [PATCH 10/12] zoo/sasquatch: dynamically choose between amd64 and arm64 --- fact_extractor/install/unpacker.py | 46 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index f16f28cf..bf5a15f7 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -1,6 +1,7 @@ import hashlib import logging import os +import platform from getpass import getuser from pathlib import Path from shlex import split @@ -167,21 +168,36 @@ }, } PIP_DEPENDENCY_FILE = Path(__file__).parent.parent.parent / 'requirements-unpackers.txt' -EXTERNAL_DEB_DEPS = [ - # zoo - ( - 'zoo_2.10-28_amd64.deb', - 'http://launchpadlibrarian.net/230277773', - '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a', - ), - # sasquatch - ( - 'sasquatch_1.0_amd64.deb', - 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', - 'bb211daf90069a43b7d5e76f136766a8542a5328015773e9b8be87541b307b60', - ), -] - +if platform.machine() == 'x86_64': + EXTERNAL_DEB_DEPS = [ + # zoo + ( + 'zoo_2.10-28_amd64.deb', + 'http://launchpadlibrarian.net/230277773', + '953f4f94095ef3813dfd30c8977475c834363aaabce15ab85ac5195e52fd816a', + ), + # sasquatch + ( + 'sasquatch_1.0_amd64.deb', + 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', + 'bb211daf90069a43b7d5e76f136766a8542a5328015773e9b8be87541b307b60', + ), + ] +elif platform.machine() == 'aarch64': + EXTERNAL_DEB_DEPS = [ + # zoo + ( + 'zoo_2.10-28_arm64.deb', + 'http://ports.ubuntu.com/pool/universe/z/zoo/', + 'e6600d4e878eddd18d1353664fae9bee015a8f9206aa62d2c9bfa070fe4cb7b3', + ), + # sasquatch + ( + 'sasquatch_1.0_arm64.deb', + 'https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4', + 'fb281906a25667414e8b6aff96b49ceb227519122a7844bbc8166f2b6a59554a', + ), + ] def install_dependencies(dependencies): apt = dependencies.get('apt', []) From 1e2489f8bbbfee627c3cbbf3f72cbde546c4ffd7 Mon Sep 17 00:00:00 2001 From: soxrok2212 Date: Thu, 25 Jul 2024 04:10:17 +0000 Subject: [PATCH 11/12] freetz: only install for x86 --- fact_extractor/install/unpacker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index bf5a15f7..03097373 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -220,7 +220,8 @@ def main(distribution): install_dependencies(DEPENDENCIES[distribution]) # installing freetz - _install_freetz() + if platform.machine() == 'x86_64': + _install_freetz() # install plug-in dependencies _install_plugins() From 84976906ce75aaea328f4d8a8e02e53971b7db43 Mon Sep 17 00:00:00 2001 From: soxrok2212 Date: Thu, 25 Jul 2024 04:14:48 +0000 Subject: [PATCH 12/12] zoo/sasquatch: remove trailing / in url --- fact_extractor/install/unpacker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index 03097373..9304ddba 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -188,7 +188,7 @@ # zoo ( 'zoo_2.10-28_arm64.deb', - 'http://ports.ubuntu.com/pool/universe/z/zoo/', + 'http://ports.ubuntu.com/pool/universe/z/zoo', 'e6600d4e878eddd18d1353664fae9bee015a8f9206aa62d2c9bfa070fe4cb7b3', ), # sasquatch