diff --git a/Packs/CommonScripts/.pack-ignore b/Packs/CommonScripts/.pack-ignore index 6103e5e17988..8df93efda629 100644 --- a/Packs/CommonScripts/.pack-ignore +++ b/Packs/CommonScripts/.pack-ignore @@ -107,6 +107,8 @@ ignore=RN114 ignore=SC106,BA124 [known_words] +pyminizip +pyzipper unzipfile zipstrings extractinbetween diff --git a/Packs/CommonScripts/ReleaseNotes/1_14_49.md b/Packs/CommonScripts/ReleaseNotes/1_14_49.md index 83310a8d972c..0858913e4d03 100644 --- a/Packs/CommonScripts/ReleaseNotes/1_14_49.md +++ b/Packs/CommonScripts/ReleaseNotes/1_14_49.md @@ -3,4 +3,4 @@ ##### StixCreator -Fixed an issue in the **TAXII2ApiModule** related to *TAXII2 server* integration. +Fixed an issue in the **TAXII2ApiModule** related to *TAXII2 server* integration. \ No newline at end of file diff --git a/Packs/CommonScripts/ReleaseNotes/1_15_0.md b/Packs/CommonScripts/ReleaseNotes/1_15_0.md new file mode 100644 index 000000000000..e628c2aa6619 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_15_0.md @@ -0,0 +1,6 @@ + +#### Scripts + +##### ZipFile +- Updated the Docker image to: *demisto/py3-tools:1.0.0.95440*. +- Replaced the *pyminizip* library with *pyzipper*. diff --git a/Packs/CommonScripts/Scripts/ZipFile/README.md b/Packs/CommonScripts/Scripts/ZipFile/README.md index ceceb3808122..44f44bbfa80c 100644 --- a/Packs/CommonScripts/Scripts/ZipFile/README.md +++ b/Packs/CommonScripts/Scripts/ZipFile/README.md @@ -36,3 +36,7 @@ Supported Cortex XSOAR versions: 5.0.0 and later. | File.SSDeep | The ssdeep hash of the file \(same as displayed in file entries\). | String | | File.Extension | The file extension, for example: 'xls'. | String | | File.Type | The file type, as determined by libmagic \(same as displayed in file entries\). | String | + +### Troubleshooting +Because of security reasons we support only AES encryption which is not supported on the Windows OS without 3rd party unzip applications. For more information about the encryption methods, see https://en.wikipedia.org/wiki/ZIP_(file_format)#Encryption. + diff --git a/Packs/CommonScripts/Scripts/ZipFile/ZipFile.py b/Packs/CommonScripts/Scripts/ZipFile/ZipFile.py index 752ddd4166e6..f6fbc25e4d9e 100644 --- a/Packs/CommonScripts/Scripts/ZipFile/ZipFile.py +++ b/Packs/CommonScripts/Scripts/ZipFile/ZipFile.py @@ -4,13 +4,41 @@ import re import shutil -import zipfile -import pyminizip +import pyzipper from os.path import isfile ESCAPE_CHARACTERS = r'[/\<>"|?*]' +def test_compression_succeeded(zip_name: str, password: str = None): + with pyzipper.AESZipFile(zip_name) as zf: + # testing for file integrity + if password: + zf.setpassword(bytes(password, 'utf-8')) + ret = zf.testzip() + if ret is not None: + demisto.info('zf.testzip() failed') + raise DemistoException('There was a problem with zipping the file: ' + ret + ' is corrupted') + + +def compress_multiple(file_names: List[str], zip_name: str, password: str = None): + """ + Compress multiple files into a zip file. + :param file_names: list of file names to compress + :param zip_name: name of the zip file to create + :param password: password to use for encryption + """ + compression = pyzipper.ZIP_DEFLATED + encryption = pyzipper.WZ_AES if password else None + demisto.debug(f'zipping {file_names=}') + with pyzipper.AESZipFile(zip_name, mode='w', compression=compression, encryption=encryption) as zf: + zf.pwd = bytes(password, 'utf-8') if password else None + for file_name in file_names: + zf.write(file_name) + test_compression_succeeded(zip_name, password) + zf.close() + + def escape_illegal_characters_in_file_name(file_name: str) -> str: if file_name: file_name = re.sub(ESCAPE_CHARACTERS, '-', file_name) @@ -20,10 +48,6 @@ def escape_illegal_characters_in_file_name(file_name: str) -> str: def main(): - try: # in order to support compression of the file - compression = zipfile.ZIP_DEFLATED - except Exception: - compression = zipfile.ZIP_STORED try: args = demisto.args() zipName = None @@ -75,21 +99,7 @@ def main(): zipName = fileCurrentName + '.zip' # zipping the file - if password: - pyminizip.compress_multiple(file_names, ['./'] * len(file_names), zipName, password, 5) - - else: - zf = zipfile.ZipFile(zipName, mode='w') - try: - for file_name in file_names: - zf.write(file_name, compress_type=compression) - # testing for file integrity - ret = zf.testzip() - if ret is not None: - raise DemistoException('There was a problem with the zipping, file: ' + ret + ' is corrupted') - - finally: - zf.close() + compress_multiple(file_names, zipName, password) with open(zipName, 'rb') as f: file_data = f.read() diff --git a/Packs/CommonScripts/Scripts/ZipFile/ZipFile.yml b/Packs/CommonScripts/Scripts/ZipFile/ZipFile.yml index ea26261fa2b4..680eb9ba654c 100644 --- a/Packs/CommonScripts/Scripts/ZipFile/ZipFile.yml +++ b/Packs/CommonScripts/Scripts/ZipFile/ZipFile.yml @@ -3,11 +3,11 @@ args: isArray: true name: entryID required: true -- description: 'Name of the output file, for example: zipName="test" would result in output file "test.zip"' +- description: 'Name of the output file, for example: zipName="test" would result in output file "test.zip".' name: zipName -- description: 'Used to create a password protected zip file. Example: password="abcd"' +- description: 'Used to create a password protected zip file. Example: password="abcd".' name: password -comment: Zip a file and upload to war room +comment: Zip a file and upload to war room. commonfields: id: ZipFile version: -1 @@ -54,7 +54,7 @@ tags: timeout: '0' type: python subtype: python3 -dockerimage: demisto/py3-tools:1.0.0.49703 +dockerimage: demisto/py3-tools:1.0.0.95440 fromversion: 5.0.0 tests: -- ZipFile-Test \ No newline at end of file +- ZipFile-Test diff --git a/Packs/CommonScripts/Scripts/ZipFile/ZipFile_test.py b/Packs/CommonScripts/Scripts/ZipFile/ZipFile_test.py index c744c828a42c..d62e3fe08a27 100644 --- a/Packs/CommonScripts/Scripts/ZipFile/ZipFile_test.py +++ b/Packs/CommonScripts/Scripts/ZipFile/ZipFile_test.py @@ -1,5 +1,8 @@ +import os import pytest -from ZipFile import escape_illegal_characters_in_file_name +import pyzipper +from ZipFile import escape_illegal_characters_in_file_name, compress_multiple +import tempfile ESCAPE_CHARACTERS_PACK = [ ('/Users/user/Downloads/b/a/testingfile.txt', '-Users-user-Downloads-b-a-testingfile.txt'), @@ -9,6 +12,80 @@ ] +def unzip(zip_file_path: str, password: str = None): + with tempfile.TemporaryDirectory() as unzip_dir, pyzipper.AESZipFile(zip_file_path) as zf: + zf.pwd = bytes(password, 'utf-8') if password else None + zf.extractall(path=unzip_dir) + + @pytest.mark.parametrize(('input_name', 'output_name'), ESCAPE_CHARACTERS_PACK) def test_escape_characters_in_file_name(input_name, output_name): assert escape_illegal_characters_in_file_name(input_name) == output_name + + +def test_compress_multiple_with_password(): + """ + Given: + - A directory with files to zip. + When: + - Calling the function compress_multiple. + Then: + - The function should not raise an exception. + """ + test_data_dir = './test_data' + file_names = [os.path.join(test_data_dir, f) for f in os.listdir(test_data_dir) if + os.path.isfile(os.path.join(test_data_dir, f))] + with tempfile.NamedTemporaryFile(suffix='.zip') as tmp_zip: + zip_name = tmp_zip.name + compress_multiple( + file_names=file_names, + zip_name=zip_name, + password='123' + ) + + +def test_zip_and_unzip_with_password(): + """ + Given: + - A directory with files to zip. + When: + - Calling the function compress_multiple with a password. + Then: + - We can unzip the file with the correct password. + """ + test_data_dir = './test_data' + file_names = [os.path.join(test_data_dir, f) for f in os.listdir(test_data_dir) if + os.path.isfile(os.path.join(test_data_dir, f))] + with tempfile.NamedTemporaryFile(suffix='.zip') as tmp_zip: + zip_name = tmp_zip.name + compress_multiple( + file_names=file_names, + zip_name=zip_name, + password='123' + ) + unzip(zip_name, '123') + + +def test_unzip_wrong_password(): + """ + Given: + - A directory with files to zip. + When: + - Calling the function compress_multiple with a password. + Then: + - We can not unzip the file with the wrong password. + """ + test_data_dir = './test_data' + file_names = [os.path.join(test_data_dir, f) for f in os.listdir(test_data_dir) if + os.path.isfile(os.path.join(test_data_dir, f))] + with tempfile.NamedTemporaryFile(suffix='.zip') as tmp_zip: + zip_name = tmp_zip.name + compress_multiple( + file_names=file_names, + zip_name=zip_name, + password='123' + ) + with pytest.raises(Exception) as e: + unzip(zip_name, '1234') + + assert 'Bad password' in e.value.args[0] diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_image.png b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_image.png new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_image.svg b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_image.svg new file mode 100644 index 000000000000..c55c7ad33364 --- /dev/null +++ b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_image.svg @@ -0,0 +1 @@ +Cortex_XSoar_logos_RGB \ No newline at end of file diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.docx b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.docx new file mode 100644 index 000000000000..47d20de11fca Binary files /dev/null and b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.docx differ diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pages b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pages new file mode 100644 index 000000000000..c6a9efcc3e6a Binary files /dev/null and b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pages differ diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pdf b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pdf new file mode 100644 index 000000000000..92d6584eeb57 Binary files /dev/null and b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.pdf differ diff --git a/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.txt b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.txt new file mode 100644 index 000000000000..9daeafb9864c --- /dev/null +++ b/Packs/CommonScripts/Scripts/ZipFile/test_data/test_txt.txt @@ -0,0 +1 @@ +test diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 79532f01fdd7..69dbaea45504 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.14.49", + "currentVersion": "1.15.0", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Tests/docker_native_image_config.json b/Tests/docker_native_image_config.json index 5593fcc9b467..521238550e95 100644 --- a/Tests/docker_native_image_config.json +++ b/Tests/docker_native_image_config.json @@ -174,6 +174,14 @@ "native:dev", "native:candidate" ] + }, + { + "id":"ZipFile", + "reason":"see CIAC-10361", + "ignored_native_images":[ + "native:8.6", + "native:candidate" + ] } ], "flags_versions_mapping":{