diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 40613db6c8..a815ff91c2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,11 +40,11 @@ jobs: path: data key: cred-data-${{ hashFiles('checksums.md5') }} - - name: Set up Python 3.8 + - name: Set up Python 3.10 if: steps.cache-data.outputs.cache-hit != 'true' uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Update PIP run: python -m pip install --upgrade pip @@ -99,10 +99,10 @@ jobs: if: steps.cache-data.outputs.cache-hit == 'true' run: ls -al . && ls -al data - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Update PIP run: python -m pip install --upgrade pip @@ -169,7 +169,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.10", "3.9", "3.10", "3.11" ] steps: @@ -388,11 +388,11 @@ jobs: mv data ${{ github.workspace }}/CredData/ mv meta ${{ github.workspace }}/CredData/ - - name: Set up Python 3.8 + - name: Set up Python 3.10 if: steps.cache-data.outputs.cache-hit != 'true' uses: actions/setup-python@v3 with: - python-version: "3.8" + python-version: "3.10" - name: Update PIP run: python -m pip install --upgrade pip diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 863a2ba65d..9270f66b6a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,7 +58,7 @@ jobs: - name: Check ml_model.onnx integrity if: ${{ always() && steps.code_checkout.conclusion == 'success' }} run: | - md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 1cbfbd7fb1e657d137c9eeec26a07ad4 + md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 62d92ab2f91a18e861d846a7b8a0c3a7 # # # Python setup diff --git a/experiment/requirements.txt b/experiment/requirements.txt index 2f459687b5..3fffd21de9 100644 --- a/experiment/requirements.txt +++ b/experiment/requirements.txt @@ -6,11 +6,12 @@ protobuf==3.20.3 tensorflow==2.13.1 tf2onnx==1.16.0 wrapt==1.14.1 -matplotlib -tensorrt +tensorrt==10.1.0 # scikit-learn 1.5.0 may not support python 3.8 -scikit-learn +scikit-learn==1.5.0 +# version types-tensorflow +matplotlib colorama diff --git a/tests/filters/test_value_allowlist_check.py b/tests/filters/test_value_allowlist_check.py index 042d0cb39a..c403f4346d 100644 --- a/tests/filters/test_value_allowlist_check.py +++ b/tests/filters/test_value_allowlist_check.py @@ -15,4 +15,3 @@ def test_value_allowlist_check_p(self, file_path: pytest.fixture, success_line: def test_value_allowlist_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueAllowlistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_array_dictionary_check.py b/tests/filters/test_value_array_dictionary_check.py index ea0c660966..3ea0cbd7f2 100644 --- a/tests/filters/test_value_array_dictionary_check.py +++ b/tests/filters/test_value_array_dictionary_check.py @@ -44,4 +44,3 @@ def test_array_assignment_p(self, token_rule: Rule, file_path: pytest.fixture, l """Evaluate that filter do not remove assignments to array or dictionary declaration""" line_data = get_line_data(file_path, line=line, pattern=token_rule.patterns[0]) assert ValueArrayDictionaryCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False - diff --git a/tests/filters/test_value_base32_data_check.py b/tests/filters/test_value_base32_data_check.py index 3232e1302f..845292da9f 100644 --- a/tests/filters/test_value_base32_data_check.py +++ b/tests/filters/test_value_base32_data_check.py @@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueBase32DataCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_base64_data_check.py b/tests/filters/test_value_base64_data_check.py index 64d3587409..e279df61f1 100644 --- a/tests/filters/test_value_base64_data_check.py +++ b/tests/filters/test_value_base64_data_check.py @@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueBase64DataCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_blocklist_check.py b/tests/filters/test_value_blocklist_check.py index 8bfb22532b..ba10a6768e 100644 --- a/tests/filters/test_value_blocklist_check.py +++ b/tests/filters/test_value_blocklist_check.py @@ -17,4 +17,3 @@ def test_value_blocklist_p(self, file_path: pytest.fixture, success_line: pytest def test_value_blocklist_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueBlocklistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_camel_case_check.py b/tests/filters/test_value_camel_case_check.py index f2f58b4438..8e623d3863 100644 --- a/tests/filters/test_value_camel_case_check.py +++ b/tests/filters/test_value_camel_case_check.py @@ -15,4 +15,3 @@ def test_value_camelcase_p(self, file_path: pytest.fixture, success_line: pytest def test_value_camelcase_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueCamelCaseCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_couple_keyword_check.py b/tests/filters/test_value_couple_keyword_check.py index 76f72e72d0..4b868a71cc 100644 --- a/tests/filters/test_value_couple_keyword_check.py +++ b/tests/filters/test_value_couple_keyword_check.py @@ -18,4 +18,3 @@ def test_value_couple_keyword_check_p(self, file_path: pytest.fixture, line: str def test_value_couple_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueCoupleKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_dictionary_keyword_check.py b/tests/filters/test_value_dictionary_keyword_check.py index 05c80f56dd..f6bbf5c096 100644 --- a/tests/filters/test_value_dictionary_keyword_check.py +++ b/tests/filters/test_value_dictionary_keyword_check.py @@ -15,4 +15,3 @@ def test_value_dictionary_keyword_check_p(self, file_path: pytest.fixture, succe def test_value_dictionary_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueDictionaryKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_dictionary_value_length_check.py b/tests/filters/test_value_dictionary_value_length_check.py index 8ec470a4cd..9de203b1c6 100644 --- a/tests/filters/test_value_dictionary_value_length_check.py +++ b/tests/filters/test_value_dictionary_value_length_check.py @@ -16,4 +16,3 @@ def test_value_dictionary_value_length_check_p(self, file_path: pytest.fixture, def test_value_dictionary_value_length_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueDictionaryValueLengthCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_entropy_base32_check.py b/tests/filters/test_value_entropy_base32_check.py index 02345e9bd7..cf824bca57 100644 --- a/tests/filters/test_value_entropy_base32_check.py +++ b/tests/filters/test_value_entropy_base32_check.py @@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueEntropyBase32Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_entropy_base36_check.py b/tests/filters/test_value_entropy_base36_check.py index ed244eabe1..e45d0276b0 100644 --- a/tests/filters/test_value_entropy_base36_check.py +++ b/tests/filters/test_value_entropy_base36_check.py @@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueEntropyBase36Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_entropy_base64_check.py b/tests/filters/test_value_entropy_base64_check.py index b752df16cd..f91fdaf113 100644 --- a/tests/filters/test_value_entropy_base64_check.py +++ b/tests/filters/test_value_entropy_base64_check.py @@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueEntropyBase64Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_file_path_check.py b/tests/filters/test_value_file_path_check.py index 39c39c4458..b2590c8d65 100644 --- a/tests/filters/test_value_file_path_check.py +++ b/tests/filters/test_value_file_path_check.py @@ -11,15 +11,17 @@ def test_value_file_path_check_p(self, file_path: pytest.fixture, success_line: line_data = get_line_data(file_path, line=success_line, pattern=LINE_VALUE_PATTERN) assert ValueFilePathCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False - @pytest.mark.parametrize("line", [ - "/home/user/tmp", # simple path - "../..", # path - "file:///Crackle/filepath/", # path from browser url - "~/.custompass", # path with synonym - "crackle/filepath_txt", - "crackle/file.path", # - "C:\\Crackle\\filepath", # - ]) + @pytest.mark.parametrize( + "line", + [ + "/home/user/tmp", # simple path + "../..", # path + "file:///Crackle/filepath/", # path from browser url + "~/.custompass", # path with synonym + "crackle/filepath_txt", + "crackle/file.path", # + "C:\\Crackle\\filepath", # + ]) def test_value_file_path_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueFilePathCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/filters/test_value_first_word_check.py b/tests/filters/test_value_first_word_check.py index 6012c8d744..5281388432 100644 --- a/tests/filters/test_value_first_word_check.py +++ b/tests/filters/test_value_first_word_check.py @@ -15,4 +15,3 @@ def test_value_first_word_check_p(self, file_path: pytest.fixture, success_line: def test_value_first_word_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueFirstWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_last_word_check.py b/tests/filters/test_value_last_word_check.py index b4893bf73a..7e036e86e9 100644 --- a/tests/filters/test_value_last_word_check.py +++ b/tests/filters/test_value_last_word_check.py @@ -15,4 +15,3 @@ def test_value_last_word_check_p(self, file_path: pytest.fixture, success_line: def test_value_last_word_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueLastWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_length_check.py b/tests/filters/test_value_length_check.py index 17ebba5d9c..69bd5f8099 100644 --- a/tests/filters/test_value_length_check.py +++ b/tests/filters/test_value_length_check.py @@ -17,4 +17,3 @@ def test_value_length_check_p(self, file_path: pytest.fixture, config: Config, def test_value_length_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_method_check.py b/tests/filters/test_value_method_check.py index 90e8bc3df1..4b2967e39e 100644 --- a/tests/filters/test_value_method_check.py +++ b/tests/filters/test_value_method_check.py @@ -15,4 +15,3 @@ def test_value_method_check_p(self, file_path: pytest.fixture, success_line: pyt def test_value_method_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueMethodCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_not_allowed_pattern.py b/tests/filters/test_value_not_allowed_pattern.py index 314eb782ac..cc1e24059e 100644 --- a/tests/filters/test_value_not_allowed_pattern.py +++ b/tests/filters/test_value_not_allowed_pattern.py @@ -34,4 +34,3 @@ def test_value_last_word_check_p(self, token_rule: Rule, file_path: pytest.fixtu def test_value_last_word_check_n(self, token_rule: Rule, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueNotAllowedPatternCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_not_part_encoded.py b/tests/filters/test_value_not_part_encoded.py index a68cfc8943..4550b6a8dd 100644 --- a/tests/filters/test_value_not_part_encoded.py +++ b/tests/filters/test_value_not_part_encoded.py @@ -25,4 +25,3 @@ def test_value_not_part_encoded_n(self, config: pytest.fixture) -> None: target = AnalysisTarget(1, [Chars.BASE64STD_CHARS.value, "XXX"], [1, 2], DUMMY_DESCRIPTOR) line_data = LineData(config, "XXX", 1, 333, "", "", "", LINE_VALUE_PATTERN) assert ValueNotPartEncodedCheck().run(line_data, target) is True - diff --git a/tests/filters/test_value_number_check.py b/tests/filters/test_value_number_check.py index 0fc7c73c68..db0a3a0fca 100644 --- a/tests/filters/test_value_number_check.py +++ b/tests/filters/test_value_number_check.py @@ -19,4 +19,3 @@ def test_value_number_check_p(self, file_path: pytest.fixture, line: str) -> Non def test_value_number_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueNumberCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_pattern_check.py b/tests/filters/test_value_pattern_check.py index 04c5624bf6..8bcbbc2da9 100644 --- a/tests/filters/test_value_pattern_check.py +++ b/tests/filters/test_value_pattern_check.py @@ -55,4 +55,3 @@ def test_value_similarity_check_p(self, file_path: pytest.fixture, config: Confi def test_value_similarity_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValuePatternCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_split_keyword_check.py b/tests/filters/test_value_split_keyword_check.py index 6e80ec5023..0c1fe8167a 100644 --- a/tests/filters/test_value_split_keyword_check.py +++ b/tests/filters/test_value_split_keyword_check.py @@ -16,4 +16,3 @@ def test_value_split_keyword_check_p(self, file_path: pytest.fixture, line: str) def test_value_split_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueSplitKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_string_type_check.py b/tests/filters/test_value_string_type_check.py index 2e8da1faa2..d2cb157e39 100644 --- a/tests/filters/test_value_string_type_check.py +++ b/tests/filters/test_value_string_type_check.py @@ -44,4 +44,3 @@ def test_value_string_type_check_not_quoted_source_file_p(self, line: str, confi pattern=pattern, ) assert ValueStringTypeCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is False - diff --git a/tests/filters/test_value_token_base32_check.py b/tests/filters/test_value_token_base32_check.py index cc3990053c..d01b152fa9 100644 --- a/tests/filters/test_value_token_base32_check.py +++ b/tests/filters/test_value_token_base32_check.py @@ -16,4 +16,3 @@ def test_value_token_base32_check_p(self, file_path: pytest.fixture, line: str) def test_value_token_base32_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueTokenBase32Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_token_base64_check.py b/tests/filters/test_value_token_base64_check.py index cd75a49ccb..0e831d2991 100644 --- a/tests/filters/test_value_token_base64_check.py +++ b/tests/filters/test_value_token_base64_check.py @@ -16,4 +16,3 @@ def test_value_token_base64_check_p(self, file_path: pytest.fixture, line: str) def test_value_token_base64_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueTokenBase64Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True - diff --git a/tests/filters/test_value_token_check.py b/tests/filters/test_value_token_check.py index 4ed64be00f..f2b7622d38 100644 --- a/tests/filters/test_value_token_check.py +++ b/tests/filters/test_value_token_check.py @@ -15,4 +15,3 @@ def test_value_token_check_p(self, file_path: pytest.fixture, success_line: pyte def test_value_token_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueTokenCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True -