benchmark migration to py3.10,

style
62d92ab2f91a18e861d846a7b8a0c3a7

.github/workflows/benchmark.yml

@@ -40,11 +40,11 @@ jobs:
           path: data
           key: cred-data-${{ hashFiles('checksums.md5') }}
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         if: steps.cache-data.outputs.cache-hit != 'true'
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip
@@ -99,10 +99,10 @@ jobs:
         if: steps.cache-data.outputs.cache-hit == 'true'
         run: ls -al . && ls -al data
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip
@@ -169,7 +169,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.10", "3.9", "3.10", "3.11" ]
 
     steps:
 
@@ -388,11 +388,11 @@ jobs:
           mv data ${{ github.workspace }}/CredData/
           mv meta ${{ github.workspace }}/CredData/
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         if: steps.cache-data.outputs.cache-hit != 'true'
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip

.github/workflows/check.yml

@@ -58,7 +58,7 @@ jobs:
     - name: Check ml_model.onnx integrity
       if: ${{ always() && steps.code_checkout.conclusion == 'success' }}
       run: |
-        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 1cbfbd7fb1e657d137c9eeec26a07ad4
+        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 62d92ab2f91a18e861d846a7b8a0c3a7
 
     # # # Python setup
 

experiment/requirements.txt

@@ -6,11 +6,12 @@ protobuf==3.20.3
 tensorflow==2.13.1
 tf2onnx==1.16.0
 wrapt==1.14.1
-matplotlib
-tensorrt
+tensorrt==10.1.0
 
 # scikit-learn 1.5.0 may not support python 3.8
-scikit-learn
+scikit-learn==1.5.0
 
+# version
 types-tensorflow
+matplotlib
 colorama

tests/filters/test_value_allowlist_check.py

@@ -15,4 +15,3 @@ def test_value_allowlist_check_p(self, file_path: pytest.fixture, success_line:
     def test_value_allowlist_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueAllowlistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_array_dictionary_check.py

@@ -44,4 +44,3 @@ def test_array_assignment_p(self, token_rule: Rule, file_path: pytest.fixture, l
         """Evaluate that filter do not remove assignments to array or dictionary declaration"""
         line_data = get_line_data(file_path, line=line, pattern=token_rule.patterns[0])
         assert ValueArrayDictionaryCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False
-

tests/filters/test_value_base32_data_check.py

@@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No
     def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueBase32DataCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_base64_data_check.py

@@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No
     def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueBase64DataCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_blocklist_check.py

@@ -17,4 +17,3 @@ def test_value_blocklist_p(self, file_path: pytest.fixture, success_line: pytest
     def test_value_blocklist_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueBlocklistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_camel_case_check.py

@@ -15,4 +15,3 @@ def test_value_camelcase_p(self, file_path: pytest.fixture, success_line: pytest
     def test_value_camelcase_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueCamelCaseCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_couple_keyword_check.py

@@ -18,4 +18,3 @@ def test_value_couple_keyword_check_p(self, file_path: pytest.fixture, line: str
     def test_value_couple_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueCoupleKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_dictionary_keyword_check.py

@@ -15,4 +15,3 @@ def test_value_dictionary_keyword_check_p(self, file_path: pytest.fixture, succe
     def test_value_dictionary_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueDictionaryKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_dictionary_value_length_check.py

@@ -16,4 +16,3 @@ def test_value_dictionary_value_length_check_p(self, file_path: pytest.fixture,
     def test_value_dictionary_value_length_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueDictionaryValueLengthCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_entropy_base32_check.py

@@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No
     def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueEntropyBase32Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_entropy_base36_check.py

@@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No
     def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueEntropyBase36Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_entropy_base64_check.py

@@ -16,4 +16,3 @@ def test_value_entropy_check_p(self, file_path: pytest.fixture, line: str) -> No
     def test_value_entropy_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueEntropyBase64Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_file_path_check.py

@@ -11,15 +11,17 @@ def test_value_file_path_check_p(self, file_path: pytest.fixture, success_line:
         line_data = get_line_data(file_path, line=success_line, pattern=LINE_VALUE_PATTERN)
         assert ValueFilePathCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False
 
-    @pytest.mark.parametrize("line", [
-        "/home/user/tmp",  # simple path
-        "../..",  # path
-        "file:///Crackle/filepath/",  # path from browser url
-        "~/.custompass",  # path with synonym
-        "crackle/filepath_txt",
-        "crackle/file.path",  #
-        "C:\\Crackle\\filepath",  #
-    ])
+    @pytest.mark.parametrize(
+        "line",
+        [
+            "/home/user/tmp",  # simple path
+            "../..",  # path
+            "file:///Crackle/filepath/",  # path from browser url
+            "~/.custompass",  # path with synonym
+            "crackle/filepath_txt",
+            "crackle/file.path",  #
+            "C:\\Crackle\\filepath",  #
+        ])
     def test_value_file_path_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueFilePathCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True

tests/filters/test_value_first_word_check.py

@@ -15,4 +15,3 @@ def test_value_first_word_check_p(self, file_path: pytest.fixture, success_line:
     def test_value_first_word_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueFirstWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_last_word_check.py

@@ -15,4 +15,3 @@ def test_value_last_word_check_p(self, file_path: pytest.fixture, success_line:
     def test_value_last_word_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueLastWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_length_check.py

@@ -17,4 +17,3 @@ def test_value_length_check_p(self, file_path: pytest.fixture, config: Config,
     def test_value_length_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_method_check.py

@@ -15,4 +15,3 @@ def test_value_method_check_p(self, file_path: pytest.fixture, success_line: pyt
     def test_value_method_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueMethodCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_not_allowed_pattern.py

@@ -34,4 +34,3 @@ def test_value_last_word_check_p(self, token_rule: Rule, file_path: pytest.fixtu
     def test_value_last_word_check_n(self, token_rule: Rule, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueNotAllowedPatternCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_not_part_encoded.py

@@ -25,4 +25,3 @@ def test_value_not_part_encoded_n(self, config: pytest.fixture) -> None:
         target = AnalysisTarget(1, [Chars.BASE64STD_CHARS.value, "XXX"], [1, 2], DUMMY_DESCRIPTOR)
         line_data = LineData(config, "XXX", 1, 333, "", "", "", LINE_VALUE_PATTERN)
         assert ValueNotPartEncodedCheck().run(line_data, target) is True
-

tests/filters/test_value_number_check.py

@@ -19,4 +19,3 @@ def test_value_number_check_p(self, file_path: pytest.fixture, line: str) -> Non
     def test_value_number_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueNumberCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_pattern_check.py

@@ -55,4 +55,3 @@ def test_value_similarity_check_p(self, file_path: pytest.fixture, config: Confi
     def test_value_similarity_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValuePatternCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_split_keyword_check.py

@@ -16,4 +16,3 @@ def test_value_split_keyword_check_p(self, file_path: pytest.fixture, line: str)
     def test_value_split_keyword_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueSplitKeywordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_string_type_check.py

@@ -44,4 +44,3 @@ def test_value_string_type_check_not_quoted_source_file_p(self, line: str, confi
             pattern=pattern,
         )
         assert ValueStringTypeCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is False
-

tests/filters/test_value_token_base32_check.py

@@ -16,4 +16,3 @@ def test_value_token_base32_check_p(self, file_path: pytest.fixture, line: str)
     def test_value_token_base32_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueTokenBase32Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_token_base64_check.py

@@ -16,4 +16,3 @@ def test_value_token_base64_check_p(self, file_path: pytest.fixture, line: str)
     def test_value_token_base64_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueTokenBase64Check().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-

tests/filters/test_value_token_check.py

@@ -15,4 +15,3 @@ def test_value_token_check_p(self, file_path: pytest.fixture, success_line: pyte
     def test_value_token_check_n(self, file_path: pytest.fixture, line: str) -> None:
         line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN)
         assert ValueTokenCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True
-