Merge branch 'main' into hashicorp

.github/workflows/benchmark.yml

@@ -136,7 +136,7 @@ jobs:
 
       - name: Run CredSweeper tool
         run: |
-          credsweeper --banner --log info --jobs $(nproc) --subtext --path data --save-json report.${{ github.event.pull_request.head.sha }}.json | tee credsweeper.${{ github.event.pull_request.head.sha }}.log
+          credsweeper --banner --log info --jobs $(nproc) --sort --subtext --path data --save-json report.${{ github.event.pull_request.head.sha }}.json | tee credsweeper.${{ github.event.pull_request.head.sha }}.log
 
       - name: Run Benchmark
         run: |
@@ -177,7 +177,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9", "3.10", "3.11" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
 

.github/workflows/check.yml

@@ -76,7 +76,7 @@ jobs:
       id: setup_python
       uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Install CredSweeper and auxiliary packages
       id: setup_credsweeper
@@ -105,6 +105,10 @@ jobs:
       if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
       run: pylint --py-version=3.11 --errors-only credsweeper
 
+    - name: Analysing the code with pylint and minimum Python version 3.12
+      if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
+      run: pylint --py-version=3.12 --errors-only credsweeper
+
     # # # mypy
 
     - name: Analysing the code with mypy and minimum Python version 3.8
@@ -127,6 +131,11 @@ jobs:
       run: |
         mypy --config-file .mypy.ini --python-version=3.11 credsweeper
 
+    - name: Analysing the code with mypy and minimum Python version 3.12
+      if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
+      run: |
+        mypy --config-file .mypy.ini --python-version=3.12 credsweeper
+
     # # # documentation
 
     - name: Analysing the code with pylint for NEW missed docstrings of classes or functions
@@ -140,7 +149,7 @@ jobs:
       if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
       run: |
         cd docs
-        pip install -r requirements.txt
+        pip install --requirement requirements.txt
         make html
         cd source
         python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html

.github/workflows/fuzz.yml

@@ -44,8 +44,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt
-        python -m pip install -r fuzz/requirements.txt
+        python -m pip install --requirement requirements.txt
+        python -m pip install --requirement fuzz/requirements.txt
 
     - name: Run fuzzing test with COVERAGE
       id: run_fuzz

.github/workflows/pypi.yml

@@ -31,12 +31,13 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       with:
-        python-version: "3.8"
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install --requirement requirements.txt
+        pip freeze
         python -m build
 
     - name: Publish

.github/workflows/test.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
 
@@ -52,7 +52,8 @@ jobs:
 
     - name: Install application
       run: |
-        pip install .
+        python -m pip install .
+        python -m pip freeze
 
     - name: Remove sources dir to check installation
       if: runner.os != 'Windows'
@@ -70,6 +71,11 @@ jobs:
       run: |
         pip install pytest pytest-random-order deepdiff
 
+    - name: Suppress warning ``...Unsupported Windows version (2022server)...``
+      if: ${{ matrix.python-version == '3.12' && matrix.os == 'windows-latest' }}
+      run: |
+        echo "PYTHONWARNINGS=ignore::UserWarning:onnxruntime.capi.onnxruntime_validation:26" >> $env:GITHUB_ENV
+
     - name: UnitTest with pytest
       run: |
         # put the command into one line to use in various OS to avoid processing differences in new line char sequence
@@ -83,7 +89,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
 
@@ -106,6 +112,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install --requirement requirements.txt
+        python -m pip freeze
 
     - name: UnitTest with pytest and coverage
       run: |
@@ -159,7 +166,7 @@ jobs:
         path: coverage_html
 
     - name: Upload coverage reports to Codecov
-      if: ${{ matrix.python-version == '3.8' }}
+      if: ${{ matrix.python-version == '3.10' }}
       uses: codecov/codecov-action@ab904c41d6ece82784817410c45d8b8c02684457 # v3.1.6
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

README.md

@@ -46,7 +46,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.8, 3.9, 3.10, 3.11
+- Python 3.8, 3.9, 3.10, 3.11, 3.12
 
 ### Installation
 

credsweeper/ml_model/features/word_in.py

@@ -44,15 +44,15 @@ def extract(self, candidate: Candidate) -> Any:
 
     def word_in_str(self, a_string: str) -> np.ndarray:
         """Returns array with words included in a string"""
-        result = np.zeros(shape=[self.dimension], dtype=np.int8)
+        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
         for i, word in self.enumerated_words:
             if word in a_string:
                 result[i] = 1
         return np.array([result])
 
     def word_in_set(self, a_strings_set: Set[str]) -> np.ndarray:
         """Returns array with words matches in a_strings_set"""
-        result = np.zeros(shape=[self.dimension], dtype=np.int8)
+        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
         for i, word in self.enumerated_words:
             if word in a_strings_set:
                 result[i] = 1

credsweeper/ml_model/ml_validator.py

@@ -2,7 +2,7 @@
 import logging
 import string
 from pathlib import Path
-from typing import List, Tuple, Union, Optional
+from typing import List, Tuple, Union, Optional, Dict
 
 import numpy as np
 import onnxruntime as ort
@@ -90,7 +90,7 @@ def __init__(
     @staticmethod
     def encode(text: str, limit: int) -> np.ndarray:
         """Encodes prepared text to array"""
-        result_array = np.zeros(shape=(limit, MlValidator.NUM_CLASSES), dtype=np.float32)
+        result_array: np.ndarray = np.zeros(shape=(limit, MlValidator.NUM_CLASSES), dtype=np.float32)
         if text is None:
             return result_array
         len_text = len(text)
@@ -122,7 +122,7 @@ def encode_value(text: str) -> np.ndarray:
 
     def _call_model(self, line_input: np.ndarray, variable_input: np.ndarray, value_input: np.ndarray,
                     feature_input: np.ndarray) -> np.ndarray:
-        input_feed = {
+        input_feed: Dict[str, np.ndarray] = {
             "line_input": line_input.astype(np.float32),
             "variable_input": variable_input.astype(np.float32),
             "value_input": value_input.astype(np.float32),
@@ -135,7 +135,7 @@ def _call_model(self, line_input: np.ndarray, variable_input: np.ndarray, value_
 
     def extract_common_features(self, candidates: List[Candidate]) -> np.ndarray:
         """Extract features that are guaranteed to be the same for all candidates on the same line with same value."""
-        feature_array = np.array([], dtype=np.float32)
+        feature_array: np.ndarray = np.array([], dtype=np.float32)
         # Extract features from credential candidate
         default_candidate = candidates[0]
         for feature in self.common_feature_list:
@@ -147,7 +147,7 @@ def extract_common_features(self, candidates: List[Candidate]) -> np.ndarray:
 
     def extract_unique_features(self, candidates: List[Candidate]) -> np.ndarray:
         """Extract features that can be different between candidates. Join them with or operator."""
-        feature_array = np.array([], dtype=np.int8)
+        feature_array: np.ndarray = np.array([], dtype=np.int8)
         default_candidate = candidates[0]
         for feature in self.unique_feature_list:
             new_feature = feature([default_candidate])[0]
@@ -220,7 +220,7 @@ def validate_groups(self, group_list: List[Tuple[CandidateKey, List[Candidate]]]
         variable_input_list = []
         value_input_list = []
         features_list = []
-        probability = np.zeros(len(group_list), dtype=np.float32)
+        probability: np.ndarray = np.zeros(len(group_list), dtype=np.float32)
         head = tail = 0
         for group_key, candidates in group_list:
             line_input, variable_input, value_input, feature_array = self.get_group_features(candidates)

docs/howto/how-to-contribute.md

@@ -4,13 +4,15 @@ Thank you for your interest in contributing to the CredSweeper tool!
 
 The document covers the process for contributing to the CredSweeper code and documentation. Contributions may be as simple as typo corrections or as complex as new features.
 
-1.  [Process for contributing](#process-for-contributing)
+1. [Process for contributing](#process-for-contributing)
+
     1. [Repository structure](#repository-structure)
-    1. [File Name](#file-name)
-    1. [Self Test & Verification](#self-test-and-verification)
-1.  [How to PR](#how-to-pr)
-1.  [DOs and DON'Ts](#dos-and-donts)
+    2. [File Name](#file-name)
+    3. [Self Test & Verification](#self-test-and-verification)
+    
+2. [How to PR](#how-to-pr)
 
+3. [DOs and DON'Ts](#dos-and-donts)
 
 ## Process for contributing
 
@@ -20,7 +22,7 @@ You need a basic understanding of [Git and GitHub.com](https://guides.github.com
 
 You can also look at our [issues](https://github.com/Samsung/CredSweeper/issues) list and volunteer to work on the ones you're interested in.
 
-**Step 2:** Fork the `/Samsung/CredSweeper` repo and create a branch for your changes.
+**Step 2:** Fork the [Samsung/CredSweeper](https://github.com/Samsung/CredSweeper/fork) repo and create a branch for your changes.
 
 For small changes, you can use GitHub's web interface. Simply click the **Edit the file in your fork of this project** on the file you'd like to change.
 GitHub creates the new branch for you when you submit the changes.
@@ -77,40 +79,46 @@ The maintainers will merge your PR into the master branch once feedback has been
    ```bash
    git clone https://github.com/YOUR-USERNAME/CredSweeper.git
    ```
+
 3. Set to synchronize the original repository and the forked repository.
 
    ```bash
    git remote -v
    git remote add upstream https://github.com/Samsung/CredSweeper.git
    git remote -v
    ```
+
 4. Create a new branch on the forked repository or the local repository,
    and switch to the new branch.
 
    ```bash
    git checkout -b <new branch name>
    ```
+
 5. Install Yapf as a pre-commit hook with
 
    ``` bash
    pip install pre-commit
    pre-commit install
    ```
+
 6. Create a local commit.
 
    ```bash
    git status
    git add
    git commit -a
    ```
+
 7. Push the branch
 
    ```bash
    git push origin <new branch name>
    ```
+
 8. Open a pull request on https://github.com/Samsung/CredSweeper.
 
-    All tests and checks MUST be passed.
+   All tests and checks MUST be passed
    - Codestyle check
    - Static analysis
    - Unit tests

docs/requirements.txt

@@ -1,5 +1,3 @@
-m2r2==0.3.2
-# ^ the version is latest which has no conflicts
-sphinx_rtd_theme==1.3.0
-sphinx-autodoc-typehints==1.24.0
-autodocsumm==0.2.11
+myst_parser[linkify]==4.0.0
+sphinx==8.1.3
+sphinx_rtd_theme==3.0.2

docs/source/conf.py

@@ -5,7 +5,6 @@
 
 import os
 import sys
-import sphinx_rtd_theme
 
 sys.path.insert(0, os.path.abspath('../..'))
 di = os.path.abspath(os.pardir)
@@ -43,16 +42,38 @@
     'sphinx.ext.viewcode',
     'sphinx.ext.todo',
     'sphinx.ext.napoleon',
-    'sphinx_autodoc_typehints',
-    'm2r2',
+    'myst_parser',
 ]
 
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3/', None),
     'numpy': ('https://numpy.org/doc/stable/', None),
 }
 
-source_suffix = ['.rst', '.md']
+myst_enable_extensions = [
+    "amsmath",
+    "attrs_inline",
+    "colon_fence",
+    "deflist",
+    "dollarmath",
+    "fieldlist",
+    "html_admonition",
+    "html_image",
+    "linkify",
+    "replacements",
+    "smartquotes",
+    "strikethrough",
+    "substitution",
+    "tasklist",
+]
+
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.txt': 'markdown',
+    '.md': 'markdown',
+}
+
+myst_heading_anchors = 3
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -94,7 +115,6 @@
 html_logo = 'https://raw.githubusercontent.com/Samsung/CredSweeper/main/docs/images/Logo.png'
 
 html_scaled_image_link = False
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # -- Options for HTMLHelp output ------------------------------------------
 

docs/source/credsweeper.deep_scanner.rst

@@ -108,6 +108,14 @@ credsweeper.deep\_scanner.pkcs12\_scanner module
    :undoc-members:
    :show-inheritance:
 
+credsweeper.deep\_scanner.pptx\_scanner module
+----------------------------------------------
+
+.. automodule:: credsweeper.deep_scanner.pptx_scanner
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 credsweeper.deep\_scanner.tar\_scanner module
 ---------------------------------------------
 
@@ -116,6 +124,14 @@ credsweeper.deep\_scanner.tar\_scanner module
    :undoc-members:
    :show-inheritance:
 
+credsweeper.deep\_scanner.xlsx\_scanner module
+----------------------------------------------
+
+.. automodule:: credsweeper.deep_scanner.xlsx_scanner
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 credsweeper.deep\_scanner.xml\_scanner module
 ---------------------------------------------