Update quality tools.

- Add pyproject-fmt to quality tools.
- Remove safety from quality tools.
- Remove duplication between unittest.sh scripts.
- Remove duplication between quality.sh scripts.
- Remove duplication between pip-compile.sh scripts.
- Remove duplication between pip-install.sh scripts.
- Add type checking to Python tests.

Closes #8928.

.circleci/config.yml

@@ -10,7 +10,7 @@ jobs:
         auth:
           username: $DOCKERHUB_USERNAME
           password: $DOCKERHUB_PASSWORD
-    parallelism: 5
+    parallelism: 6
     steps:
       - checkout
       - run: |
@@ -19,7 +19,8 @@ jobs:
             1) component=components/notifier;;
             2) component=components/api_server;;
             3) component=components/shared_code;;
-            4) component=tests/feature_tests;;
+            4) component=tests/application_tests;;
+            5) component=tests/feature_tests;;
           esac
           cd $component
           mkdir -p build
@@ -36,6 +37,10 @@ jobs:
           path: components/api_server/build
       - store_artifacts:
           path: components/shared_code/build
+      - store_artifacts:
+          path: components/application_tests/build
+      - store_artifacts:
+          path: components/feature_tests/build
 
   unittest_frontend:
     docker:
@@ -51,7 +56,7 @@ jobs:
           ci/unittest.sh
           ci/quality.sh
 
-  unittest_docs:
+  unittest_other:
     machine:
       image: default
     steps:
@@ -64,6 +69,12 @@ jobs:
           ci/pip-install.sh
           ci/unittest.sh
           ci/quality.sh
+      - run: |
+          cd release
+          python3 -m venv venv
+          . venv/bin/activate
+          ci/pip-install.sh
+          ci/quality.sh
 
   application_tests:
     machine:
@@ -117,7 +128,7 @@ workflows:
           context: QualityTime
       - unittest_frontend:
           context: QualityTime
-      - unittest_docs:
+      - unittest_other:
           context: QualityTime
       - docker/hadolint:
           context: QualityTime

.github/workflows/application-tests-quality.yml

@@ -0,0 +1,23 @@
+name: Application tests quality
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install dependencies
+      run: |
+        cd tests/application_tests
+        ci/pip-install.sh
+    - name: Quality
+      run: |
+        cd tests/application_tests
+        ci/quality.sh

.github/workflows/feature-tests-quality.yml

@@ -17,10 +17,6 @@ jobs:
       run: |
         cd tests/feature_tests
         ci/pip-install.sh
-    - name: Test
-      run: |
-        cd tests/feature_tests
-        ci/unittest.sh
     - name: Quality
       run: |
         cd tests/feature_tests

.github/workflows/release-quality.yml

@@ -0,0 +1,23 @@
+name: Release script quality
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install dependencies
+      run: |
+        cd release
+        ci/pip-install.sh
+    - name: Quality
+      run: |
+        cd release
+        ci/quality.sh

ci/base.sh

@@ -4,29 +4,19 @@
 
 set -e
 
-run () {
-    # Show the invoked command using a subdued text color so it's clear which tool is running.
+run() {
+    # Show the invoked command using a subdued text color so it is clear which tool is running.
     header='\033[95m'
     endstyle='\033[0m'
     echo -e "${header}$*${endstyle}"
     eval "$*"
 }
 
-spec () {
-    # The versions of tools are specified in pyproject.toml. This function calls the spec.py script which in turn
-    # reads the version numbers from the pyproject.toml file.
-
-    # Get the dir of this script so the spec.py script that is in the same dir as this script can be invoked:
-    SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-    python $SCRIPT_DIR/spec.py $*
+script_dir() {
+    # Get the dir of this script so that scripts that are in the same dir as this script can be invoked.
+    # See https://stackoverflow.com/questions/39340169/dir-cd-dirname-bash-source0-pwd-how-does-that-work.
+    echo $( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 }
 
-# Don't install tools in the global pipx home folder, but locally for each component:
-export PIPX_HOME=.pipx
-export PIPX_BIN_DIR=$PIPX_HOME/bin
-
 # For Windows compatibility; prevent path from ending with a ':'
-export PYTHONPATH=`python -c 'import sys;print(":".join(sys.argv[1:]))' src $PYTHONPATH`
-
-# Insert a custom compile command in generated requirements file, so it's clear how they are generated:
-export CUSTOM_COMPILE_COMMAND="ci/pip-compile.sh"
+export PYTHONPATH=$(python -c 'import sys;print(":".join(sys.argv[1:]))' src $PYTHONPATH)

ci/pip-base.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source base.sh
+
+# Insert a custom compile command in generated requirements file so it is clear how they are generated:
+export CUSTOM_COMPILE_COMMAND="ci/pip-compile.sh"
+
+run_pip_compile() {
+    for requirements_file in $(python $(script_dir)/requirements_files.py); do
+        extra=$([[ "$requirements_file" == *"-dev"* ]] && echo "--extra dev" || echo "")
+        run pip-compile $extra --output-file $requirements_file pyproject.toml
+    done
+}
+
+run_pip_install() {
+    run pip install --ignore-installed --quiet --use-pep517 $@
+}

ci/pipx-base.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source base.sh
+
+spec() {
+    # The versions of tools are specified in pyproject.toml. This function calls the spec.py script which in turn
+    # reads the version numbers from the pyproject.toml file. The function takes one argument: the package to return
+    # the spec for.
+    python $(script_dir)/spec.py $1
+}
+
+run_pipx() {
+    # Look up the version of the command using the spec function and run the command using pipx.
+    command_spec=$(spec $1)
+    shift 1
+    run pipx run $command_spec $@
+}
+
+# Don't install tools in the global pipx home folder, but locally for each component:
+export PIPX_HOME=.pipx
+export PIPX_BIN_DIR=$PIPX_HOME/bin

ci/python_files_and_folders.py

@@ -0,0 +1,13 @@
+"""Determine the Python files and folders in the current directory."""
+
+from pathlib import Path
+
+def python_files_and_folders() -> list[str]:
+    """Return the Python files and folders in the current directory."""
+    python_files = [python_file.name for python_file in Path(".").glob('*.py') if not python_file.name.startswith(".")]
+    python_folders = [folder_name for folder_name in ("src", "tests") if Path(folder_name).exists()]
+    return python_files + python_folders
+
+
+if __name__ == "__main__":
+    print(" ".join(python_files_and_folders()))

ci/quality-base.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+source pipx-base.sh
+
+PYTHON_FILES_AND_FOLDERS=$(python $(script_dir)/python_files_and_folders.py)
+
+run_ruff() {
+    run_pipx ruff check $PYTHON_FILES_AND_FOLDERS
+    run_pipx ruff format --check $PYTHON_FILES_AND_FOLDERS
+}
+
+run_fixit() {
+    run_pipx fixit lint $PYTHON_FILES_AND_FOLDERS
+}
+
+run_mypy() {
+    # Run mypy with or without pydantic plugin depending on whether pydantic is listed as dependency in the tools
+    # section of the optional dependencies in the pyproject.toml file.
+    pydantic_spec=$(spec pydantic)
+    if [[ "$pydantic_spec" == "" ]]; then
+        run_pipx mypy --python-executable=$(which python) $PYTHON_FILES_AND_FOLDERS
+    else
+        # To use the pydantic plugin, we need to first install mypy and then inject pydantic
+        run pipx install --force $(spec mypy) # --force works around this bug: https://github.com/pypa/pipx/issues/795
+        run pipx inject mypy $pydantic_spec
+        run $PIPX_BIN_DIR/mypy --python-executable=$(which python) $PYTHON_FILES_AND_FOLDERS
+    fi
+}
+
+run_pyproject_fmt() {
+    run_pipx pyproject-fmt --check pyproject.toml
+}
+
+run_bandit() {
+    run_pipx bandit --configfile pyproject.toml --quiet --recursive $PYTHON_FILES_AND_FOLDERS
+}
+
+run_pip_audit() {
+    run_pipx pip-audit --strict --progress-spinner=off $(python $(script_dir)/requirements_files.py "-r %s")
+}
+
+run_vulture() {
+    run_pipx vulture --min-confidence 0 $PYTHON_FILES_AND_FOLDERS .vulture_ignore_list.py $@
+}
+
+run_vale() {
+    run_pipx vale sync
+    run_pipx vale --no-wrap --glob "*.md" src
+}
+
+run_markdownlint() {
+    run ./node_modules/markdownlint-cli/markdownlint.js src/**/*.md
+}
+
+check_python_quality() {
+    run_ruff
+    run_fixit
+    run_mypy
+    run_pyproject_fmt
+    run_pip_audit
+    run_bandit
+    run_vulture
+}

ci/requirements_files.py

@@ -0,0 +1,27 @@
+"""Determine the Python requirements files.
+
+The script returns the requirements files as space separated string:
+
+$ requirements_files.py
+requirements/requirements.txt requirements/requirements-dev.txt
+
+The script takes an optional template argument that is used to wrap each requirements filename. For example:
+
+$ requirements_files.py "-r %s"
+-r requirements/requirements.txt -r requirements/requirements-dev.txt
+"""
+
+import sys
+from pathlib import Path
+
+
+def requirements_files() -> list[str]:
+    """Return the Python requirements files in the requirements directory."""
+    requirements_files = Path(".").glob("requirements/requirements*.txt")
+    # We never return the internal requirements file, because it does not need to be checked nor compiled
+    return [str(filename) for filename in requirements_files if "requirements-internal" not in filename.name]
+
+
+if __name__ == "__main__":
+    template = sys.argv[1] if len(sys.argv) > 1 else "%s"
+    print(" ".join([template % filename for filename in requirements_files()]))

ci/spec.py

@@ -6,11 +6,15 @@
 
 
 def spec(package: str, pyproject_toml_path: Path) -> str:
-    """Return the spec for the package from the pyproject.toml file."""
+    """Return the spec for the package from the tools section in the pyproject.toml file.
+
+    Returns an empty string if no spec can be found for the specified package.
+    """
     with pyproject_toml_path.open("rb") as pyproject_toml_file:
         pyproject_toml = tomllib.load(pyproject_toml_file)
     tools = pyproject_toml["project"]["optional-dependencies"]["tools"]
-    return [spec for spec in tools if spec.split("==")[0] == package][0]
+    package_specs = [spec for spec in tools if spec.split("==")[0] == package]
+    return package_specs[0] if package_specs else ""
 
 
 if __name__ == "__main__":

ci/unittest-base.sh

@@ -1,8 +1,13 @@
 #!/bin/bash
 
-# Get the dir of this script so the vbase.sh script that is in the same dir as this script can be sourced:
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-source $SCRIPT_DIR/base.sh
+source base.sh
 
 # Turn on development mode, see https://docs.python.org/3/library/devmode.html
 export PYTHONDEVMODE=1
+
+run_coverage() {
+    run coverage run -m unittest --quiet
+    run coverage report --fail-under=0
+    run coverage html --fail-under=0
+    run coverage xml  # Fail if coverage is too low, but only after the text and HTML reports have been generated
+}

components/api_server/ci/pip-compile.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+PATH="$PATH:../../ci"
+source pip-base.sh
 
-# Update the compiled requirements files
-run pip-compile --output-file requirements/requirements.txt pyproject.toml
-run pip-compile --extra dev --output-file requirements/requirements-dev.txt pyproject.toml
+run_pip_compile

components/api_server/ci/pip-install.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+PATH="$PATH:../../ci"
+source pip-base.sh
 
-# Install the requirements
-run pip install --ignore-installed --quiet --use-pep517 -r requirements/requirements-dev.txt
-run pip install --ignore-installed --quiet --use-pep517 -r requirements/requirements-internal.txt
+run_pip_install -r requirements/requirements-dev.txt
+run_pip_install -r requirements/requirements-internal.txt

components/api_server/ci/quality.sh

@@ -1,31 +1,6 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+PATH="$PATH:../../ci"
+source quality-base.sh
 
-# Ruff
-run pipx run `spec ruff` check .
-run pipx run `spec ruff` format --check .
-
-# Fixit
-run pipx run `spec fixit` lint src tests
-
-# Mypy
-run pipx run `spec mypy` --python-executable=$(which python) src
-
-# pip-audit
-run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements.txt -r requirements/requirements-dev.txt
-
-# Safety
-# Vulnerability ID: 67599
-# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
-# highest version number, even if the user had intended to obtain a private package from a private index. This only
-# affects use of the --extra-index-url option, and exploitation requires that the...
-# CVE-2018-20225
-# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
-run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements.txt -r requirements/requirements-dev.txt
-
-# Bandit
-run pipx run `spec bandit` --quiet --recursive src/
-
-# Vulture
-run pipx run `spec vulture` --min-confidence 0 src/ tests/ .vulture_ignore_list.py
+check_python_quality