Skip to content

Commit

Permalink
Update quality tools.
Browse files Browse the repository at this point in the history
- Add pyproject-fmt to quality tools.
- Remove safety from quality tools.

Closes #8928.
  • Loading branch information
fniessink committed Jun 14, 2024
1 parent ba8c7ae commit 6167840
Show file tree
Hide file tree
Showing 26 changed files with 545 additions and 251 deletions.
23 changes: 21 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
parallelism: 5
parallelism: 6
steps:
- checkout
- run: |
Expand All @@ -19,7 +19,8 @@ jobs:
1) component=components/notifier;;
2) component=components/api_server;;
3) component=components/shared_code;;
4) component=tests/feature_tests;;
4) component=tests/application_tests;;
5) component=tests/feature_tests;;
esac
cd $component
mkdir -p build
Expand All @@ -36,6 +37,10 @@ jobs:
path: components/api_server/build
- store_artifacts:
path: components/shared_code/build
- store_artifacts:
path: components/application_tests/build
- store_artifacts:
path: components/feature_tests/build

unittest_frontend:
docker:
Expand Down Expand Up @@ -65,6 +70,18 @@ jobs:
ci/unittest.sh
ci/quality.sh
unittest_release:
machine:
image: default
steps:
- checkout
- run: |
cd release
python3 -m venv venv
. venv/bin/activate
ci/pip-install.sh
ci/quality.sh
application_tests:
machine:
image: default
Expand Down Expand Up @@ -119,6 +136,8 @@ workflows:
context: QualityTime
- unittest_docs:
context: QualityTime
- unittest_release:
context: QualityTime
- docker/hadolint:
context: QualityTime
dockerfiles: "components/collector/Dockerfile:components/database/Dockerfile:\
Expand Down
27 changes: 27 additions & 0 deletions .github/workflows/application-tests-quality.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: Application tests quality

on: [push]

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/[email protected]
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dependencies
run: |
cd tests/application_tests
ci/pip-install.sh
- name: Test
run: |
cd tests/application_tests
ci/unittest.sh
- name: Quality
run: |
cd tests/application_tests
ci/quality.sh
22 changes: 22 additions & 0 deletions .github/workflows/release-quality.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: Release script quality

on: [push]

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/[email protected]
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dependencies and run quality checks
run: |
cd release
python -m venv venv
. venv/bin/activate
ci/pip-install.sh
ci/quality.sh
12 changes: 3 additions & 9 deletions components/api_server/ci/quality.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,12 @@ run pipx run `spec fixit` lint src tests
# Mypy
run pipx run `spec mypy` --python-executable=$(which python) src

# Pyproject-fmt
run pipx run `spec pyproject-fmt` --check pyproject.toml

# pip-audit
run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Safety
# Vulnerability ID: 67599
# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
# highest version number, even if the user had intended to obtain a private package from a private index. This only
# affects use of the --extra-index-url option, and exploitation requires that the...
# CVE-2018-20225
# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Bandit
run pipx run `spec bandit` --quiet --recursive src/

Expand Down
150 changes: 84 additions & 66 deletions components/api_server/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,110 +1,128 @@
[project]
name = "api_server"
name = "api-server"
version = "5.13.0"
requires-python = ">=3.12"
classifiers = [
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.12",
]
dependencies = [
"bottle==0.12.25",
"cryptography==42.0.8",
"gevent==24.2.1",
"ldap3==2.9.1",
"lxml[html_clean]==5.2.2",
"lxml[html-clean]==5.2.2",
"pymongo==4.7.3",
"requests==2.32.3"
"requests==2.32.3",
]

[project.optional-dependencies]
dev = [
optional-dependencies.dev = [
"coverage==7.5.3",
"pip==24.0",
"pip-tools==7.4.1", # To add hashes to requirements
"pipx==1.6.0",
"pip-tools==7.4.1", # To add hashes to requirements
"pydantic==2.7.4", # Needed for importing the data model in the tests from the shared code component
"pydantic==2.7.4", # Needed for importing the data model in the tests from the shared code component
"types-cryptography==3.3.23.2",
"types-ldap3==2.9.13.20240205",
"types-requests==2.32.0.20240602",
"unittest-xml-reporting==3.2.0", # Needed to generate JUnit XML output for Sonarcloud.io
"unittest-xml-reporting==3.2.0", # Needed to generate JUnit XML output for Sonarcloud.io
]
tools = [
optional-dependencies.tools = [
"bandit==1.7.9",
"fixit==2.1.0",
"mypy==1.10.0",
"pip-audit==2.7.3",
"pyproject-fmt==2.1.3",
"ruff==0.4.8",
"safety==3.2.3",
"vulture==2.11"
"vulture==2.11",
]

[tool.ruff]
target-version = "py312"
line-length = 120
src = [
"src",
]
lint.select = [
"ALL",
]
lint.ignore = [
"ANN001", # https://docs.astral.sh/ruff/rules/missing-type-function-argument/ - too many untyped arguments atm to turn this rule on
"ANN002", # https://docs.astral.sh/ruff/rules/missing-type-args/ - leads to false positives for super().__init__(*args, **kwargs)
"ANN003", # https://docs.astral.sh/ruff/rules/missing-type-kwargs/ - leads to false positives for super().__init__(*args, **kwargs)
"ANN101", # https://docs.astral.sh/ruff/rules/missing-type-self/ - type checkers can infer the type of `self`, so annotating it is superfluous
"ANN102", # https://docs.astral.sh/ruff/rules/missing-type-cls/ - type checkers can infer the type of `cls`, so annotating it is superfluous
"ANN201", # https://docs.astral.sh/ruff/rules/missing-return-type-undocumented-public-function/ - too many untyped return values atm to turn this rule on
"COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/ - this rule may cause conflicts when used with the ruff formatter
"D107", # https://docs.astral.sh/ruff/rules/undocumented-public-init/ - requiring __init__() methods to have docstrings seems a bit much
"D203", # https://docs.astral.sh/ruff/rules/one-blank-line-before-class/ - prevent warning: `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible. Ignoring `one-blank-line-before-class`
"D213", # https://docs.astral.sh/ruff/rules/multi-line-summary-second-line/ - prevent warning: `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible. Ignoring `multi-line-summary-second-line`
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt - not sure of the value of preventing "boolean traps"
"I001", # https://docs.astral.sh/ruff/rules/unsorted-imports/ - (probably) because ruff is run with pipx it can't differentiate between dependencies and modules
"ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/ - this rule may cause conflicts when used with the ruff formatter
"PD", # https://docs.astral.sh/ruff/rules/#pandas-vet-pd - pandas isn't used
"PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt - pytest isn't used
]
lint.per-file-ignores.".vulture_ignore_list.py" = [
"ALL",
]
lint.per-file-ignores."__init__.py" = [
"D104", # https://docs.astral.sh/ruff/rules/undocumented-public-package/ - don't require doc strings in __init__.py files
"F401", # https://docs.astral.sh/ruff/rules/unused-import/ - routes are imported in __init__.py files to flatten the module hierarchy
]
lint.per-file-ignores."src/model/issue_tracker.py" = [
"BLE001", # https://docs.astral.sh/ruff/rules/blind-except/ - allow for catching blind exception `Exception`
]
lint.per-file-ignores."src/quality_time_server.py" = [
"E402",
"INP001", # https://docs.astral.sh/ruff/rules/implicit-namespace-package/ - false positive because this is the main script
]
lint.per-file-ignores."tests/**/*.py" = [
"ANN201", # https://docs.astral.sh/ruff/rules/missing-return-type-undocumented-public-function/ - don't require test functions to have return types
"S105", # https://docs.astral.sh/ruff/rules/hardcoded-password-string/ - hardcoded passwords in test code are test data
"S106", # https://docs.astral.sh/ruff/rules/hardcoded-password-func-arg/ - hardcoded passwords in test code are test data
]
lint.isort.section-order = [
"future",
"standard-library",
"third-party",
"second-party",
"first-party",
"tests",
"local-folder",
]
lint.isort.sections."second-party" = [
"shared",
"shared_data_model",
]
lint.isort.sections.tests = [
"tests",
]

[tool.pyproject-fmt]
indent = 4
keep_full_version = true # Remove trailing zero's from version specifiers?

[tool.mypy]
ignore_missing_imports = false
incremental = false
warn_redundant_casts = true
warn_return_any = true
warn_unreachable = true
warn_unused_ignores = true
disable_error_code = "valid-type" # mypy does not yet support PEP 695, Type Parameter Syntax. See https://github.com/python/mypy/issues/15238
disable_error_code = "valid-type" # mypy does not yet support PEP 695, Type Parameter Syntax. See https://github.com/python/mypy/issues/15238

[[tool.mypy.overrides]]
module = [
"bottle",
"gevent",
"lxml.html",
"lxml.html.clean"
"lxml.html.clean",
]
ignore_missing_imports = true

[tool.pip-tools]
allow_unsafe = true
generate_hashes = true
quiet = true
strip_extras = false # Needed for lxml[html-clean]
strip_extras = false # Needed for lxml[html-clean]
upgrade = true

[tool.ruff]
target-version = "py312"
line-length = 120
src = ["src"]

[tool.ruff.lint]
select = ["ALL"]
ignore = [
"ANN001", # https://docs.astral.sh/ruff/rules/missing-type-function-argument/ - too many untyped arguments atm to turn this rule on
"ANN002", # https://docs.astral.sh/ruff/rules/missing-type-args/ - leads to false positives for super().__init__(*args, **kwargs)
"ANN003", # https://docs.astral.sh/ruff/rules/missing-type-kwargs/ - leads to false positives for super().__init__(*args, **kwargs)
"ANN101", # https://docs.astral.sh/ruff/rules/missing-type-self/ - type checkers can infer the type of `self`, so annotating it is superfluous
"ANN102", # https://docs.astral.sh/ruff/rules/missing-type-cls/ - type checkers can infer the type of `cls`, so annotating it is superfluous
"ANN201", # https://docs.astral.sh/ruff/rules/missing-return-type-undocumented-public-function/ - too many untyped return values atm to turn this rule on
"COM812", # https://docs.astral.sh/ruff/rules/missing-trailing-comma/ - this rule may cause conflicts when used with the ruff formatter
"D107", # https://docs.astral.sh/ruff/rules/undocumented-public-init/ - requiring __init__() methods to have docstrings seems a bit much
"D203", # https://docs.astral.sh/ruff/rules/one-blank-line-before-class/ - prevent warning: `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible. Ignoring `one-blank-line-before-class`
"D213", # https://docs.astral.sh/ruff/rules/multi-line-summary-second-line/ - prevent warning: `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible. Ignoring `multi-line-summary-second-line`
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt - not sure of the value of preventing "boolean traps"
"I001", # https://docs.astral.sh/ruff/rules/unsorted-imports/ - (probably) because ruff is run with pipx it can't differentiate between dependencies and modules
"ISC001", # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/ - this rule may cause conflicts when used with the ruff formatter
"PD", # https://docs.astral.sh/ruff/rules/#pandas-vet-pd - pandas isn't used
"PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt - pytest isn't used
]

[tool.ruff.lint.isort]
section-order = ["future", "standard-library", "third-party", "second-party", "first-party", "tests", "local-folder"]

[tool.ruff.lint.isort.sections]
"second-party" = ["shared", "shared_data_model"]
"tests" = ["tests"]

[tool.ruff.lint.per-file-ignores]
".vulture_ignore_list.py" = ["ALL"]
"__init__.py" = [
"D104", # https://docs.astral.sh/ruff/rules/undocumented-public-package/ - don't require doc strings in __init__.py files
"F401", # https://docs.astral.sh/ruff/rules/unused-import/ - routes are imported in __init__.py files to flatten the module hierarchy
]
"src/quality_time_server.py" = [
"E402",
"INP001", # https://docs.astral.sh/ruff/rules/implicit-namespace-package/ - false positive because this is the main script
]
"src/model/issue_tracker.py" = [
"BLE001" # https://docs.astral.sh/ruff/rules/blind-except/ - allow for catching blind exception `Exception`
]
"tests/**/*.py" = [
"ANN201", # https://docs.astral.sh/ruff/rules/missing-return-type-undocumented-public-function/ - don't require test functions to have return types
"S105", # https://docs.astral.sh/ruff/rules/hardcoded-password-string/ - hardcoded passwords in test code are test data
"S106", # https://docs.astral.sh/ruff/rules/hardcoded-password-func-arg/ - hardcoded passwords in test code are test data
]
9 changes: 0 additions & 9 deletions components/collector/ci/quality.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ run pipx run `spec mypy` --python-executable=$(which python) src tests
# See https://github.com/aio-libs/aiohttp/issues/6772 for why we ignore the CVE
run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Safety
# Vulnerability ID: 67599
# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
# highest version number, even if the user had intended to obtain a private package from a private index. This only
# affects use of the --extra-index-url option, and exploitation requires that the...
# CVE-2018-20225
# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Bandit
run pipx run `spec bandit` --quiet --recursive src/

Expand Down
1 change: 0 additions & 1 deletion components/collector/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ tools = [
"mypy==1.10.0",
"pip-audit==2.7.3",
"ruff==0.4.8",
"safety==3.2.3",
"vulture==2.11"
]

Expand Down
9 changes: 0 additions & 9 deletions components/notifier/ci/quality.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ run pipx run `spec mypy` --python-executable=$(which python) src
# See https://github.com/aio-libs/aiohttp/issues/6772 for why we ignore the CVE
run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Safety
# Vulnerability ID: 67599
# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
# highest version number, even if the user had intended to obtain a private package from a private index. This only
# affects use of the --extra-index-url option, and exploitation requires that the...
# CVE-2018-20225
# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements.txt -r requirements/requirements-dev.txt

# Bandit
run pipx run `spec bandit` --quiet --recursive src/

Expand Down
1 change: 0 additions & 1 deletion components/notifier/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ tools = [
"mypy==1.10.0",
"pip-audit==2.7.3",
"ruff==0.4.8",
"safety==3.2.3",
"vulture==2.11"
]

Expand Down
9 changes: 0 additions & 9 deletions components/shared_code/ci/quality.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,6 @@ run $PIPX_BIN_DIR/mypy src --python-executable=$(which python)
# pip-audit
run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements-dev.txt

# Safety
# Vulnerability ID: 67599
# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
# highest version number, even if the user had intended to obtain a private package from a private index. This only
# affects use of the --extra-index-url option, and exploitation requires that the...
# CVE-2018-20225
# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements-dev.txt

# Bandit
run pipx run `spec bandit` --quiet --recursive src/

Expand Down
1 change: 0 additions & 1 deletion components/shared_code/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ tools = [
"pip-audit==2.7.3",
"pydantic==2.7.4", # Needed because pipx needs to inject Pydantic into the mpyp venv, see ci/quality.sh
"ruff==0.4.8",
"safety==3.2.3",
"vulture==2.11"
]

Expand Down
Loading

0 comments on commit 6167840

Please sign in to comment.