From 3103a3a9d249a59dfdc44a7c6f113bedc83ef9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Morosi?= Date: Mon, 7 Oct 2024 15:56:48 +0200 Subject: [PATCH] Use the cookiecutter template for e3-aws Ref it/org/operation_support/iaas/projects#84 --- .gitlab-ci.yml | 196 ++++++++++++++++++++++++++++++++++++++-------- pyproject.toml | 4 + tests/__init__.py | 0 tox.ini | 36 +++++---- 4 files changed, 188 insertions(+), 48 deletions(-) create mode 100644 tests/__init__.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c30aa42c..4b6114ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,39 +1,88 @@ variables: - GITLAB_REMOTE: "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/" + GITLAB_REMOTE: + description: "The remote gitlab URL used." + value: "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/" + LATEST_PYTHON: + description: "The latest python version used to test this project." + options: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + value: "3.12" stages: - - check - - test + - checks + - tests linux + - tests windows default: + services: + - run_as_root:false interruptible: true + +# Common + +.tox-common: + before_script: + - python -m pip install --force tox + script: + # Should be quoted using \' to deal with ':' in the command + - 'echo "Tox run environment: ${CI_TOX_ENV:=py${PYTHON_VERSION:0:1}${PYTHON_VERSION:2:2}-cov-xdist}"' + - python -m tox --colored yes -e ${CI_TOX_ENV} + +### Linux jobs ### + +.linux-image: services: - image:all-pythons before_script: - - git config --global --add - url."${GITLAB_REMOTE}/it/black.git".insteadOf - https://github.com/ambv/black - - git config --global --add - url."${GITLAB_REMOTE}/it/flake8.git".insteadOf - https://github.com/pycqa/flake8 - - source /it/activate-${PYTHON_VERSION} - - python -m pip install --force tox + - source /it/activate-py${PYTHON_VERSION:0:1}${PYTHON_VERSION:2:2} + - python -m pip install -U pip + +.linux-common: + extends: + - .linux-image + - .tox-common + before_script: + - !reference [.linux-image, before_script] + - !reference [.tox-common, before_script] + +# Stage: Checks check: - stage: check - script: + stage: checks + extends: .linux-common + needs: [] + before_script: + - !reference [.linux-common, before_script] + - git config --global --add + url."${GITLAB_REMOTE}/it/black.git".insteadOf + https://github.com/ambv/black + - git config --global --add + url."${GITLAB_REMOTE}/it/flake8.git".insteadOf + https://github.com/pycqa/flake8 - python -m pip install pre-commit - pre-commit install + script: - pre-commit run -a --show-diff-on-failure - - python -m tox --colored yes -e check + - !reference [.linux-common, script] variables: - PYTHON_VERSION: py311 + PYTHON_VERSION: ${LATEST_PYTHON} + CI_TOX_ENV: check + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" -.test-py-common: - stage: test - script: - - python -m tox --colored yes -e ${PYTHON_VERSION}-cov +.test-linux: + stage: tests linux + extends: .linux-common + services: + - !reference [.linux-common, services] + - cpu:4 coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' + parallel: + matrix: + - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"] artifacts: when: always paths: @@ -44,23 +93,102 @@ check: path: coverage.xml junit: pytest-report.xml -test-py39: - extends: .test-py-common - variables: - PYTHON_VERSION: py39 +linux python: + extends: .test-linux + needs: ["check"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" -test-py310: - extends: .test-py-common - variables: - PYTHON_VERSION: py310 +# A job triggered by 'run linux tests'. This jobs will run without waiting any others +# jobs. +linux python (always): + extends: .test-linux + needs: [] + rules: + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_MERGE_REQUEST_ID && $CI_PROJECT_NAME == "e3-core" && $ALWAYS_LINUX_TESTS == "y" -test-py311: - extends: .test-py-common +# A manual job to run Linux tests even if "check" job has failed +run linux tests: + stage: tests linux + needs: [] + trigger: + include: .gitlab-ci.yml + strategy: depend variables: - PYTHON_VERSION: py311 + ALWAYS_LINUX_TESTS: "y" + ALWAYS_WINDOWS_TESTS: "n" + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + allow_failure: true + # Contrary to what the documentation might suggest, manual_confirmation + # is not currently usable with our gitlab. + # However, when it is, adding a manual confirmation to warn the user that + # this job should only be used when previous steps have failed seems + # useful. Something like: + # + # manual_confirmation: |- + # Are you sure you want to run Linux tests? + # + # This is only useful if the previous stages have failed and you still want to run the tests. -test-py312: - extends: .test-py-common - variables: - PYTHON_VERSION: py312 +### Windows jobs ### +.windows-image: + services: + - image:e3-windows-core-2022 + - platform:x86_64-windows-2022 + - cpu:2 + - mem:4 + before_script: + - source /it/activate-python ${PYTHON_VERSION} + - mkdir -p "C:/tmp/Trash" + - python -m pip install -U pip + +.windows-common: + extends: + - .windows-image + - .tox-common + before_script: + - !reference [.windows-image, before_script] + - !reference [.tox-common, before_script] + +.test-windows: + stage: tests windows + extends: .windows-common + parallel: + matrix: + - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"] + +# A job tiggered by 'Run Windows tests'. This jobs will run without waiting any others +# jobs. +windows python (always): + extends: .test-windows + needs: [] + rules: + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_MERGE_REQUEST_ID && $CI_PROJECT_NAME == "e3-aws" && $ALWAYS_WINDOWS_TESTS == "y" + +# A manual job to run Windows tests even if previous jobs have failed +run windows tests: + stage: tests windows + needs: [] + trigger: + include: .gitlab-ci.yml + strategy: depend + variables: + ALWAYS_LINUX_TESTS: "n" + ALWAYS_WINDOWS_TESTS: "y" + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + allow_failure: true + # Contrary to what the documentation might suggest, manual_confirmation + # is not currently usable with our gitlab. + # However, when it is, adding a manual confirmation to warn the user that + # this job should only be used when previous steps have failed seems + # useful. Something like: + # + # manual_confirmation: |- + # Are you sure you want to run Windows tests? + # + # This is only useful if the previous stages have failed and you still want to run the tests. diff --git a/pyproject.toml b/pyproject.toml index 653233ae..0d8fe0de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ s3-boto3 = "e3.aws.handler.s3:S3Handler" test = [ "awscli", "pytest", + "pytest-html", "mock", "requests_mock", "httpretty", @@ -38,6 +39,9 @@ test = [ check = [ "mypy==1.8.0", + "pytest", + "flask", + "moto[sts, dynamodb]", "bandit", "pip-audit", "types-colorama", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tox.ini b/tox.ini index 26266b69..5ef7fe0a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,36 +1,44 @@ [tox] -envlist = py311-cov,check -isolated_build = True +envlist = + py311-cov-xdist, + check +isolated_build=True [testenv] +passenv = CI,GITHUB_*,CODECOV_* + deps = + xdist: pytest-xdist[psutil] cov: pytest-cov codecov: codecov -extras = - test - -passenv = CI,GITHUB_*,CODECOV_* +extras = test -# Run testsuite with coverage when '-cov' is in the env name +# Run testsuite with coverage when '-cov' and with multi-threading when '-xdist' +# is in the env name commands= - pytest --ignore=build -vv \ + pytest -vv --ignore=build -v --html=pytest-report.html \ + --junit-xml=pytest-report.xml --self-contained-html \ + xdist: -n auto \ cov: --e3-cov-rewrite {envsitepackagesdir} src \ cov: --cov {envsitepackagesdir}/e3/aws --cov-report= --cov-fail-under=0 \ [] codecov: codecov [testenv:check] -# Run mypy, pip audit, and bandit extras = - config - check + config + check commands = - bandit -r src/e3 -ll -ii -s B102,B108,B202,B301,B303,B506 - mypy -- {toxinidir}/src +# Run bandit checks. Accept yaml.load(), pickle, and exec since this +# is needed by e3. There is also e3.env.tmp_dir that returns the TMPDIR +# environment variable. Don't check for that. +# Ignore B324 that is no longer similar to B303 since Python3.9. + bandit -r {toxinidir}/src -ll -ii -s B102,B108,B301,B506,B303,B324,B202 + mypy {toxinidir}/src {toxinidir}/tests [flake8] exclude = .git,__pycache__,build,dist,.tox -ignore = B905, C901, E203, E266, E501, W503,D100,D101,D102,D102,D103,D104,D105,D106,D107,D203,D403,D213 +ignore = B301,C901,E203,E266,E501,W503,D100,D101,D102,D102,D103,D104,D105,D106,D107,D203,D403,D213,E704,B905 # line length is intentionally set to 80 here because black uses Bugbear # See https://github.com/psf/black/blob/master/README.md#line-length for more details max-line-length = 80