From cd867403f5f0e71002870514e42ff0e4ddc86cc2 Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:22:47 +0100 Subject: [PATCH] OIDC-based releases (#904) * OIDC-based releases * OIDC-based test releases * Added CODEOWNERS file for changes to GitHub workflows --- .github/CODEOWNERS | 2 ++ .github/workflows/EVENT_release.yml | 46 ++++++++++++++--------------- .github/workflows/JOB_e2e.yml | 9 +++++- .github/workflows/JOB_tests.yml | 2 +- docs/release_process.md | 33 +++++++++++++-------- 5 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..5008a658a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Notify code owners about changes to GitHub actions +.github/ @JBWilkie \ No newline at end of file diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index fcc6e0019..19c1e163f 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -48,13 +48,18 @@ jobs: uses: ./.github/workflows/JOB_tests.yml run_e2e: - needs: run_tests uses: ./.github/workflows/JOB_e2e.yml + secrets: + E2E_API_KEY: ${{ secrets.E2E_API_KEY }} + E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }} + E2E_TEAM: ${{ secrets.E2E_TEAM }} release: needs: [run_tests, run_e2e] if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest + permissions: + id-token: write steps: - uses: actions/checkout@v4 with: @@ -67,20 +72,22 @@ jobs: - run: pip install pip --upgrade - name: Setup Poetry - uses: abatilo/actions-poetry@v2 + uses: abatilo/actions-poetry@c31426b23a8080795905ec73c9e458a2447cb2f2 with: poetry-version: "1.3.1" - - name: Publish on pypi.org - env: - POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} - POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - poetry publish --build + + - name: Build package + run: poetry build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 test_release: needs: [run_tests, run_e2e] if: startsWith(github.ref, 'refs/tags/test-') runs-on: ubuntu-latest + permissions: + id-token: write steps: - uses: actions/checkout@v4 with: @@ -92,26 +99,17 @@ jobs: python-version: "3.9" - run: pip install pip --upgrade - name: Setup Poetry - uses: abatilo/actions-poetry@v2 + uses: abatilo/actions-poetry@c31426b23a8080795905ec73c9e458a2447cb2f2 with: poetry-version: "1.3.1" - - name: Check secrets are set + - name: Build package run: | - if [[ -z "${{ secrets.TEST_PYPI_USERNAME }}" || -z "${{ secrets.TEST_PYPI_PASSWORD }}" ]]; then - echo "TEST_PYPI_USERNAME and TEST_PYPI_PASSWORD must be set" - exit 1 - fi - - name: Publish on test.pypi.org - env: - POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} - POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} - run: | - python ./deploy/nightly_package_setup.py - poetry config repositories.test-pypi https://test.pypi.org/legacy/ - poetry config http-basic.test-pypi ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }} + poetry build - poetry publish --build --repository test-pypi - python ./deploy/revert_nightly_setup.py + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 + with: + repository-url: https://test.pypi.org/legacy/ notify_release: needs: [release] diff --git a/.github/workflows/JOB_e2e.yml b/.github/workflows/JOB_e2e.yml index 96ba76d0b..ccd58b548 100644 --- a/.github/workflows/JOB_e2e.yml +++ b/.github/workflows/JOB_e2e.yml @@ -8,6 +8,13 @@ on: paths: - "e2e_tests/**" workflow_call: + secrets: + E2E_API_KEY: + required: true + E2E_ENVIRONMENT: + required: true + E2E_TEAM: + required: true jobs: e2e: name: End to End Testing @@ -26,7 +33,7 @@ jobs: - name: Upgrade pip run: python -m pip install --upgrade pip - name: Setup Poetry - uses: abatilo/actions-poetry@v2 + uses: abatilo/actions-poetry@c31426b23a8080795905ec73c9e458a2447cb2f2 with: poetry-version: ${{ matrix.poetry-version }} - name: Install dependencies diff --git a/.github/workflows/JOB_tests.yml b/.github/workflows/JOB_tests.yml index bb6d122c1..2916f2f8c 100644 --- a/.github/workflows/JOB_tests.yml +++ b/.github/workflows/JOB_tests.yml @@ -34,7 +34,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies with retry - uses: nick-invision/retry@v3 + uses: nick-invision/retry@7152eba30c6575329ac0576536151aca5a72780e with: timeout_minutes: 10 max_attempts: 3 diff --git a/docs/release_process.md b/docs/release_process.md index 9f70806b3..9e1237883 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -10,34 +10,42 @@ How to create releases of `darwin-py`. - [Make a non-standard release](#make-a-non-standard-release) - [Contact](#contact) - ## Introduction -`darwin-py` is released on [pypi.org](https://pypi.org/project/darwin-py/). It uses [Poetry's]([https://poetry-python](https://python-poetry.org/)) "Mason" engine, which builds both [SDIST](https://docs.python.org/3/distutils/sourcedist.html) and [Wheel](https://pythonwheels.com/) artefacts, and uploads them to PyPi using `setuptools`. The build system is [`PEP-518`](https://peps.python.org/pep-0518/) compliant, which is required to install with `pip` versions 25 and over. +`darwin-py` is released on [pypi.org](https://pypi.org/project/darwin-py/). + +It uses [Poetry's](https://python-poetry.org/) "Mason" engine, which builds both [SDIST](https://docs.python.org/3/distutils/sourcedist.html) and [Wheel](https://pythonwheels.com/) artifacts. The package is then uploaded to PyPI using GitHub Actions with OIDC-based authentication. + +The build system is [`PEP-518`](https://peps.python.org/pep-0518/) compliant, which is required to install with `pip` versions 25 and over. -These instructions are only applicable to those with permissions to release on the repository, within the V7 organisation, not community contributors. You can request we make a new release with contributions you have made by emailing [info@v7labs.com](mailto:info@v7labs.com). +These instructions are only applicable to those with permissions to release on the repository, within the V7 organisation, not community contributors. + +You can request we make a new release with contributions you have made by emailing [info@v7labs.com](mailto:info@v7labs.com). ## Make a standard release -These are the steps for releasing a passing `master` branch. If you are releasing a hotfix or other selective build, you need to follow the instructions for a [hotfix release](#make-a-hotfix-release). +These are the steps for releasing a passing `master` branch. + +If you are releasing a hotfix or other selective build, you need to follow the instructions for a [non-standard release](#make-a-non-standard-release). ### Steps **Pre-release** -1. Ensure all tickets to be included are QA'd and to be included in a release. Speak to PM's and code owners if unsure. +1. Ensure all tickets to be included are QA'd and to be included in a release. Speak to PM's and code owners if unsure. 2. Once passed on QA, merge all PRs to be included into `master` **Release** -This is the process for making a simple semver _patch_ release. If you need to make a _minor_ or _major_ version release, follow the instructions here [Making a non-standard release](#make-a-non-standard-release) +This is the process for making a simple semver _patch_ release. If you need to make a _minor_ or _major_ version release, follow the instructions for [Making a non-standard release](#make-a-non-standard-release) 1. Run the script `deploy/create_release.sh` - follow the prompts, and the script will: * Increment the version in all places it exists * Commit this, and push these changes, along with a tag for the version * Check the script didn't throw any errors, if it didn't, the script will prompt you to look at the Actions dialog, to see when the tests and quality checks all pass. **NB: If checks fail, the release will fail** -2. A draft release will be created in Github, you can release this with the UI, or the `gh` CLI +2. A draft release will be created in Github. You can release this with the UI or the `gh` CLI. +3. Once the release is published, the GitHub Actions workflow will automatically build and publish the package to PyPI using OIDC-based authentication. **Done!** @@ -45,16 +53,17 @@ This is the process for making a simple semver _patch_ release. If you need to ## Make a non-standard release -Making a hotfix release is _broadly_ the same as the process for [making a normal release](#make-a-standard-release), but with a few more steps: +Making a non-standard release (like a hotfix) is _broadly_ the same as the process for [making a normal release](#make-a-standard-release), but with a few more steps: -1. Run the script `python deploy/increase_version.py` and use `--patch`, `--minor`, or `--major` to set the version change you want to make. If you need to make a more complex version change (changing more than one aspect of the version), you can pass `--new-version` followed by a valid semver version. +1. Run the script `python deploy/increase_version.py` and use `--patch`, `--minor`, or `--major` to set the version change you want to make. If you need to make a more complex version change (changing more than one aspect of the version), you can pass `--new-version` followed by a valid semver version. 2. Commit and push the changes to master -3. Add a tag using `git tag origin v0.0.0`, substituiting the right version number (**NB: the preceeding 'v' is necessary for release**) +3. Add a tag using `git tag v0.0.0`, substituting the right version number (**NB: the preceding 'v' is necessary for release**) 4. Push the tag `git push origin v0.0.0`, again substituting the right version number -5. A draft release will be created in Github, you can release this with the UI, or the `gh` CLI +5. A draft release will be created in Github. You can release this with the UI or the `gh` CLI. +6. Once the release is published, the GitHub Actions workflow will automatically build and publish the package to PyPI using OIDC-based authentication. **Make sure you update the tickets to 'Done'** ## Contact -Any issues, reach out in the appropriate channels on slack, or contact owen@v7labs.com - Slack is faster. +Any issues, reach out in the appropriate channels on slack, or contact john@v7labs.com - Slack is faster. \ No newline at end of file