From b294e0ee5eebeb42324320fe25159fa7cf5cdae8 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 14:46:14 +0100 Subject: [PATCH 01/11] CICD: Release --- .github/workflows/EVENT_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index 75a685289..b47c33d97 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -70,6 +70,8 @@ jobs: run: | poetry install --no-interaction --no-root --all-extras -vvv poetry build + ls -la + ls -la dist - name: Add build to release uses: actions/upload-release-asset@v1 From 3527ebd2090cb703dfff119dd62dfb574bdd27ef Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 15:08:23 +0100 Subject: [PATCH 02/11] CICD: Correct working directory --- .github/workflows/EVENT_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index b47c33d97..da058348e 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -77,7 +77,7 @@ jobs: uses: actions/upload-release-asset@v1 with: upload_url: ${{ github.event.release.upload_url }} - asset_path: ./dist/* + asset_path: dist asset_name: ${{ env.release_id }}.tar.gz asset_content_type: application/gzip From 8a1cbafa44116be0cdd924438049024e7a5883b0 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 16:36:19 +0100 Subject: [PATCH 03/11] CICD: Release confirmation and E2E test slack notif removed --- .github/workflows/EVENT_release.yml | 17 +++++++++-------- .github/workflows/JOB_e2e.yml | 14 +------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index da058348e..0ff93d52c 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -73,13 +73,14 @@ jobs: ls -la ls -la dist - - name: Add build to release - uses: actions/upload-release-asset@v1 - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: dist - asset_name: ${{ env.release_id }}.tar.gz - asset_content_type: application/gzip + # Ticketed to be added later + # - name: Add build to release + # uses: actions/upload-release-asset@v1 + # with: + # upload_url: ${{ github.event.release.upload_url }} + # asset_path: dist + # asset_name: ${{ env.release_id }}.tar.gz + # asset_content_type: application/gzip release: needs: [run_tests, build] @@ -91,7 +92,7 @@ jobs: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - # poetry publish + poetry publish echo "Publishing to pypi.org - Emulated" test_release: diff --git a/.github/workflows/JOB_e2e.yml b/.github/workflows/JOB_e2e.yml index 41ff46cda..b3e231841 100644 --- a/.github/workflows/JOB_e2e.yml +++ b/.github/workflows/JOB_e2e.yml @@ -58,17 +58,5 @@ jobs: :link: - https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }} - :warning: ${{ github.workflow }} failed + :warning: ${{ github.workflow }} failed. - - name: Send Slack Notification - run: | - PAYLOAD=$(cat < Date: Wed, 11 Oct 2023 16:56:21 +0100 Subject: [PATCH 04/11] CICD: refactor to remove unnecessary steps. --- .github/workflows/EVENT_release.yml | 42 ++++++++++++----------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index 0ff93d52c..577cc49b7 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -47,8 +47,9 @@ jobs: needs: validate_tag uses: ./.github/workflows/JOB_tests.yml - build: - needs: validate_tag + release: + needs: [run_tests] + if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -65,28 +66,6 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: "1.3.1" - - - name: Install dependencies - run: | - poetry install --no-interaction --no-root --all-extras -vvv - poetry build - ls -la - ls -la dist - - # Ticketed to be added later - # - name: Add build to release - # uses: actions/upload-release-asset@v1 - # with: - # upload_url: ${{ github.event.release.upload_url }} - # asset_path: dist - # asset_name: ${{ env.release_id }}.tar.gz - # asset_content_type: application/gzip - - release: - needs: [run_tests, build] - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - steps: - name: Publish on pypi.org env: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} @@ -96,10 +75,23 @@ jobs: echo "Publishing to pypi.org - Emulated" test_release: - needs: [run_tests, build] + needs: [run_tests] if: startsWith(github.ref, 'refs/tags/test-') runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - run: pip install pip --upgrade + - name: Setup Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: "1.3.1" - name: Publish on test.pypi.org env: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} From 40e6241d4ab36d34c2f2e53fdb4570aa4ed818fd Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 17:12:25 +0100 Subject: [PATCH 05/11] CICD: Add back in build options --- .github/workflows/EVENT_release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index 577cc49b7..a3890ca90 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -71,8 +71,7 @@ jobs: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - poetry publish - echo "Publishing to pypi.org - Emulated" + poetry publish --build test_release: needs: [run_tests] @@ -97,8 +96,7 @@ jobs: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} run: | - poetry publish - echo "Publishing to test.pypi.org - Emulated" + poetry publish --build # Linear tickets update From b1bc56c381fc03e5de554db098b968cbeabd97ba Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 18:09:07 +0100 Subject: [PATCH 06/11] CICD: Test pypi setup --- .github/workflows/EVENT_release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index a3890ca90..bbfe79396 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -96,7 +96,10 @@ jobs: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} run: | - poetry publish --build + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config http-basic.test-pypi ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }} + + poetry publish --build --repository test-pypi # Linear tickets update From 2129aed8f188677df05ee81f65635a351a2d8a98 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 11 Oct 2023 18:22:35 +0100 Subject: [PATCH 07/11] CICD: Fix slack messaging --- .github/workflows/EVENT_release.yml | 50 +++++++++++++---------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index bbfe79396..bebf9f093 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -106,35 +106,29 @@ jobs: notify_release: needs: [release, test_release] if: always() && contains(needs.*.result, 'success') - runs-on: ubuntu-latest - steps: - - name: Notify Slack - uses: ./.github/workflows/JOB_slack_message.yml - with: - icon: ":rocket:" - at_team: ${{ env.is_scheduled || !env.is_draft }} - secrets: inherit - message: | - :tada: *${{ env.release_tag }}* has been released! - :link: - - https://pypi.org/project/darwin-py - - ${{ github.event.release.html_url }} + uses: ./.github/workflows/JOB_slack_message.yml + secrets: inherit + with: + icon: ":rocket:" + at_team: true + message: | + :tada: *${{ inputs.release_tag || github.event.release.tag_name }}* has been released! + :link: + - https://pypi.org/project/darwin-py + - ${{ github.event.release.html_url }} notify_failed_release: needs: [release, test_release] if: always() && contains(needs.*.result, 'failure') - runs-on: ubuntu-latest - steps: - - name: Notify Slack - uses: ./.github/workflows/JOB_slack_message.yml - with: - secrets: inherit - icon: ":warning:" - at_team: true - message: | - :warning: *${{ env.release_tag }}* has failed to be released! - - *An error occurred performing release, and you may need to release manually.* - - :link: - - ${{ github.event.release.html_url }} + uses: ./.github/workflows/JOB_slack_message.yml + secrets: inherit + with: + icon: ":warning:" + at_team: true + message: | + :warning: *${{ inputs.release_tag || github.event.release.tag_name }}* Release has failed to be released! + + *An error occurred performing release, and you may need to release manually.* + + :link: + - ${{ github.event.release.html_url }} From 5652895f578fdd60774217ef9123cef323e63cc9 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 12 Oct 2023 09:58:13 +0100 Subject: [PATCH 08/11] CICD: Change package name for test publish --- .github/workflows/EVENT_release.yml | 2 ++ deploy/change_package_name.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 deploy/change_package_name.py diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index bebf9f093..1281cda0e 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -96,10 +96,12 @@ jobs: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} run: | + python ./deploy/change_package_name.py darwin-nightly poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config http-basic.test-pypi ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }} poetry publish --build --repository test-pypi + python ./deploy/change_package_name.py darwin-py # Linear tickets update diff --git a/deploy/change_package_name.py b/deploy/change_package_name.py new file mode 100644 index 000000000..e24785952 --- /dev/null +++ b/deploy/change_package_name.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Require a single string argument, the new package name +# Example: python change_package_name.py com.example.newname + +from argparse import ArgumentParser + +parser = ArgumentParser(description="Change package name in pyproject.toml") +parser.add_argument("new_package_name", type=str, help="New package name") + + +def main() -> None: + args = parser.parse_args() + new_package_name = args.new_package_name + + with open("pyproject.toml", "r") as f: + lines = f.readlines() + + with open("pyproject.toml", "w") as f: + for line in lines: + if line.startswith("name ="): + line = f'name = "{new_package_name}"\n' + f.write(line) + + print(f"Changed package name to {new_package_name} in pyproject.toml") + + +if __name__ == "__main__": + main() From e6d7444e4a9a897cbe9d2aedcc07638c24eb43e5 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 12 Oct 2023 11:59:54 +0100 Subject: [PATCH 09/11] CICD: Remove typecheck for now --- .github/workflows/EVENT_pull_request.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/EVENT_pull_request.yml b/.github/workflows/EVENT_pull_request.yml index 4c4c1df46..947e975a6 100644 --- a/.github/workflows/EVENT_pull_request.yml +++ b/.github/workflows/EVENT_pull_request.yml @@ -31,12 +31,12 @@ jobs: with: files: ${{ needs.get_changed_files.outputs.python_changed_files }} - typecheck: - name: Analyse types in python - needs: get_changed_files - uses: ./.github/workflows/JOB_typecheck.yml - with: - files: ${{ needs.get_changed_files.outputs.python_changed_files }} + # typecheck: + # name: Analyse types in python + # needs: get_changed_files + # uses: ./.github/workflows/JOB_typecheck.yml + # with: + # files: ${{ needs.get_changed_files.outputs.python_changed_files }} run_tests: needs: [format, lint, typecheck] From 9a8a5d982d46ca1e0982ce2a03a45d2328954222 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 12 Oct 2023 12:00:30 +0100 Subject: [PATCH 10/11] CICD: Update Pr flow --- .github/workflows/EVENT_pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/EVENT_pull_request.yml b/.github/workflows/EVENT_pull_request.yml index 947e975a6..de477c3e6 100644 --- a/.github/workflows/EVENT_pull_request.yml +++ b/.github/workflows/EVENT_pull_request.yml @@ -39,7 +39,7 @@ jobs: # files: ${{ needs.get_changed_files.outputs.python_changed_files }} run_tests: - needs: [format, lint, typecheck] + needs: [format, lint] name: Run tests uses: ./.github/workflows/JOB_tests.yml From e14549a4c7a68deebaa5ac4c77665f0895538559 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 12 Oct 2023 12:41:23 +0100 Subject: [PATCH 11/11] CICD: Nightly build first steps --- .github/workflows/EVENT_release.yml | 18 ++++++++---- deploy/change_package_name.py | 30 -------------------- deploy/nightly_package_setup.py | 44 +++++++++++++++++++++++++++++ deploy/revert_nightly_setup.py | 41 +++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 36 deletions(-) delete mode 100644 deploy/change_package_name.py create mode 100644 deploy/nightly_package_setup.py create mode 100644 deploy/revert_nightly_setup.py diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index 1281cda0e..b35b55c14 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -91,23 +91,29 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: "1.3.1" + - name: Check secrets are set + run: | + if [[ -z "${{ secrets.TEST_PYPI_USERNAME }}" || -z "${{ secrets.TEST_PYPI_PASSWORD }}" ]]; then + echo "TEST_PYPI_USERNAME and TEST_PYPI_PASSWORD must be set" + exit 1 + fi - name: Publish on test.pypi.org env: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} run: | - python ./deploy/change_package_name.py darwin-nightly + python ./deploy/nightly_package_setup.py poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config http-basic.test-pypi ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }} poetry publish --build --repository test-pypi - python ./deploy/change_package_name.py darwin-py + python ./deploy/revert_nightly_setup.py # Linear tickets update notify_release: - needs: [release, test_release] - if: always() && contains(needs.*.result, 'success') + needs: [release] + if: success() uses: ./.github/workflows/JOB_slack_message.yml secrets: inherit with: @@ -120,8 +126,8 @@ jobs: - ${{ github.event.release.html_url }} notify_failed_release: - needs: [release, test_release] - if: always() && contains(needs.*.result, 'failure') + needs: [release] + if: failure() uses: ./.github/workflows/JOB_slack_message.yml secrets: inherit with: diff --git a/deploy/change_package_name.py b/deploy/change_package_name.py deleted file mode 100644 index e24785952..000000000 --- a/deploy/change_package_name.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Require a single string argument, the new package name -# Example: python change_package_name.py com.example.newname - -from argparse import ArgumentParser - -parser = ArgumentParser(description="Change package name in pyproject.toml") -parser.add_argument("new_package_name", type=str, help="New package name") - - -def main() -> None: - args = parser.parse_args() - new_package_name = args.new_package_name - - with open("pyproject.toml", "r") as f: - lines = f.readlines() - - with open("pyproject.toml", "w") as f: - for line in lines: - if line.startswith("name ="): - line = f'name = "{new_package_name}"\n' - f.write(line) - - print(f"Changed package name to {new_package_name} in pyproject.toml") - - -if __name__ == "__main__": - main() diff --git a/deploy/nightly_package_setup.py b/deploy/nightly_package_setup.py new file mode 100644 index 000000000..216251ecd --- /dev/null +++ b/deploy/nightly_package_setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Require a single string argument, the new package name +# Example: python change_package_name.py com.example.newname + +from datetime import datetime +from os import linesep +from pathlib import Path + + +def main() -> None: + epoch_timestring = datetime.now().strftime("%s") + + this_file_path = Path(__file__).parent.resolve() + path_to_pyproject = this_file_path / ".." / "pyproject.toml" + path_to_version = this_file_path / ".." / "version.txt" + + try: + assert path_to_pyproject.exists() + except AssertionError: + print("No pyproject.toml found.") + exit(1) + + lines = path_to_pyproject.read_text().splitlines() + lines_to_write = [] + + for line in lines: + if line.startswith("name ="): + lines_to_write.append('name = "darwin-nightly"\n') + elif line.startswith("version ="): + version = line.split("=")[1].strip() + path_to_version.write_text(version) + lines_to_write.append(f'version = "{epoch_timestring}"\n') + else: + lines_to_write.append(line) + + path_to_pyproject.write_text(linesep.join(lines_to_write)) + + print(f"Set build to a nightly in pyproject.toml - darwin-nightly@{epoch_timestring}") + + +if __name__ == "__main__": + main() diff --git a/deploy/revert_nightly_setup.py b/deploy/revert_nightly_setup.py new file mode 100644 index 000000000..12737b6aa --- /dev/null +++ b/deploy/revert_nightly_setup.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from datetime import datetime +from os import path +from pathlib import Path + + +def main() -> None: + new_package_name = "darwin-py" + + this_file_path = Path(__file__).parent.resolve() + path_to_pyproject = this_file_path / ".." / "pyproject.toml" + path_to_version = this_file_path / ".." / "version.txt" + + try: + assert path_to_pyproject.exists() + assert path_to_version.exists() + except AssertionError: + print("No nightly build in place to revert") + exit(1) + + lines = path_to_pyproject.read_text().splitlines() + new_version = path_to_version.read_text().strip() + + lines_to_write = [] + + for line in lines: + if line.startswith("name ="): + line = f'name = "{new_package_name}"\n' + if line.startswith("version ="): + line = f'version = {new_version}\n' + lines_to_write.append(line) + + path_to_pyproject.write_text("\n".join(lines_to_write)) + + print(f"Changed package name to {new_package_name}@{new_version} in pyproject.toml") + + +if __name__ == "__main__": + main()