Skip to content

[SCHEMATIC-210] Updates publish.yml for Test PyPi deployments (#1559) #5324

[SCHEMATIC-210] Updates publish.yml for Test PyPi deployments (#1559)

[SCHEMATIC-210] Updates publish.yml for Test PyPi deployments (#1559) #5324

Workflow file for this run

# Built from:
# https://docs.github.com/en/actions/guides/building-and-testing-python
# https://github.com/Sage-Bionetworks/challengeutils/blob/master/.github/workflows/pythonapp.yml
# https://github.com/snok/install-poetry#workflows-and-tips
name: Test schematic
on:
push:
branches: ['main', 'develop']
pull_request:
branches: ['*']
workflow_dispatch: # Allow manually triggering the workflow
concurrency:
# cancel the current running workflow from the same branch, PR when a new workflow is triggered
# when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group
# {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows
# {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag
# {{ github.event.pull_request.number}}: get PR number
# {{ github.sha }}: full commit sha
# credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml
group: >-
${{ github.workflow }}-${{ github.ref_type }}-
${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
env:
POETRY_VERSION: 1.3.0
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10"]
steps:
#----------------------------------------------
# check-out repo and set-up python
#----------------------------------------------
- name: Check out repository
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
#----------------------------------------------
# verify runner environment
#----------------------------------------------
# - name: Print runner environment information
# run: |
# echo "Running on runner: $RUNNER_NAME"
# echo "Runner OS: $RUNNER_OS"
# echo "Runner OS version: $RUNNER_OS_VERSION"
# echo "Runner architecture: $RUNNER_ARCH"
# echo "Total memory: $(free -h)"
# echo "CPU info: $(lscpu)"
#----------------------------------------------
# install & configure poetry
#----------------------------------------------
- name: Install Poetry
run: |
curl -sSL https://install.python-poetry.org \
| python3 - --version ${{ env.POETRY_VERSION }};
poetry config virtualenvs.create true;
poetry config virtualenvs.in-project true;
#----------------------------------------------
# install dependencies and root project
#----------------------------------------------
- name: Install dependencies and root project
run: poetry install --no-interaction --all-extras
#----------------------------------------------
# perform linting
#----------------------------------------------
# Disabled until we agree to turn it on
# - name: Lint with flake8
# run: |
# # stop the build if there are Python syntax errors or undefined names
# flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
# flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
#----------------------------------------------
# check formatting
#----------------------------------------------
- name: Code formatting with black
run: |
# ran only on certain files for now
# add here when checked
poetry run black schematic tests schematic_api --check
#----------------------------------------------
# type checking/enforcement
#----------------------------------------------
- name: Type checking with mypy
run: |
# ran only on certain files for now
# add here when checked
# poetry run mypy --install-types --non-interactive
# add here when enforced
poetry run mypy --disallow-untyped-defs --install-types --non-interactive schematic/schemas/ schematic/configuration/ schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py schematic/visualization schematic/utils/
#----------------------------------------------
# linting
#----------------------------------------------
- name: Lint with pylint
run: |
# ran only on certain files for now
# add here when checked
poetry run pylint schematic/visualization/* schematic/configuration/*.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py schematic/utils/*.py schematic/schemas/*.py
#----------------------------------------------
# run unit test suite
#----------------------------------------------
- name: Run unit tests
env:
SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
run: >
poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov
--cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit;
#----------------------------------------------
# run integration test suite
#----------------------------------------------
- name: Retrieve telemetry access token from IDP
if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
id: retrieve-telemetry-access-token
run: |
response=$(curl --request POST \
--url ${{ vars.TELEMETRY_AUTH_CLIENT_URL }} \
--header 'content-type: application/json' \
--data '{"client_id":"${{ vars.TELEMETRY_AUTH_CLIENT_ID }}","client_secret":"${{ secrets.TELEMETRY_AUTH_CLIENT_SECRET }}","audience":"${{ vars.TELEMETRY_AUTH_AUDIENCE }}","grant_type":"client_credentials"}')
access_token=$(echo $response | jq -r .access_token)
echo "::add-mask::$access_token"
echo "TELEMETRY_ACCESS_TOKEN=$access_token" >> "$GITHUB_OUTPUT"
- name: Run integration tests
if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
env:
SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer ${{ steps.retrieve-telemetry-access-token.outputs.TELEMETRY_ACCESS_TOKEN }}"
DEPLOYMENT_ENVIRONMENT: ${{ vars.DEPLOYMENT_ENVIRONMENT }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
TRACING_EXPORT_FORMAT: ${{ vars.TRACING_EXPORT_FORMAT }}
LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }}
TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }}
LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }}
SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }}
run: >
poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
-m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit
- name: Run integration tests single process
if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
env:
SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer ${{ steps.retrieve-telemetry-access-token.outputs.TELEMETRY_ACCESS_TOKEN }}"
DEPLOYMENT_ENVIRONMENT: ${{ vars.DEPLOYMENT_ENVIRONMENT }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
TRACING_EXPORT_FORMAT: ${{ vars.TRACING_EXPORT_FORMAT }}
LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }}
TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }}
LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }}
run: >
poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
-m "single_process_execution" --reruns 4 --ignore=tests/unit
- name: Upload pytest test results
uses: actions/upload-artifact@v4
with:
name: pytest-results-${{ matrix.python-version }}
path: htmlcov
# Use always() to always run this step to publish test results when there are test failures
if: ${{ always() && contains(fromJSON('["3.10"]'), matrix.python-version) }}
- name: Upload XML coverage report
id: upload_coverage_report
uses: actions/upload-artifact@v4
# Only upload a single python version to pass along to sonarcloud
if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) && always() }}
with:
name: coverage-report
path: coverage.xml
sonarcloud:
needs: [test]
if: ${{ always() && !cancelled()}}
name: SonarCloud
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Check coverage-report artifact existence
id: check_coverage_report
uses: LIT-Protocol/artifact-exists-action@v0
with:
name: "coverage-report"
- name: Download coverage report
uses: actions/download-artifact@v4
if: steps.check_coverage_report.outputs.exists == 'true'
with:
name: coverage-report
- name: Check coverage.xml file existence
id: check_coverage_xml
uses: andstor/file-existence-action@v3
with:
files: "coverage.xml"
# This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
- name: Override Coverage Source Path for Sonar
if: steps.check_coverage_xml.outputs.files_exists == 'true'
run: sed -i "s/<source>\/home\/runner\/work\/schematic\/schematic\/schematic<\/source>/<source>\/github\/workspace\/schematic<\/source>/g" coverage.xml
- name: SonarCloud Scan
uses: SonarSource/sonarcloud-github-action@master
if: ${{ always() }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}