PR.yml

name: Run linters and tests

on:
  - pull_request
  - merge_group

# Cancel previous runs on the same branch \ PR number if they are still running
# From: https://stackoverflow.com/a/72408109
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true


jobs:
  run-linters:
    name: Run linters
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.9, '3.10', 3.11]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      uses: ./.github/actions/install-deps-and-canopy
      with:
        python-version: ${{ matrix.python-version }}
        install-canopy: false
    - name: Run flake8 Check
      run: poetry run flake8 .
    - name: Run mypy check
      run: poetry run mypy src

  run-tests:
    name: Run tests
    runs-on: ubuntu-latest
    needs: run-linters
    strategy:
      matrix:
        python-version: [3.9, '3.10', 3.11]
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies and canopy
      uses: ./.github/actions/install-deps-and-canopy
      with:
        python-version: ${{ matrix.python-version }}
        install-canopy: true
    - name: Run unit tests
      run: poetry run pytest --html=report.html --self-contained-html tests/unit
    - name: Set Index Name for System / E2E
      if: github.event_name == 'merge_group'
      id: gen_suffix
      run: |
        RAW_SUFFIX="${{ matrix.python-version }}"
        SUFFIX="${RAW_SUFFIX//./-}"
        echo "${SUFFIX}"
        echo "INDEX_NAME_SUFFIX=${SUFFIX}" >> $GITHUB_OUTPUT
    - name: Run system tests
      id: system_tests
      if: github.event_name == 'merge_group'
      env:
        INDEX_NAME: system-${{ steps.gen_suffix.outputs.INDEX_NAME_SUFFIX }}
        PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        OCTOAI_API_KEY: ${{ secrets.OCTOAI_API_KEY }}
        ANYSCALE_API_KEY: ${{ secrets.ANYSCALE_API_KEY }}
        CO_API_KEY: ${{ secrets.CO_API_KEY }}
        AZURE_DEPLOYMENT_NAME: ${{ secrets.AZURE_DEPLOYMENT_NAME }}
        AZURE_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.AZURE_EMBEDDING_DEPLOYMENT_NAME }}
        AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
        AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
      run: |
        run_id=$(uuidgen | tr -d '-' | tr '[:upper:]' '[:lower:]')
        echo "run_id=${run_id}" >> $GITHUB_OUTPUT
        echo "Test Run ID: ${run_id}"
        poetry run pytest -n 3 --dist loadscope --testrunuid $run_id --html=report_system.html --self-contained-html tests/system
    - name: Run e2e tests
      id: e2e_tests
      if: github.event_name == 'merge_group'
      env:
        INDEX_NAME: e2e-${{ steps.gen_suffix.outputs.INDEX_NAME_SUFFIX }}
        PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        OCTOAI_API_KEY: ${{ secrets.OCTOAI_API_KEY }}
        ANYSCALE_API_KEY: ${{ secrets.ANYSCALE_API_KEY }}
        CO_API_KEY: ${{ secrets.CO_API_KEY }}
        CE_LOG_FILENAME: e2e.log
        AZURE_DEPLOYMENT_NAME: ${{ secrets.AZURE_DEPLOYMENT_NAME }}
        AZURE_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.AZURE_EMBEDDING_DEPLOYMENT_NAME }}
        AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
        AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
      run: |
        run_id=$(uuidgen | tr -d '-' | tr '[:upper:]' '[:lower:]')
        echo "run_id=${run_id}" >> $GITHUB_OUTPUT
        echo "Test Run ID: ${run_id}"
        poetry run pytest -n 3 --dist loadscope --testrunuid $run_id --html=report_e2e.html --self-contained-html tests/e2e
    - name: Cleanup indexes
      if: (cancelled() || failure()) && github.event_name == 'merge_group'
      env:
        PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
      run: |
        export PYTHONPATH=.
        poetry run python scripts/cleanup_indexes.py "${{ steps.e2e_tests.outputs.run_id }}"
        poetry run python scripts/cleanup_indexes.py "${{ steps.system_tests.outputs.run_id }}"
    - name: upload pytest report.html
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: pytest-report-py${{ matrix.python-version }}
        path: report*.html
    - name: upload e2e test log
      uses: actions/upload-artifact@v3
      if: failure() && github.event_name == 'merge_group'
      with:
        name: e2e-log-failure-report-py${{ matrix.python-version }}
        path: e2e.log