conbench_upload.yml

# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Upload Benchmark Results
on:
  workflow_dispatch:
    inputs:
      run_id:
        description: 'workflow run id to use the artifacts from'
        required: true
  workflow_run:
    workflows: ["Ubuntu Benchmark"]
    types:
      - completed

permissions:
  contents: read
  actions: read
  statuses: write

jobs:
  upload:
    runs-on: ubuntu-latest
    # Disabling workflow due to potential security sev
    if: false
    steps:

    - name: 'Download artifacts'
      id: 'download'
      uses: actions/github-script@v6
      with:
        script: |
          const run_id = "${{ github.event.workflow_run.id || inputs.run_id }}";
          let benchmark_run = await github.rest.actions.getWorkflowRun({
            owner: context.repo.owner,
            repo: context.repo.repo,
            run_id: run_id,
          });

          let artifacts = await github.rest.actions.listWorkflowRunArtifacts({
            owner: context.repo.owner,
            repo: context.repo.repo,
            run_id: run_id,
          });

          let result_artifact = artifacts.data.artifacts.filter((artifact) => {
            return artifact.name == "benchmark-results"
          })[0];

          let pr_artifact = artifacts.data.artifacts.filter((artifact) => {
            return artifact.name == "pr_number"
          })[0];

          let result_download = await github.rest.actions.downloadArtifact({
            owner: context.repo.owner,
            repo: context.repo.repo,
            artifact_id: result_artifact.id,
            archive_format: 'zip',
          });

          let pr_download = await github.rest.actions.downloadArtifact({
            owner: context.repo.owner,
            repo: context.repo.repo,
            artifact_id: pr_artifact.id,
            archive_format: 'zip',
          });

          var fs = require('fs');
          fs.writeFileSync('${{github.workspace}}/benchmark-results.zip', Buffer.from(result_download.data));
          fs.writeFileSync('${{github.workspace}}/pr_number.zip', Buffer.from(pr_download.data));

          core.setOutput('contender_sha', benchmark_run.data.head_sha);

          if (benchmark_run.data.event == 'push') {
            core.setOutput('merge_commit_message', benchmark_run.data.head_commit.message);
          } else {
            core.setOutput('merge_commit_message', '');
          }

    - name: Extract artifact
      id: extract
      run: |
        unzip benchmark-results.zip -d benchmark-results
        unzip pr_number.zip

        pr_number=$(grep -ox '[[:digit:]]*' pr_number.txt | head -1)

        if [ "$pr_number" -ge 0 ]; then
          echo "Found PR number: $pr_number"
        else
          echo '::error :: Malformed input, aborting!'
          exit 1
        fi

        echo "pr_number=$pr_number" >> $GITHUB_OUTPUT

    - uses: actions/checkout@v3
      with:
        path: velox
    - uses: actions/setup-python@v4
      with:
        python-version: '3.8'
        cache: 'pip'
        cache-dependency-path: "velox/scripts/*"

    - name: "Install dependencies"
      run: python -m pip install -r velox/scripts/benchmark-requirements.txt

    - name: "Upload results"
      env:
        CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
        CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-8-core"
        CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}"
        CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}"
        CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}"
        CONBENCH_PROJECT_COMMIT: "${{ steps.download.outputs.contender_sha }}"
      run: |
        if [ "${{ steps.extract.outputs.pr_number }}" -gt 0]; then
          export CONBENCH_PROJECT_PR_NUMBER="${{ steps.extract.outputs.pr_number }}"
        fi

        ./velox/scripts/benchmark-runner.py upload \
          --run_id "GHA-${{ github.run_id }}-${{ github.run_attempt }}" \
          --pr_number "${{ steps.extract.outputs.pr_number }}" \
          --sha "${{ steps.download.outputs.contender_sha }}" \
          --output_dir "${{ github.workspace }}/benchmark-results/contender/"

    - name: "Check the status of the upload"
      # Status functions like failure() only work in `if:`
      if: failure()
      id: status
      run: echo "failed=true" >> $GITHUB_OUTPUT

    - name: "Create a GitHub Status on the contender commit (whether the upload was successful)"
      uses: actions/github-script@v6
      if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
      with:
        script: |
          let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'
          let state = 'success'
          let description = 'Result upload succeeded!'

          if(${{ steps.status.outputs.failed || false }}) {
            state = 'failure'
            description = 'Result upload failed!'
          }

          github.rest.repos.createCommitStatus({
            owner: context.repo.owner,
            repo: context.repo.repo,
            sha: '${{ steps.download.outputs.contender_sha }}',
            state: state,
            target_url: url,
            description: description,
            context: 'Benchmark Result Upload'
          })

    - name: Create a GitHub Check benchmark report on the contender comment, and if merge-commit, a comment on the merged PR
      env:
        CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
        GITHUB_APP_ID: "${{ secrets.GH_APP_ID }}"
        GITHUB_APP_PRIVATE_KEY: "${{ secrets.GH_APP_PRIVATE_KEY }}"
      run: |
        ./velox/scripts/benchmark-alert.py \
          --contender-sha "${{ steps.download.outputs.contender_sha }}" \
          --merge-commit-message "${{ steps.download.outputs.merge_commit_message }}" \
          --z-score-threshold 50