Regression Detector (trusted) #368
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression Detector (trusted) | |
| on: | |
| workflow_run: | |
| workflows: ["Regression Detector (untrusted)"] | |
| types: | |
| - completed | |
| jobs: | |
| compute-metadata: | |
| name: Compute metadata for regression experiments | |
| runs-on: ubuntu-22.04 | |
| if: > | |
| github.event.workflow_run.event == 'pull_request' && | |
| github.event.workflow_run.conclusion == 'success' | |
| outputs: | |
| cpus: ${{ steps.system.outputs.CPUS }} | |
| memory: ${{ steps.system.outputs.MEMORY }} | |
| comparison-sha: ${{ steps.metadata.outputs.COMPARISON_SHA }} | |
| comparison-tag: ${{ steps.metadata.outputs.COMPARISON_TAG }} | |
| baseline-sha: ${{ steps.metadata.outputs.BASELINE_SHA }} | |
| baseline-tag: ${{ steps.metadata.outputs.BASELINE_TAG }} | |
| head-sha: ${{ steps.metadata.outputs.HEAD_SHA }} | |
| checkout-sha: ${{ steps.metadata.outputs.CHECKOUT_SHA }} | |
| github-event-number: ${{ steps.metadata.outputs.GITHUB_EVENT_NUMBER }} | |
| replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} | |
| warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} | |
| total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} | |
| p-value: ${{ steps.experimental-meta.outputs.P_VALUE }} | |
| smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} | |
| lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} | |
| smp-wait-timeout-minutes: ${{ steps.experimental-meta.outputs.SMP_WAIT_TIMEOUT }} | |
| steps: | |
| - name: Setup experimental metadata | |
| id: experimental-meta | |
| run: | | |
| export WARMUP_SECONDS="45" | |
| export REPLICAS="10" | |
| export TOTAL_SAMPLES="600" | |
| export P_VALUE="0.1" | |
| export SMP_CRATE_VERSION="0.8.0" | |
| export SMP_WAIT_TIMEOUT="55" | |
| export LADING_VERSION="0.16.0" | |
| echo "warmup seconds: ${WARMUP_SECONDS}" | |
| echo "replicas: ${REPLICAS}" | |
| echo "total samples: ${TOTAL_SAMPLES}" | |
| echo "regression p-value: ${P_VALUE}" | |
| echo "smp crate version: ${SMP_CRATE_VERSION}" | |
| echo "lading version: ${LADING_VERSION}" | |
| echo "smp wait timeout: ${SMP_WAIT_TIMEOUT}" | |
| echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT | |
| echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT | |
| echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT | |
| echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT | |
| echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT | |
| echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT | |
| echo "SMP_WAIT_TIMEOUT=${SMP_WAIT_TIMEOUT}" >> $GITHUB_OUTPUT | |
| - name: Setup system details | |
| id: system | |
| run: | | |
| export CPUS="8" | |
| export MEMORY="16g" | |
| echo "cpus total: ${CPUS}" | |
| echo "memory total: ${MEMORY}" | |
| echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT | |
| echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT | |
| # github.rest.actions.listWorkflowRunArtifacts only returns first 30 | |
| # artifacts, and returns a { data, headers, status, url } object. The | |
| # "data" part of this object contains the artifact data we care about. | |
| # The fields of this data object correspond to the fields in the | |
| # "Example Response" JSON object in | |
| # https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts. | |
| # To return more than 30 responses, use the github.paginate API in | |
| # https://octokit.github.io/rest.js/v19#custom-requests | |
| # `github-script` aliases `octokit` to the `github` namespace. | |
| - name: 'Download metadata' | |
| uses: actions/[email protected] | |
| with: | |
| script: | | |
| var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: ${{github.event.workflow_run.id }}, | |
| }); | |
| var matchArtifact = artifacts.data.artifacts.filter((artifact) => { | |
| return artifact.name == "meta" | |
| })[0]; | |
| console.log("Downloading artifact %s", matchArtifact.id); | |
| var download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifact.id, | |
| archive_format: 'zip', | |
| }); | |
| var fs = require('fs'); | |
| fs.writeFileSync('${{github.workspace}}/meta.zip', Buffer.from(download.data)); | |
| - run: unzip meta.zip | |
| - name: Setup metadata | |
| id: metadata | |
| run: | | |
| cat meta | |
| cat meta >> $GITHUB_OUTPUT | |
| confirm-valid-credentials: | |
| name: Confirm AWS credentials are minimally valid | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - compute-metadata | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp | |
| ## | |
| ## SUBMIT | |
| ## | |
| upload-baseline-image-to-ecr: | |
| name: Upload images to ECR | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - compute-metadata | |
| - confirm-valid-credentials | |
| steps: | |
| - name: 'Download baseline image' | |
| uses: actions/[email protected] | |
| with: | |
| script: | | |
| var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: ${{github.event.workflow_run.id }}, | |
| }); | |
| var matchArtifact = artifacts.data.artifacts.filter((artifact) => { | |
| return artifact.name == "baseline-image" | |
| })[0]; | |
| console.log("Downloading artifact %s", matchArtifact.id); | |
| var download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifact.id, | |
| archive_format: 'zip', | |
| }); | |
| var fs = require('fs'); | |
| fs.writeFileSync('${{github.workspace}}/baseline-image.zip', Buffer.from(download.data)); | |
| - run: unzip baseline-image.zip | |
| - name: Load baseline image | |
| run: | | |
| docker load --input baseline-image.tar | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Docker Login to ECR | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ steps.login-ecr.outputs.registry }} | |
| - name: Tag & push baseline image | |
| run: | | |
| docker tag lading:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} | |
| docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} | |
| upload-comparison-image-to-ecr: | |
| name: Upload images to ECR | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - compute-metadata | |
| - confirm-valid-credentials | |
| steps: | |
| - name: 'Download comparison image' | |
| uses: actions/[email protected] | |
| with: | |
| script: | | |
| var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: ${{github.event.workflow_run.id }}, | |
| }); | |
| var matchArtifact = artifacts.data.artifacts.filter((artifact) => { | |
| return artifact.name == "comparison-image" | |
| })[0]; | |
| console.log("Downloading artifact %s", matchArtifact.id); | |
| var download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifact.id, | |
| archive_format: 'zip', | |
| }); | |
| var fs = require('fs'); | |
| fs.writeFileSync('${{github.workspace}}/comparison-image.zip', Buffer.from(download.data)); | |
| - run: unzip comparison-image.zip | |
| - name: Load comparison image | |
| run: | | |
| docker load --input comparison-image.tar | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Docker Login to ECR | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ steps.login-ecr.outputs.registry }} | |
| - name: Tag & push comparison image | |
| run: | | |
| docker tag lading:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} | |
| docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} | |
| submit-job: | |
| name: Submit regression job | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - compute-metadata | |
| - upload-baseline-image-to-ecr | |
| - upload-comparison-image-to-ecr | |
| steps: | |
| - name: Check status, in-progress | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='pending' \ | |
| -f description='Experiments submitted to the Regression Detector cluster.' \ | |
| -f context='Regression Detector / submission' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.compute-metadata.outputs.checkout-sha }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp | |
| - name: Submit job | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ | |
| --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ | |
| --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ | |
| --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ | |
| --replicas ${{ needs.compute-metadata.outputs.replicas }} \ | |
| --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} \ | |
| --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} \ | |
| --baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \ | |
| --comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \ | |
| --target-config-dir ${{ github.workspace }}/regression/ \ | |
| --target-name lading-target \ | |
| --target-command "/usr/local/bin/lading --no-target --config-path /etc/lading-target/lading.yaml --capture-path /dev/null --experiment-duration-seconds 1200" \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: lading-submission-metadata | |
| path: ${{ runner.temp }}/submission-metadata | |
| - name: Await job | |
| timeout-minutes: 60 | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ | |
| --wait \ | |
| --wait-delay-seconds 60 \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| - name: Handle cancellation if necessary | |
| if: ${{ cancelled() }} | |
| timeout-minutes: 60 | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| - name: Check status, cancelled | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='failure' \ | |
| -f description='Experiments submitted to the Regression Detector cluster cancelled.' \ | |
| -f context='Regression Detector / submission' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Check status, success | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='success' \ | |
| -f description='Experiments submitted to the Regression Detector cluster successfully.' \ | |
| -f context='Regression Detector / submission' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Check status, failure | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='success' \ | |
| -f description='Experiments submitted to the Regression Detector cluster failed.' \ | |
| -f context='Regression Detector / submission' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| ## | |
| ## ANALYZE | |
| ## | |
| download-analysis: | |
| name: Download regression analysis & upload report | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - submit-job | |
| - compute-metadata | |
| steps: | |
| - name: Check status, in-progress | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='pending' \ | |
| -f description='Analyze experimental results from Regression Detector.' \ | |
| -f context='Regression Detector / analyze-experiment' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.compute-metadata.outputs.checkout-sha }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp | |
| - name: Download submission metadata | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: lading-submission-metadata | |
| path: ${{ runner.temp }}/ | |
| - name: Sync regression report to local system | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata \ | |
| --output-path "${{ runner.temp }}/outputs" | |
| - name: Read regression report | |
| id: read-analysis | |
| uses: juliangruber/read-file-action@v1 | |
| with: | |
| path: ${{ runner.temp }}/outputs/report.html | |
| - name: Post report to PR | |
| uses: peter-evans/create-or-update-comment@v2 | |
| with: | |
| issue-number: ${{ needs.compute-metadata.outputs.github-event-number }} | |
| edit-mode: append | |
| body: ${{ steps.read-analysis.outputs.content }} | |
| - name: Upload regression report to artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: capture-artifacts | |
| path: ${{ runner.temp }}/outputs/* | |
| - name: Check status, cancelled | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='failure' \ | |
| -f description='Analyze experimental results from Regression Detector cancelled.' \ | |
| -f context='Regression Detector / analyze-experiment' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Check status, success | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='success' \ | |
| -f description='Analyze experimental results from Regression Detector succeeded.' \ | |
| -f context='Regression Detector / analyze-experiment' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Check status, failure | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ | |
| -f state='success' \ | |
| -f description='Analyze experimental results from Regression Detector failed.' \ | |
| -f context='Regression Detector / analyze-experiment' \ | |
| -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |