Skip to content

Regression Detector (trusted) #369

Regression Detector (trusted)

Regression Detector (trusted) #369

name: Regression Detector (trusted)
on:
workflow_run:
workflows: ["Regression Detector (untrusted)"]
types:
- completed
jobs:
compute-metadata:
name: Compute metadata for regression experiments
runs-on: ubuntu-22.04
if: >
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success'
outputs:
cpus: ${{ steps.system.outputs.CPUS }}
memory: ${{ steps.system.outputs.MEMORY }}
comparison-sha: ${{ steps.metadata.outputs.COMPARISON_SHA }}
comparison-tag: ${{ steps.metadata.outputs.COMPARISON_TAG }}
baseline-sha: ${{ steps.metadata.outputs.BASELINE_SHA }}
baseline-tag: ${{ steps.metadata.outputs.BASELINE_TAG }}
head-sha: ${{ steps.metadata.outputs.HEAD_SHA }}
checkout-sha: ${{ steps.metadata.outputs.CHECKOUT_SHA }}
github-event-number: ${{ steps.metadata.outputs.GITHUB_EVENT_NUMBER }}
replicas: ${{ steps.experimental-meta.outputs.REPLICAS }}
warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }}
total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }}
p-value: ${{ steps.experimental-meta.outputs.P_VALUE }}
smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }}
lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }}
smp-wait-timeout-minutes: ${{ steps.experimental-meta.outputs.SMP_WAIT_TIMEOUT }}
steps:
- name: Setup experimental metadata
id: experimental-meta
run: |
export WARMUP_SECONDS="45"
export REPLICAS="10"
export TOTAL_SAMPLES="600"
export P_VALUE="0.1"
export SMP_CRATE_VERSION="0.8.0"
export SMP_WAIT_TIMEOUT="55"
export LADING_VERSION="0.16.0"
echo "warmup seconds: ${WARMUP_SECONDS}"
echo "replicas: ${REPLICAS}"
echo "total samples: ${TOTAL_SAMPLES}"
echo "regression p-value: ${P_VALUE}"
echo "smp crate version: ${SMP_CRATE_VERSION}"
echo "lading version: ${LADING_VERSION}"
echo "smp wait timeout: ${SMP_WAIT_TIMEOUT}"
echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT
echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT
echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT
echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT
echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT
echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT
echo "SMP_WAIT_TIMEOUT=${SMP_WAIT_TIMEOUT}" >> $GITHUB_OUTPUT
- name: Setup system details
id: system
run: |
export CPUS="8"
export MEMORY="16g"
echo "cpus total: ${CPUS}"
echo "memory total: ${MEMORY}"
echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT
echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT
# github.rest.actions.listWorkflowRunArtifacts only returns first 30
# artifacts, and returns a { data, headers, status, url } object. The
# "data" part of this object contains the artifact data we care about.
# The fields of this data object correspond to the fields in the
# "Example Response" JSON object in
# https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts.
# To return more than 30 responses, use the github.paginate API in
# https://octokit.github.io/rest.js/v19#custom-requests
# `github-script` aliases `octokit` to the `github` namespace.
- name: 'Download metadata'
uses: actions/[email protected]
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "meta"
})[0];
console.log("Downloading artifact %s", matchArtifact.id);
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/meta.zip', Buffer.from(download.data));
- run: unzip meta.zip
- name: Setup metadata
id: metadata
run: |
cat meta
cat meta >> $GITHUB_OUTPUT
confirm-valid-credentials:
name: Confirm AWS credentials are minimally valid
runs-on: ubuntu-22.04
needs:
- compute-metadata
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
##
## SUBMIT
##
upload-baseline-image-to-ecr:
name: Upload images to ECR
runs-on: ubuntu-22.04
needs:
- compute-metadata
- confirm-valid-credentials
steps:
- name: 'Download baseline image'
uses: actions/[email protected]
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "baseline-image"
})[0];
console.log("Downloading artifact %s", matchArtifact.id);
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/baseline-image.zip', Buffer.from(download.data));
- run: unzip baseline-image.zip
- name: Load baseline image
run: |
docker load --input baseline-image.tar
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Docker Login to ECR
uses: docker/login-action@v2
with:
registry: ${{ steps.login-ecr.outputs.registry }}
- name: Tag & push baseline image
run: |
docker tag lading:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }}
docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }}
upload-comparison-image-to-ecr:
name: Upload images to ECR
runs-on: ubuntu-22.04
needs:
- compute-metadata
- confirm-valid-credentials
steps:
- name: 'Download comparison image'
uses: actions/[email protected]
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "comparison-image"
})[0];
console.log("Downloading artifact %s", matchArtifact.id);
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/comparison-image.zip', Buffer.from(download.data));
- run: unzip comparison-image.zip
- name: Load comparison image
run: |
docker load --input comparison-image.tar
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Docker Login to ECR
uses: docker/login-action@v2
with:
registry: ${{ steps.login-ecr.outputs.registry }}
- name: Tag & push comparison image
run: |
docker tag lading:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }}
docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }}
submit-job:
name: Submit regression job
runs-on: ubuntu-22.04
needs:
- compute-metadata
- upload-baseline-image-to-ecr
- upload-comparison-image-to-ecr
steps:
- name: Check status, in-progress
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='pending' \
-f description='Experiments submitted to the Regression Detector cluster.' \
-f context='Regression Detector / submission' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- uses: actions/checkout@v3
with:
ref: ${{ needs.compute-metadata.outputs.checkout-sha }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
- name: Submit job
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \
--lading-version ${{ needs.compute-metadata.outputs.lading-version }} \
--total-samples ${{ needs.compute-metadata.outputs.total-samples }} \
--warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \
--replicas ${{ needs.compute-metadata.outputs.replicas }} \
--baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} \
--comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} \
--baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \
--comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \
--target-config-dir ${{ github.workspace }}/regression/ \
--target-name lading-target \
--target-command "/usr/local/bin/lading --no-target --config-path /etc/lading-target/lading.yaml --capture-path /dev/null --experiment-duration-seconds 1200" \
--submission-metadata ${{ runner.temp }}/submission-metadata
- uses: actions/upload-artifact@v3
with:
name: lading-submission-metadata
path: ${{ runner.temp }}/submission-metadata
- name: Await job
timeout-minutes: 60
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \
--wait \
--wait-delay-seconds 60 \
--submission-metadata ${{ runner.temp }}/submission-metadata
- name: Handle cancellation if necessary
if: ${{ cancelled() }}
timeout-minutes: 60
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \
--submission-metadata ${{ runner.temp }}/submission-metadata
- name: Check status, cancelled
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='failure' \
-f description='Experiments submitted to the Regression Detector cluster cancelled.' \
-f context='Regression Detector / submission' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Check status, success
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='success' \
-f description='Experiments submitted to the Regression Detector cluster successfully.' \
-f context='Regression Detector / submission' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Check status, failure
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='success' \
-f description='Experiments submitted to the Regression Detector cluster failed.' \
-f context='Regression Detector / submission' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
##
## ANALYZE
##
download-analysis:
name: Download regression analysis & upload report
runs-on: ubuntu-22.04
needs:
- submit-job
- compute-metadata
steps:
- name: Check status, in-progress
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='pending' \
-f description='Analyze experimental results from Regression Detector.' \
-f context='Regression Detector / analyze-experiment' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- uses: actions/checkout@v3
with:
ref: ${{ needs.compute-metadata.outputs.checkout-sha }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
- name: Download submission metadata
uses: actions/download-artifact@v3
with:
name: lading-submission-metadata
path: ${{ runner.temp }}/
- name: Sync regression report to local system
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \
--submission-metadata ${{ runner.temp }}/submission-metadata \
--output-path "${{ runner.temp }}/outputs"
- name: Read regression report
id: read-analysis
uses: juliangruber/read-file-action@v1
with:
path: ${{ runner.temp }}/outputs/report.html
- name: Post report to PR
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ needs.compute-metadata.outputs.github-event-number }}
edit-mode: append
body: ${{ steps.read-analysis.outputs.content }}
- name: Upload regression report to artifacts
uses: actions/upload-artifact@v3
with:
name: capture-artifacts
path: ${{ runner.temp }}/outputs/*
- name: Check status, cancelled
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='failure' \
-f description='Analyze experimental results from Regression Detector cancelled.' \
-f context='Regression Detector / analyze-experiment' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Check status, success
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='success' \
-f description='Analyze experimental results from Regression Detector succeeded.' \
-f context='Regression Detector / analyze-experiment' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Check status, failure
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \
-f state='success' \
-f description='Analyze experimental results from Regression Detector failed.' \
-f context='Regression Detector / analyze-experiment' \
-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}