diff --git a/.github/workflows/github-actions-essential-ci.yml b/.github/workflows/github-actions-essential-ci.yml index 3fd2bdaca675..a7730545c54e 100644 --- a/.github/workflows/github-actions-essential-ci.yml +++ b/.github/workflows/github-actions-essential-ci.yml @@ -344,3 +344,53 @@ jobs: - name: clean up run: ./build/github/cleanup-engflow-keys.sh if: always() + cockroach-microbench-ci: + runs-on: [ self-hosted, basic_runner_group ] + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 # Fetch only the latest commit + + - name: Get the latest commit of the target branch + id: get_latest_commit_target_branch + run: | + # Get the target branch name + TARGET_BRANCH=${{ github.event.pull_request.base.ref }} + + # Fetch the latest commit of the target branch + git fetch origin $TARGET_BRANCH --depth=1 + + # Get the latest commit hash of the target branch + LATEST_COMMIT=$(git rev-parse FETCH_HEAD) + + # Output the latest commit hash + echo "Latest commit on target branch ($TARGET_BRANCH): $LATEST_COMMIT" + + # Set the latest commit hash as an output variable + echo "::set-output name=latest_commit::$LATEST_COMMIT" + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: compute metadata + run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV" + - run: ./build/github/get-engflow-keys.sh + - run: ./build/github/prepare-summarize-build.sh + - name: run scripts + run: ./build/github/cockroach-microbench-ci.sh + env: + COMPARE_THRESHOLD: "5.00" + PUSH_STEP: ${{ github.event_name == 'push' }} + BASE_SHA: ${{ steps.get_latest_commit_target_branch.outputs.latest_commit }} + + # TODO(sambhav-jain-16): add Performance note check and use this flag. Currently set to false (https://github.com/cockroachdb/cockroach/issues/106661) + SKIP_COMPARISON: false + - name: upload build results + run: ./build/github/summarize-build.sh bes.bin + if: always() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: clean up + run: ./build/github/cleanup-engflow-keys.sh + if: always() + diff --git a/build/github/cockroach-microbench-ci.sh b/build/github/cockroach-microbench-ci.sh new file mode 100755 index 000000000000..9ef680ed5a94 --- /dev/null +++ b/build/github/cockroach-microbench-ci.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -euxo pipefail + +# Directories and file names +output_dir="./artifacts/microbench" +cleaned_current_dir="$output_dir/current" +cleaned_base_dir="$output_dir/base" + +# hardcoded the file name as of now since `compare` requires a particular format for the file name. +benchmark_file_name="pkg_sql-report.log" +log_output_file_path="$output_dir/microbench.log" +storage_bucket_url="gs://cockroach-microbench-ci" + +# Threshold for comparison +threshold=${COMPARE_THRESHOLD:-0} +success_exit_status=0 +error_exit_status=1 + +# Exit early if SKIP_COMPARISON is true and not a push step +if $SKIP_COMPARISON && ! $PUSH_STEP; then + echo "Exiting since skip comparison is enabled and this is not a push step." + exit $success_exit_status +fi + +# Build binary with Bazel +bazel build --config crosslinux $(./build/github/engflow-args.sh) --jobs 100 //pkg/cmd/roachprod-microbench + +roachprod_microbench_dir="_bazel/bin/pkg/cmd/roachprod-microbench/roachprod-microbench_" + +mkdir -p "$output_dir" + +# Run benchmarks and clean output +# running count=4 here because that's the minimum required for a comparison +bazel test //pkg/sql/tests:tests_test \ + --config=use_ci_timeouts \ + --strategy=TestRunner=sandboxed \ + --jobs 100 \ + --config=crosslinux \ + --remote_download_minimal \ + $(./build/github/engflow-args.sh) \ + --test_arg=-test.run=- \ + --test_arg='-test.bench=^BenchmarkKV$' \ + --test_sharding_strategy=disabled \ + --test_arg=-test.cpu --test_arg=1 \ + --test_arg=-test.v \ + --test_arg=-test.count=4 \ + --test_arg=-test.benchmem \ + --crdb_test_off \ + --test_output=all > "$log_output_file_path" + +$roachprod_microbench_dir/roachprod-microbench clean "$log_output_file_path" "$cleaned_current_dir/$benchmark_file_name" + +# Push artifact if this is a base merge and skip comparison +if $PUSH_STEP; then + gcloud storage cp "$cleaned_current_dir/$benchmark_file_name" "$storage_bucket_url/$GITHUB_HEAD_REF/$GITHUB_SHA.log" + echo "Skipping comparison since this is a base merge." + exit $success_exit_status +fi + +# Compare benchmarks +if ! gcloud storage cp "$storage_bucket_url/$GITHUB_BASE_REF/$BASE_SHA.log" "$cleaned_base_dir/$benchmark_file_name"; then + echo "Couldn't download base bench file, exiting." + exit $success_exit_status +fi + +if ! $roachprod_microbench_dir/roachprod-microbench compare "$cleaned_current_dir" "$cleaned_base_dir" --threshold "$threshold"; then + echo "There is an error during comparison. If it's a perf regression, please try to fix it. This won't block your change for merging currently." + exit $error_exit_status +fi + +rm -rf "$output_dir" +exit $success_exit_status