From 8cde169e7ece3b1fcc254d294ed3e4649591d956 Mon Sep 17 00:00:00 2001
From: Sambhav Jain <sambhav.jain@cockroachlabs.com>
Date: Wed, 10 Jul 2024 11:45:45 +0530
Subject: [PATCH] workflow: add cockroach-microbench-ci step in github actions

Fixes: https://github.com/cockroachdb/cockroach/issues/106661
Release note: None
Epic: https://cockroachlabs.atlassian.net/browse/CRDB-29662

This change intends to add a new CI step which will run `BenchmarkKV`
microbenchmark for the current commit and compares the result with that
of the base commit's results which is stored in a cloud bucket.

This is needed to avoid minor perf regressions so that any regression is
caught in the PR itself. The current threshold is set to 5%.

There are highly 3 steps 1. Run benchmarks 2. If this is PR branch,
download the latest commit of the target branch's bench results from the
cloud storage and compare the results. If there's a regression fail here
itself 3. If it is a push step (PR merge to master/release), upload the
result to the cloud storage with the current commit sha.

Currently, this change won't block the PR. However it will still run the
microbenchmarks and upload the results during the push step.
---
 .../workflows/github-actions-essential-ci.yml | 50 +++++++++++++
 build/github/cockroach-microbench-ci.sh       | 73 +++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100755 build/github/cockroach-microbench-ci.sh

diff --git a/.github/workflows/github-actions-essential-ci.yml b/.github/workflows/github-actions-essential-ci.yml
index 3fd2bdaca675..a7730545c54e 100644
--- a/.github/workflows/github-actions-essential-ci.yml
+++ b/.github/workflows/github-actions-essential-ci.yml
@@ -344,3 +344,53 @@ jobs:
       - name: clean up
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
+  cockroach-microbench-ci:
+    runs-on: [ self-hosted, basic_runner_group ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1  # Fetch only the latest commit
+
+      - name: Get the latest commit of the target branch
+        id: get_latest_commit_target_branch
+        run: |
+          # Get the target branch name
+          TARGET_BRANCH=${{ github.event.pull_request.base.ref }}
+          
+          # Fetch the latest commit of the target branch
+          git fetch origin $TARGET_BRANCH --depth=1
+          
+          # Get the latest commit hash of the target branch
+          LATEST_COMMIT=$(git rev-parse FETCH_HEAD)
+          
+          # Output the latest commit hash
+          echo "Latest commit on target branch ($TARGET_BRANCH): $LATEST_COMMIT"
+          
+          # Set the latest commit hash as an output variable
+          echo "::set-output name=latest_commit::$LATEST_COMMIT"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
+      - run: ./build/github/get-engflow-keys.sh
+      - run: ./build/github/prepare-summarize-build.sh
+      - name: run scripts
+        run: ./build/github/cockroach-microbench-ci.sh
+        env:
+          COMPARE_THRESHOLD: "5.00"
+          PUSH_STEP: ${{ github.event_name == 'push' }}
+          BASE_SHA: ${{ steps.get_latest_commit_target_branch.outputs.latest_commit }}
+
+          # TODO(sambhav-jain-16): add Performance note check and use this flag. Currently set to false (https://github.com/cockroachdb/cockroach/issues/106661)
+          SKIP_COMPARISON: false
+      - name: upload build results
+        run: ./build/github/summarize-build.sh bes.bin
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: clean up
+        run: ./build/github/cleanup-engflow-keys.sh
+        if: always()
+
diff --git a/build/github/cockroach-microbench-ci.sh b/build/github/cockroach-microbench-ci.sh
new file mode 100755
index 000000000000..9ef680ed5a94
--- /dev/null
+++ b/build/github/cockroach-microbench-ci.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# Directories and file names
+output_dir="./artifacts/microbench"
+cleaned_current_dir="$output_dir/current"
+cleaned_base_dir="$output_dir/base"
+
+# hardcoded the file name as of now since `compare` requires a particular format for the file name.
+benchmark_file_name="pkg_sql-report.log"
+log_output_file_path="$output_dir/microbench.log"
+storage_bucket_url="gs://cockroach-microbench-ci"
+
+# Threshold for comparison
+threshold=${COMPARE_THRESHOLD:-0}
+success_exit_status=0
+error_exit_status=1
+
+# Exit early if SKIP_COMPARISON is true and not a push step
+if $SKIP_COMPARISON && ! $PUSH_STEP; then
+  echo "Exiting since skip comparison is enabled and this is not a push step."
+  exit $success_exit_status
+fi
+
+# Build binary with Bazel
+bazel build --config crosslinux $(./build/github/engflow-args.sh) --jobs 100 //pkg/cmd/roachprod-microbench
+
+roachprod_microbench_dir="_bazel/bin/pkg/cmd/roachprod-microbench/roachprod-microbench_"
+
+mkdir -p "$output_dir"
+
+# Run benchmarks and clean output
+# running count=4 here because that's the minimum required for a comparison
+bazel test //pkg/sql/tests:tests_test \
+  --config=use_ci_timeouts \
+  --strategy=TestRunner=sandboxed \
+  --jobs 100 \
+  --config=crosslinux \
+  --remote_download_minimal \
+  $(./build/github/engflow-args.sh) \
+  --test_arg=-test.run=- \
+  --test_arg='-test.bench=^BenchmarkKV$' \
+  --test_sharding_strategy=disabled \
+  --test_arg=-test.cpu --test_arg=1 \
+  --test_arg=-test.v \
+  --test_arg=-test.count=4 \
+  --test_arg=-test.benchmem \
+  --crdb_test_off \
+  --test_output=all > "$log_output_file_path"
+
+$roachprod_microbench_dir/roachprod-microbench clean "$log_output_file_path" "$cleaned_current_dir/$benchmark_file_name"
+
+# Push artifact if this is a base merge and skip comparison
+if $PUSH_STEP; then
+  gcloud storage cp "$cleaned_current_dir/$benchmark_file_name" "$storage_bucket_url/$GITHUB_HEAD_REF/$GITHUB_SHA.log"
+  echo "Skipping comparison since this is a base merge."
+  exit $success_exit_status
+fi
+
+# Compare benchmarks
+if ! gcloud storage cp "$storage_bucket_url/$GITHUB_BASE_REF/$BASE_SHA.log" "$cleaned_base_dir/$benchmark_file_name"; then
+  echo "Couldn't download base bench file, exiting."
+  exit $success_exit_status
+fi
+
+if ! $roachprod_microbench_dir/roachprod-microbench compare "$cleaned_current_dir" "$cleaned_base_dir" --threshold "$threshold"; then
+  echo "There is an error during comparison. If it's a perf regression, please try to fix it. This won't block your change for merging currently."
+  exit $error_exit_status
+fi
+
+rm -rf "$output_dir"
+exit $success_exit_status