ci: move benchmark to Github Action (#2251)

* ci: remove deprecated benchmark scripts

* ci: add upload benchmark script

* ci: add benchmark github actions script

* ci: slient curl when uploading benchmark datapoint

* ci: add name as a tag

* empty commit 1

* empty commit 2

* empty commit 3

* empty commit 4

* empty commit 5

* empty commit 6

* empty commit 7

* empty commit 8

* empty commit 9

* empty commit 10

* unify benchmark result format

.github/workflows/benchmark.yml

@@ -0,0 +1,98 @@
+name: Benchmark
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  benchmark:
+    name: benchmark
+    runs-on: benchmark
+    strategy:
+      fail-fast: false
+      matrix:
+        # before adding your benchmark. please check these steps:
+        # 1. generate a file that includes your benchmark result. it should looks like
+        #
+        #     ```
+        #     test bench_accounts_delta_hash                         ... bench:  48,035,858 ns/iter (+/- 2,118,806)
+        #     ```
+        #
+        # 2. run `DRY_RUN=1 ./ci/upload-benchmark.sh <YOUR_BENCHMARK_RESULT_FILE_PATH>` to ensure the datapoints are correct
+        #    it should looks similar to this:
+        #
+        #    ```
+        #    datapoint: ,commit=xxxx,test_suite=xxxx,name=bench_accounts_delta_hash median=48035858,deviation=2118806i
+        #    ```
+        #
+        #    you only need to check `name`, `median` and `deviation`
+        #
+        test:
+          - {
+              name: "solana-sdk",
+              commands: ["cargo +$rust_nightly bench -p solana-sdk"],
+            }
+          - {
+              name: "solana-runtime",
+              commands: ["cargo +$rust_nightly bench -p solana-runtime"],
+            }
+          - {
+              name: "solana-gossip",
+              commands: ["cargo +$rust_nightly bench -p solana-gossip"],
+            }
+          - {
+              name: "solana-poh",
+              commands: ["cargo +$rust_nightly bench -p solana-poh"],
+            }
+          - {
+              name: "solana-core",
+              commands: ["cargo +$rust_nightly bench -p solana-core"],
+            }
+          - {
+              name: "sbf",
+              before_command: "make -C programs/sbf all",
+              commands:
+                [
+                  "cargo +$rust_nightly bench --manifest-path programs/sbf/Cargo.toml --features=sbf_c",
+                ],
+            }
+          # spliting solana-accounts-db because it includes criterion bench
+          - {
+              name: "solana-accounts-db",
+              commands:
+                [
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts_index",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench append_vec",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_accounts_file -- --output-format bencher",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_hashing -- --output-format bencher",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_serde -- --output-format bencher",
+                ],
+            }
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Before Command
+        if: ${{ matrix.test.before_command != '' }}
+        run: |
+          ${{ matrix.test.before_command }}
+
+      - name: Command
+        run: |
+          source ci/rust-version.sh nightly
+          echo '${{ toJson(matrix.test.commands) }}' | jq -r '.[]' | while read command; do
+            eval $command | tee -a benchmark
+          done
+
+      - name: Upload Result
+        run: |
+          TEST_SUITE="${{ matrix.test.name }}" \
+          COMMIT_HASH="$(git rev-parse HEAD)" \
+          INFLUX_HOST="${{ secrets.BENCHMARK_INFLUX_HOST }}" \
+          INFLUX_DB="${{ secrets.BENCHMARK_INFLUX_DB }}" \
+          INFLUX_USER="${{ secrets.BENCHMARK_INFLUX_USER }}" \
+          INFLUX_PASSWORD="${{ secrets.BENCHMARK_INFLUX_PASSWORD }}" \
+          INFLUX_MEASUREMENT="${{ secrets.BENCHMARK_INFLUX_MEASUREMENT }}" \
+          ./ci/upload-benchmark.sh benchmark

ci/buildkite-pipeline.sh

@@ -283,23 +283,6 @@ EOF
       "wasm skipped as no relevant files were modified"
   fi
 
-  # Benches...
-  if affects \
-             .rs$ \
-             Cargo.lock$ \
-             Cargo.toml$ \
-             ^ci/rust-version.sh \
-             ^ci/test-coverage.sh \
-             ^ci/test-bench.sh \
-             ^ci/bench \
-             .buildkite/scripts/build-bench.sh \
-      ; then
-    .buildkite/scripts/build-bench.sh >> "$output_file"
-  else
-    annotate --style info --context test-bench \
-      "Bench skipped as no .rs files were modified"
-  fi
-
   # Coverage...
   if affects \
              .rs$ \

ci/buildkite-solana-private.sh

@@ -214,21 +214,6 @@ EOF
       "wasm skipped as no relevant files were modified"
   fi
 
-  # Benches...
-  if affects \
-             .rs$ \
-             Cargo.lock$ \
-             Cargo.toml$ \
-             ^ci/rust-version.sh \
-             ^ci/test-coverage.sh \
-             ^ci/test-bench.sh \
-      ; then
-    .buildkite/scripts/build-bench.sh sol-private >> "$output_file"
-  else
-    annotate --style info --context test-bench \
-      "Bench skipped as no .rs files were modified"
-  fi
-
   # Coverage...
   if affects \
              .rs$ \

ci/upload-benchmark.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+set -e
+
+usage() {
+  cat <<EOF >&2
+USAGE:
+    $0 <BENCHMARK_FILEPATH>
+
+REQUIRED ENVIRONMENTS:
+    INFLUX_HOST           Hostname or IP address of the InfluxDB server
+    INFLUX_DB             Name of the InfluxDB database
+    INFLUX_USER           Username for InfluxDB
+    INFLUX_PASSWORD       Password for InfluxDB
+    INFLUX_MEASUREMENT    Measurement for InfluxDB
+
+OPTIONAL ENVIRONMENTS:
+    COMMIT_HASH   Commit hash of the benchmark file
+    TEST_SUITE    The group name for all tests in the benchmark file
+    DRY_RUN       Dry run
+
+ARGS:
+    <BENCHMARK_FILEPATH>    The output file generated by running
+                            \`cargo bench -- -Z unstable-options --format=json\`
+                            contains the benchmark results in JSON format
+EOF
+}
+
+print_error_and_exit() {
+  local msg="$1"
+  echo "error: $msg" >&2
+  echo ""
+  usage
+  exit 1
+}
+
+check_env() {
+  local var_name="$1"
+  if [ -z "${!var_name}" ]; then
+    print_error_and_exit "Environment variable $var_name is required"
+  fi
+}
+
+filepath="$1"
+if [ ! -f "$filepath" ]; then
+  print_error_and_exit "invalid <BENCHMARK_FILEPATH>"
+fi
+
+if [ -z "$COMMIT_HASH" ]; then
+  COMMIT_HASH=$(uuidgen)
+fi
+
+if [ -z "$TEST_SUITE" ]; then
+  TEST_SUITE="$(basename "${BENCHMARK_FILEPATH}")-$(date +%s)"
+fi
+
+if [ -z "$DRY_RUN" ]; then
+  required_env_vars=(
+    "INFLUX_HOST"
+    "INFLUX_DB"
+    "INFLUX_USER"
+    "INFLUX_PASSWORD"
+    "INFLUX_MEASUREMENT"
+  )
+  for var in "${required_env_vars[@]}"; do
+    check_env "$var"
+  done
+fi
+
+while IFS= read -r line; do
+
+  if [[ $line =~ ^test\ (.*)\ \.\.\.\ bench:\ *([0-9,]+)\ ns\/iter\ \(\+\/-\ *([0-9,]+)\) ]]; then
+    test_name="${BASH_REMATCH[1]}"
+    ns_iter="${BASH_REMATCH[2]}"
+    plus_minus="${BASH_REMATCH[3]}"
+
+    ns_iter=$(echo "$ns_iter" | tr -d ',')
+    plus_minus=$(echo "$plus_minus" | tr -d ',')
+
+    datapoint="${INFLUX_MEASUREMENT},commit=${COMMIT_HASH},test_suite=${TEST_SUITE},name=${test_name} median=${ns_iter}i,deviation=${plus_minus}i"
+    echo "datapoint: $datapoint"
+
+    if [[ -z "$DRY_RUN" ]]; then
+      curl -s -X POST "${INFLUX_HOST}/write?db=${INFLUX_DB}" --data-binary "$datapoint"
+    fi
+  fi
+
+done <"$filepath"