benchmark.sh

# Benchmark script to compare the benchmarks caused by the changes of the
# current PR against the main branch

# Run the benchmarks on the current PR and save it in a JSON file
#
# TODO: naveen, Benchmark seperate targets and merge their JSON. Currently,
# `--save-baseline` only supports saving benchmarks for a single target. For
# now, we'll benchmark only the `execute` target
# See: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
#
# cargo bench --bench execute --bench generate_ir --bench validation --bench parser -- --save-baseline current-benchmarks;
cargo bench --bench execute -- --save-baseline current-benchmarks;
critcmp --export current-benchmarks > current-benchmarks.json;

# Clone into the main branch of v3-engine
# https://$2@github.com/hasura/v3-engine.git
git clone --branch main https://$2@github.com/hasura/v3-engine.git main-copy;
pushd main-copy;
# Run the benchmarks on the main branch and save it in a JSON file
cargo bench --bench execute -- --save-baseline main-benchmarks;
critcmp --export main-benchmarks > /app/main-benchmarks.json;
popd

# Compare between the benchmarks of main and the benchmarks generated by the PR
critcmp main-benchmarks.json current-benchmarks.json > diff.txt;

# Format the result of benchmark into a Code comment, so that when we 
# comment on the PR, the benchmark results are displayed in a nice format
sed -i '1s/^/Benchmark report\n ```\n/' diff.txt;
echo '```' >> diff.txt;

# Format the results into a json format
# { "body": "<BENCHMARK AS STRING>""}
echo '{}' | jq --rawfile diff diff.txt '.body=$diff' > bench_comment.json 

curl -s -H "Authorization: token $2" \
  -X GET "https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" > /tmp/comments.json
comment_id=$(jq 'map(select(.body | contains ("Benchmark report"))) | last | .id' /tmp/comments.json)
echo $comment_id
# Post the benchmark as comment on PR or update existing one
if [ "$comment_id" = "null" ]; then
  curl -L \
    -X POST \
    -H "Accept: application/vnd.github+json" \
    -H "Authorization: Bearer $2" \
    -H "X-GitHub-Api-Version: 2022-11-28" \
    "https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" \
    -d @bench_comment.json
else
  curl -L \
    -X PATCH \
    -H "Accept: application/vnd.github+json" \
    -H "Authorization: Bearer $2" \
    -H "X-GitHub-Api-Version: 2022-11-28" \
    "https://api.github.com/repos/hasura/v3-engine/issues/comments/$comment_id" \
    -d @bench_comment.json
fi