From fa92f87e41eb6a07bc920355f6be6a45ed460ebf Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:01:52 +0200 Subject: [PATCH 1/7] introduce GitHub Action for CRON execution to detect regressions --- .github/workflows/run-coach.yml | 73 ++++++++++++++++++++++++++++ coach-things.sh | 86 +++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 .github/workflows/run-coach.yml create mode 100644 coach-things.sh diff --git a/.github/workflows/run-coach.yml b/.github/workflows/run-coach.yml new file mode 100644 index 0000000..7e0b886 --- /dev/null +++ b/.github/workflows/run-coach.yml @@ -0,0 +1,73 @@ +name: Coach + +on: + push: + branches: [ "master", "automation" ] + schedule: + # every day at 12 CET + - cron: "0 14 * * *" + workflow_dispatch: + inputs: + qdrant_version: + description: "Version of qdrant to pull the container for (tags/v1.6.1, , my-branch)" + default: dev + coaching_time: + description: "Duration in seconds, default is 3600s (60 min) " + default: 3600 + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Process inputs + id: default_inputs + run: | + qdrant_version="${{ inputs.qdrant_version }}" + if [ -z "$qdrant_version" ]; then + qdrant_version=dev + fi + + coaching_time="${{ inputs.coaching_time }}" + if [ -z "coaching_time" ]; then + coaching_time=3600 + fi + + echo "qdrant_version=$qdrant_version" >> $GITHUB_OUTPUT + echo "coaching_time=coaching_time" >> $GITHUB_OUTPUT + - name: Install minimal stable + uses: dtolnay/rust-toolchain@stable + # checkout coach + - name: Checkout Coach + uses: actions/checkout@v4 + - name: Build Coach (release) + run: cargo build --release + - name: Coach things + shell: bash + run: | + coaching_time="${{ steps.default_inputs.outputs.coaching_time }}" + qdrant_version="${{ inputs.default_inputs.outputs.qdrant_version }}" + + ./coach-things.sh "$coaching_time" "$qdrant_version" + - name: Send Notification + if: failure() || cancelled() + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + { + "text": "Coach run status: ${{ job.status }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Something is wrong.\nView the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" + } + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.COACH_CHANNEL_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/coach-things.sh b/coach-things.sh new file mode 100644 index 0000000..526db4a --- /dev/null +++ b/coach-things.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -e + +RUN_TIME=${1:-300} +QDRANT_VERSION=${2:dev} + +REST_PORT="6333" +GRPC_PORT="6334" +QDRANT_HOST=localhost:${REST_PORT} + +echo "Running for $RUN_TIME seconds" +echo "QDRANT_HOST: $QDRANT_HOST" +echo "QDRANT_VERSION: $QDRANT_VERSION" + +# start qdrant docker +docker run -d --rm \ + -p ${REST_PORT}:${REST_PORT} \ + -p ${GRPC_PORT}:${GRPC_PORT} \ + -e QDRANT__SERVICE__GRPC_PORT=${GRPC_PORT} \ + --name qdrant_test qdrant/qdrant:"${QDRANT_VERSION}" ./qdrant --disable-telemetry + +trap stop_docker SIGINT +trap stop_docker ERR + +until $(curl --output /dev/null --silent --get --fail http://$QDRANT_HOST/collections); do + printf 'waiting for server to start...' + sleep 5 +done + +# start coach against qdrant +COACH_CMD=( + cargo run --release + -- + -p 1 # number of concurrent workloads + --stop-at-first-error + --grpc-timeout-ms 10000 + --grpc-health-check-timeout-ms 1000 +) + +echo "" +echo "${COACH_CMD[*]}" +"${COACH_CMD[@]}" & + +pid=$! + +echo "The PID is $pid" + +function cleanup() { + if ps -p $pid >/dev/null + then + kill -KILL $pid + stop_docker + fi +} + +function stop_docker() +{ + echo "stopping qdrant_test" + docker stop qdrant_test +} + + +trap cleanup EXIT + +trap 'exit $?' ERR +trap exit INT + +started=$(date +%s) + +while ps -p $pid >/dev/null && (( $(date +%s) - started < RUN_TIME )) +do + sleep 10 +done + +if ps -p $pid >/dev/null +then + echo "The process is still running. Stopping the process..." + kill $pid + stop_docker + echo "OK" +else + echo "The process has unexpectedly terminated on its own. Check the logs." + stop_docker + exit 1 +fi From 59c6a6c0e577455d2c284dc41179ca9486a977c1 Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:07:35 +0200 Subject: [PATCH 2/7] retrigger --- coach-things.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coach-things.sh b/coach-things.sh index 526db4a..33ea3cf 100644 --- a/coach-things.sh +++ b/coach-things.sh @@ -44,7 +44,7 @@ echo "${COACH_CMD[*]}" pid=$! -echo "The PID is $pid" +echo "The coach PID is $pid" function cleanup() { if ps -p $pid >/dev/null From 817e954827a0ffad468fff60a951371e61777afa Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:11:41 +0200 Subject: [PATCH 3/7] permissions --- coach-things.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 coach-things.sh diff --git a/coach-things.sh b/coach-things.sh old mode 100644 new mode 100755 From d78c90f116f70e21f2fe99d25c106cae327a00e5 Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:14:27 +0200 Subject: [PATCH 4/7] fix --- .github/workflows/run-coach.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-coach.yml b/.github/workflows/run-coach.yml index 7e0b886..90a8209 100644 --- a/.github/workflows/run-coach.yml +++ b/.github/workflows/run-coach.yml @@ -48,7 +48,7 @@ jobs: shell: bash run: | coaching_time="${{ steps.default_inputs.outputs.coaching_time }}" - qdrant_version="${{ inputs.default_inputs.outputs.qdrant_version }}" + qdrant_version="${{ steps.default_inputs.outputs.qdrant_version }}" ./coach-things.sh "$coaching_time" "$qdrant_version" - name: Send Notification From 665b08887a734b2439288e360ed1ab521c3204d6 Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:19:40 +0200 Subject: [PATCH 5/7] fix --- .github/workflows/run-coach.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-coach.yml b/.github/workflows/run-coach.yml index 90a8209..020e2de 100644 --- a/.github/workflows/run-coach.yml +++ b/.github/workflows/run-coach.yml @@ -36,7 +36,7 @@ jobs: fi echo "qdrant_version=$qdrant_version" >> $GITHUB_OUTPUT - echo "coaching_time=coaching_time" >> $GITHUB_OUTPUT + echo "coaching_time=$coaching_time" >> $GITHUB_OUTPUT - name: Install minimal stable uses: dtolnay/rust-toolchain@stable # checkout coach From 2311a88408c9e4a589ef25558ff5f8a8b51a9e5b Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 10:31:33 +0200 Subject: [PATCH 6/7] don't give up --- .github/workflows/run-coach.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-coach.yml b/.github/workflows/run-coach.yml index 020e2de..029290f 100644 --- a/.github/workflows/run-coach.yml +++ b/.github/workflows/run-coach.yml @@ -31,7 +31,7 @@ jobs: fi coaching_time="${{ inputs.coaching_time }}" - if [ -z "coaching_time" ]; then + if [ -z "$coaching_time" ]; then coaching_time=3600 fi From 37afde82c38aa5b5b85a0c0472ba61e5609453bd Mon Sep 17 00:00:00 2001 From: Arnaud Gourlay Date: Fri, 30 Aug 2024 12:35:52 +0200 Subject: [PATCH 7/7] version arg abd bump number of concurrent workload --- coach-things.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coach-things.sh b/coach-things.sh index 33ea3cf..1ac476c 100755 --- a/coach-things.sh +++ b/coach-things.sh @@ -3,7 +3,7 @@ set -e RUN_TIME=${1:-300} -QDRANT_VERSION=${2:dev} +QDRANT_VERSION=${2:-dev} REST_PORT="6333" GRPC_PORT="6334" @@ -32,7 +32,7 @@ done COACH_CMD=( cargo run --release -- - -p 1 # number of concurrent workloads + -p 2 # number of concurrent workloads --stop-at-first-error --grpc-timeout-ms 10000 --grpc-health-check-timeout-ms 1000