⚡️(api) automate API performance testing using Locust

API performances are now automatically tested in the CI and collected data are stored to track their evolution. One can also test API performance locally using the locust service.
MTES-MCT · Dec 2, 2024 · fdbbfb5 · fdbbfb5
1 parent 91bcfc7
commit fdbbfb5
Show file tree

Hide file tree

Showing 15 changed files with 1,637 additions and 200 deletions.
diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml
@@ -28,7 +28,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd src/api
-          pipenv install -d .
+          pipenv install -d
 
   lint-api:
     needs: build-api
@@ -53,6 +53,226 @@ jobs:
       - name: Lint with MyPy
         run: pipenv run mypy qualicharge tests
 
+  lint-bench:
+    needs: build-api
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src/bench
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+          cache-dependency-path: "src/api/Pipfile.lock"
+      - name: Lint with Black
+        run: pipenv run black --check .
+      - name: Lint with Ruff
+        run: pipenv run ruff check .
+      - name: Lint with MyPy
+        run: pipenv run mypy .
+
+  bench-api:
+    needs: build-api
+    runs-on: ubuntu-latest
+    services:
+      postgresql:
+        image: timescale/timescaledb-ha:pg14-ts2.14-oss
+        env:
+          POSTGRES_DB: qualicharge-api
+          POSTGRES_USER: qualicharge
+          POSTGRES_PASSWORD: pass
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    permissions:
+      pull-requests: write
+      contents: write
+    defaults:
+      run:
+        working-directory: ./src/api
+    env:
+      PORT: 8000
+      QUALICHARGE_DB_ENGINE: postgresql+psycopg
+      QUALICHARGE_DB_HOST: localhost
+      QUALICHARGE_DB_NAME: qualicharge-api
+      QUALICHARGE_TEST_DB_NAME: test-qualicharge-api
+      QUALICHARGE_OIDC_IS_ENABLED: False
+      QUALICHARGE_ALLOWED_HOSTS: '["http://localhost:8000"]'
+      QUALICHARGE_API_STATIQUE_BULK_CREATE_MAX_SIZE: 1000
+      QUALICHARGE_DEBUG: 0
+      QUALICHARGE_PROFILING: 0
+      QUALICHARGE_UVICORN_WORKERS: 1
+      QUALICHARGE_DB_CONNECTION_MAX_OVERFLOW: 200
+      QUALICHARGE_DB_CONNECTION_POOL_SIZE: 50
+      QUALICHARGE_STATIQUE_DATA_PATH: /home/runner/work/qualicharge/qualicharge/data/irve-statique.json.gz
+      QUALICHARGE_API_ADMIN_USER: admin
+      QUALICHARGE_API_ADMIN_PASSWORD: admin
+      # This is a fake setting required to run the app
+      QUALICHARGE_OIDC_PROVIDER_BASE_URL: http://localhost:8000/fake
+      QUALICHARGE_OAUTH2_TOKEN_ENCODING_KEY: thisissupersecret
+      QUALICHARGE_OAUTH2_TOKEN_ISSUER: http://test:8000
+      QUALICHARGE_EXECUTION_ENVIRONMENT: ci
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Create postgis extension
+        run: psql "postgresql://qualicharge:pass@localhost:5432/qualicharge-api" -c "create extension postgis;"
+      - name: Install pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+          cache-dependency-path: "src/api/Pipfile.lock"
+      - name: Run database migrations
+        run: pipenv run alembic -c qualicharge/alembic.ini upgrade head
+      - name: Create API superuser
+        run: |
+          pipenv run python -m qualicharge create-user \
+            admin \
+            --email [email protected] \
+            --password admin \
+            --is-active \
+            --is-superuser \
+            --is-staff \
+            --force
+      - name: Seed API database
+        run: |
+          pipenv install -d --skip-lock qualicharge-client
+          pipenv run honcho start &
+          sleep 10
+          zcat ../../data/irve-statique.json.gz | \
+            head -n 500 | \
+            pipenv run qcc static bulk --chunk-size 100
+        env:
+          QCC_API_LOGIN_USERNAME: admin
+          QCC_API_LOGIN_PASSWORD: admin
+          QCC_API_ROOT_URL: "http://localhost:8000/api/v1"
+      # API server is still running here
+      - name: Run locust
+        run: |
+          pipenv run locust \
+            -f ../bench/locustfile.py \
+            --headless \
+            -u 30 \
+            -r 1 \
+            --run-time 30s \
+            -H "http://localhost:${PORT}/api/v1" \
+            --csv bench_admin \
+            --exit-code-on-error 0 \
+            APIAdminUser
+      - name: Add bench file metadata
+        run: |
+          pipenv run \
+            python ../bench/cli.py \
+              stamp bench_admin_stats.csv $(git rev-parse --short "${GITHUB_SHA}") \
+                > bench_admin_stats_stamped.csv
+      - name: Save bench CSV as artefact
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-admin-benchmark
+          path: ./src/api/bench_admin_stats_stamped.csv
+      - name: Generate markdown table
+        run: |
+          echo -e "### Current benchmark\n\n" >> bench_admin_stats.md && \
+          pipenv run csvlook -I bench_admin_stats_stamped.csv >> bench_admin_stats.md && \
+          echo -e "\n### Comparison with the latest previous benchmark\n\n" >> bench_admin_stats.md && \
+          echo -e "> A lower (negative) value means the current version performs better than the previous one.\n\n" >> bench_admin_stats.md && \
+          pipenv run \
+            python ../bench/cli.py diff ../../data/bench.csv bench_admin_stats_stamped.csv | \
+            pipenv run \
+              csvlook -I >> bench_admin_stats.md
+          cat bench_admin_stats.md
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('node:fs');
+            fs.readFile('/home/runner/work/qualicharge/qualicharge/src/api/bench_admin_stats.md', 'utf8', (err, data) => {
+              if (err) {
+                console.error(err);
+                return;
+              }
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: data
+              });
+            });
+
+  # Only when a PR is merged
+  update-bench-db:
+    if: github.event.pull_request.merged == true
+    needs:
+      - build-api
+      - bench-api
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: write
+    defaults:
+      run:
+        working-directory: ./src/api
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Install pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+          cache-dependency-path: "src/api/Pipfile.lock"
+      - name: Get latest bench CSV artefact
+        uses: actions/download-artifact@v4
+        with:
+          name: api-admin-benchmark
+          path: ./src/api
+      - name: Merge Bench database
+        run: |
+          pipenv run \
+            csvstack \
+              ../../data/bench.csv \
+              bench_admin_stats_stamped.csv \
+                > /tmp/bench.csv
+          cp -f /tmp/bench.csv ../../data/bench.csv
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          add-paths: |
+            data/bench.csv
+          commit-message: |
+            ⚡️(api) update benchmark database
+
+            Update bench database.
+          branch: update-api-bench-db
+          title: "⚡️(api) update benchmark database"
+          body: |
+            ## Purpose
+
+            Each time a PR is merged and a new benchmark has been released, the bench database is updated.
+
+            ## Proposal
+
+            - [x] update `data/bench.csv`
+
+          labels: |
+            API
+            needs review
+
   test-database-migrations:
     needs: build-api
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -49,6 +49,7 @@ scripts/*.csv
 
 # -- Tools
 .coverage
+src/api/bench_*.csv
 
 # Prefect
 src/prefect/.htpasswd

diff --git a/Makefile b/Makefile
@@ -30,6 +30,29 @@ data/afirev-charging.csv: data
 	@echo "You should download CSV file from $(AFIREV_CHARGING_DATASET_URL)"
 
 # -- Docker/compose
+bench-reset-db: ## Reset API database to run benchmark
+	$(COMPOSE) stop
+	$(COMPOSE) up -d --wait --force-recreate postgresql
+	$(MAKE) migrate-api
+	$(MAKE) create-api-superuser
+	$(COMPOSE) up -d --wait api
+	zcat data/irve-statique.json.gz | head -n 500 | \
+		bin/qcc static bulk --chunk-size 1000
+.PHONY: bench-reset-db
+
+bench: ## run API benchmark
+	$(COMPOSE_RUN_API_PIPENV) \
+		locust \
+		  -f /mnt/bench/locustfile.py \
+			--headless \
+			-u 30 \
+			-r 1 \
+			--run-time 30s \
+			-H 'http://api:8000/api/v1' \
+			--csv bench_admin \
+			APIAdminUser
+.PHONY: bench
+
 bootstrap: ## bootstrap the project for development
 bootstrap: \
   build \
@@ -61,6 +84,10 @@ build-client: ## build the client image
 	$(COMPOSE) build client
 .PHONY: build-client
 
+build-locust: ## build locust image
+	@$(COMPOSE) build locust
+.PHONY: build-locust
+
 build-notebook: ## build custom jupyter notebook image
 	@$(COMPOSE) build notebook
 .PHONY: build-notebook
@@ -134,6 +161,10 @@ run-dashboard: ## run the dashboard service
 	$(COMPOSE_UP) dashboard
 .PHONY: run-dashboard
 
+run-locust: ## run the locust service
+	$(COMPOSE_UP) --wait locust-worker
+.PHONY: run-locust
+
 status: ## an alias for "docker compose ps"
 	@$(COMPOSE) ps
 .PHONY: status
@@ -285,7 +316,6 @@ jupytext--to-ipynb: ## convert remote md files into ipynb
 
 reset-db: ## Reset the PostgreSQL database
 	$(COMPOSE) stop
-	$(COMPOSE) down postgresql metabase
 	$(MAKE) migrate-api
 	$(MAKE) create-api-superuser
 	$(MAKE) create-api-test-db
@@ -339,6 +369,7 @@ seed-dashboard: ## seed dashboard
 lint: ## lint all sources
 lint: \
 	lint-api \
+	lint-bench \
 	lint-client \
 	lint-prefect \
 	lint-dashboard
@@ -351,6 +382,13 @@ lint-api: \
   lint-api-mypy
 .PHONY: lint-api
 
+lint-bench: ## lint api python sources
+lint-bench: \
+  lint-bench-black \
+  lint-bench-ruff \
+  lint-bench-mypy
+.PHONY: lint-bench
+
 lint-client: ## lint client python sources
 lint-client: \
   lint-client-black \
@@ -393,6 +431,26 @@ lint-api-mypy: ## lint api python sources with mypy
 	@$(COMPOSE_RUN_API_PIPENV) mypy qualicharge tests
 .PHONY: lint-api-mypy
 
+lint-bench-black: ## lint bench python sources with black
+	@echo 'lint:black started…'
+	@$(COMPOSE_RUN_API_PIPENV) black /mnt/bench
+.PHONY: lint-bench-black
+
+lint-bench-ruff: ## lint bench python sources with ruff
+	@echo 'lint:ruff started…'
+	@$(COMPOSE_RUN_API_PIPENV) ruff check /mnt/bench
+.PHONY: lint-bench-ruff
+
+lint-bench-ruff-fix: ## lint and fix api python sources with ruff
+	@echo 'lint:ruff-fix started…'
+	@$(COMPOSE_RUN_API_PIPENV) ruff check --fix /mnt/bench
+.PHONY: lint-bench-ruff-fix
+
+lint-bench-mypy: ## lint bench python sources with mypy
+	@echo 'lint:mypy started…'
+	@$(COMPOSE_RUN_API_PIPENV) mypy /mnt/bench
+.PHONY: lint-bench-mypy
+
 lint-client-black: ## lint api python sources with black
 	@echo 'lint:black started…'
 	@$(COMPOSE_RUN_CLIENT) black qcc tests

diff --git a/data/bench.csv b/data/bench.csv
@@ -0,0 +1,5 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%,git,timestamp,version
+GET,/auth/whoami,449,0,41.0,190.682040076386,3.025668993359432,1548.9330790005624,160.0,15.502850350234926,0.0,41,250,290,320,530,760,920,1000,1500,1500,1500,e4bea93,2024-11-25 15:53:10.141468+00:00,0.15.0
+GET,/statique/?limit=10,415,0,190.0,309.7470774553037,15.45504099340178,1597.348899987992,13822.0,14.328915134404218,0.0,190,330,510,550,830,1000,1200,1400,1600,1600,1600,e4bea93,2024-11-25 15:53:10.141468+00:00,0.15.0
+GET,/statique/?limit=100,424,0,320.0,451.6924511111904,96.59699301118962,1808.202302985592,129002.0,14.639662691535875,0.0,320,510,590,720,1000,1100,1600,1700,1800,1800,1800,e4bea93,2024-11-25 15:53:10.141468+00:00,0.15.0
+,Aggregated,1288,0,220.0,314.9680686408331,3.025668993359432,1808.202302985592,46975.79037267081,44.47142817617502,0.0,230,320,500,540,810,1000,1200,1500,1800,1800,1800,e4bea93,2024-11-25 15:53:10.141468+00:00,0.15.0