✨(prefect) add prefect base config and test flow

Add prefect to our stack to run and orchestrate data pipelines.

.github/workflows/prefect.yml

@@ -0,0 +1,91 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Prefect
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  build-prefect:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+      - name: Install dependencies
+        run: |
+          cd src/prefect
+          pipenv install -de .
+
+  lint-prefect:
+    needs: build-prefect
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src/prefect
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+      - name: Lint with Black
+        run: pipenv run black indicators tests
+      - name: Lint with Ruff
+        run: pipenv run ruff check indicators tests
+      - name: Lint with MyPy
+        run: pipenv run mypy indicators tests
+
+  test-prefect:
+    needs: build-prefect
+    runs-on: ubuntu-latest
+    services:
+      postgresql:
+        image: timescale/timescaledb-ha:pg14-ts2.14-oss
+        env:
+          POSTGRES_DB: qualicharge-api
+          POSTGRES_USER: qualicharge
+          POSTGRES_PASSWORD: pass
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    defaults:
+      run:
+        working-directory: ./src/prefect
+    env:
+      QUALICHARGE_DATABASE_URL: "postgresql+psycopg://qualicharge:pass@localhost:5432/qualicharge-api"
+      QUALICHARGE_DATABASE_URL_NO_DRIVER: "postgresql://qualicharge:pass@localhost:5432/qualicharge-api"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Restore database dump
+        run: |
+          pg_restore -s -F c -d "${QUALICHARGE_DATABASE_URL_NO_DRIVER}" ../../data/qualicharge-api-schema.sql
+          pg_restore -a -F c -d "${QUALICHARGE_DATABASE_URL_NO_DRIVER}" ../../data/qualicharge-api-data.sql
+      - name: Install pipenv
+        run: pipx install pipenv
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pipenv"
+      - name: Test with pytest
+        run: pipenv run pytest

.gitignore

@@ -50,6 +50,9 @@ scripts/*.csv
 # -- Tools
 .coverage
 
+# Prefect
+src/prefect/.htpasswd
+
 # Jupytext: we version converted md files not ipynb sources
 *.ipynb
 

Makefile

@@ -2,12 +2,13 @@
 SHELL := /bin/bash
 
 # -- Docker
-COMPOSE                = bin/compose
-COMPOSE_UP             = $(COMPOSE) up -d
-COMPOSE_RUN            = $(COMPOSE) run --rm --no-deps
-COMPOSE_RUN_API        = $(COMPOSE_RUN) api
-COMPOSE_RUN_API_PIPENV = $(COMPOSE_RUN_API) pipenv run
-COMPOSE_RUN_CLIENT     = $(COMPOSE_RUN) client
+COMPOSE                    = bin/compose
+COMPOSE_UP                 = $(COMPOSE) up -d
+COMPOSE_RUN                = $(COMPOSE) run --rm --no-deps
+COMPOSE_RUN_API            = $(COMPOSE_RUN) api
+COMPOSE_RUN_API_PIPENV     = $(COMPOSE_RUN_API) pipenv run
+COMPOSE_RUN_CLIENT         = $(COMPOSE_RUN) client
+COMPOSE_RUN_PREFECT_PIPENV = $(COMPOSE_RUN) prefect pipenv run
 
 # -- Tools
 CURL = $(COMPOSE_RUN) curl
@@ -34,6 +35,7 @@ bootstrap: \
   migrate-api \
   create-api-test-db \
   create-metabase-db \
+  create-prefect-db \
   seed-metabase \
   seed-oidc \
   create-superuser \
@@ -61,6 +63,10 @@ build-opendata: ## build opendata image
 	@$(COMPOSE) build opendata
 .PHONY: build-opendata
 
+build-prefect: ## build prefect image
+	@$(COMPOSE) build prefect
+.PHONY: build-prefect
+
 down: ## stop and remove all containers
 	@$(COMPOSE) down
 .PHONY: down
@@ -81,6 +87,10 @@ logs-opendata: ## display opendata logs (follow mode)
 	@$(COMPOSE) logs -f opendata
 .PHONY: logs-opendata
 
+logs-prefect: ## display prefect logs (follow mode)
+	@$(COMPOSE) logs -f prefect prefect-worker
+.PHONY: logs-prefect
+
 run: ## run the api server (and dependencies)
 	$(COMPOSE_UP) --wait api
 .PHONY: run
@@ -101,6 +111,11 @@ run-opendata: ## run the opendata service
 	$(COMPOSE_UP) opendata
 .PHONY: run-opendata
 
+run-prefect: ## run the prefect service
+	$(COMPOSE_UP) --wait prefect
+	$(COMPOSE_UP) prefect-worker
+.PHONY: run-prefect
+
 status: ## an alias for "docker compose ps"
 	@$(COMPOSE) ps
 .PHONY: status
@@ -110,6 +125,36 @@ stop: ## stop all servers
 .PHONY: stop
 
 # -- Provisioning
+data/qualicharge-api-schema.sql:
+	$(COMPOSE) exec postgresql pg_dump -s -Z 9 -U qualicharge -F c qualicharge-api > data/qualicharge-api-schema.sql
+
+data/qualicharge-api-data.sql:
+	$(COMPOSE) exec postgresql pg_dump -a -Z 9 -U qualicharge -F c qualicharge-api > data/qualicharge-api-data.sql
+
+backup-api-db: ## create API database backup
+backup-api-db: \
+	data/qualicharge-api-schema.sql \
+  data/qualicharge-api-data.sql
+.PHONY: backup-api-db
+
+restore-api-db-data: ## restore API database backup data
+restore-api-db-data: data/qualicharge-api-data.sql
+	cat data/qualicharge-api-data.sql | \
+		$(COMPOSE) exec -T postgresql pg_restore -a -U qualicharge -F c -d qualicharge-api
+.PHONY: restore-api-db-data
+
+restore-api-db-schema: ## restore API database backup schema
+restore-api-db-schema: data/qualicharge-api-schema.sql
+	cat data/qualicharge-api-schema.sql | \
+		$(COMPOSE) exec -T postgresql pg_restore -s -U qualicharge -F c -d qualicharge-api
+.PHONY: restore-api-db-schema
+
+restore-api-db: ## restore API database backup
+restore-api-db: \
+	restore-api-db-schema \
+	restore-api-db-data
+.PHONY: restore-api-db
+
 create-api-test-db: ## create API test database
 	@echo "Creating api service test database…"
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "create database \"$${QUALICHARGE_TEST_DB_NAME}\";"' || echo "Duly noted, skipping database creation."
@@ -121,6 +166,11 @@ create-metabase-db: ## create metabase database
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "create database \"$${MB_DB_DBNAME}\";"' || echo "Duly noted, skipping database creation."
 .PHONY: create-metabase-db
 
+create-prefect-db: ## create prefect database
+	@echo "Creating prefect service database…"
+	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "create database \"$${PREFECT_API_DATABASE_NAME}\";"' || echo "Duly noted, skipping database creation."
+.PHONY: create-prefect-db
+
 drop-api-test-db: ## drop API test database
 	@echo "Droping api service test database…"
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "drop database \"$${QUALICHARGE_TEST_DB_NAME}\";"' || echo "Duly noted, skipping database deletion."
@@ -180,12 +230,12 @@ jupytext--to-ipynb: ## convert remote md files into ipynb
 reset-db: ## Reset the PostgreSQL database
 	$(COMPOSE) stop
 	$(COMPOSE) down postgresql metabase
-	$(COMPOSE_UP) --force-recreate api
 	$(MAKE) migrate-api
 	$(MAKE) create-superuser
 	$(MAKE) create-api-test-db
 	$(MAKE) create-metabase-db
 	$(MAKE) seed-metabase
+	$(MAKE) create-prefect-db
 .PHONY: reset-db
 
 seed-api: ## seed the API database (static data)
@@ -220,7 +270,8 @@ seed-oidc: ## seed the OIDC provider
 lint: ## lint all sources
 lint: \
 	lint-api \
-	lint-client
+	lint-client \
+	lint-prefect
 .PHONY: lint
 
 lint-api: ## lint api python sources
@@ -237,6 +288,13 @@ lint-client: \
   lint-client-mypy
 .PHONY: lint-client
 
+lint-prefect: ## lint api python sources
+lint-prefect: \
+  lint-prefect-black \
+  lint-prefect-ruff \
+  lint-prefect-mypy
+.PHONY: lint-prefect
+
 lint-api-black: ## lint api python sources with black
 	@echo 'lint:black started…'
 	@$(COMPOSE_RUN_API_PIPENV) black qualicharge tests
@@ -277,11 +335,31 @@ lint-client-mypy: ## lint api python sources with mypy
 	@$(COMPOSE_RUN_CLIENT) mypy qcc tests
 .PHONY: lint-client-mypy
 
+lint-prefect-black: ## lint prefect python sources with black
+	@echo 'lint:black started…'
+	@$(COMPOSE_RUN_PREFECT_PIPENV) black indicators tests
+.PHONY: lint-prefect-black
+
+lint-prefect-ruff: ## lint prefect python sources with ruff
+	@echo 'lint:ruff started…'
+	@$(COMPOSE_RUN_PREFECT_PIPENV) ruff check indicators tests
+.PHONY: lint-prefect-ruff
+
+lint-prefect-ruff-fix: ## lint and fix prefect python sources with ruff
+	@echo 'lint:ruff-fix started…'
+	@$(COMPOSE_RUN_PREFECT_PIPENV) ruff check --fix indicators tests
+.PHONY: lint-prefect-ruff-fix
+
+lint-prefect-mypy: ## lint prefect python sources with mypy
+	@echo 'lint:mypy started…'
+	@$(COMPOSE_RUN_PREFECT_PIPENV) mypy indicators tests
+.PHONY: lint-prefect-mypy
 
 test: ## run all services tests
 test: \
 	test-api \
-	test-client
+	test-client \
+	test-prefect
 .PHONY: test
 
 test-api: ## run API tests
@@ -292,6 +370,10 @@ test-client: ## run client tests
 	SERVICE=client bin/pytest
 .PHONY: test-client
 
+test-prefect: ## run prefect tests
+	SERVICE=prefect-test bin/pytest
+.PHONY: test-prefect
+
 # -- Misc
 help:
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

bin/htpasswd

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+docker compose run \
+    httpd \
+    htpasswd "$@"

bin/prefect

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+declare DOCKER_USER
+DOCKER_UID="$(id -u)"
+DOCKER_GID="$(id -g)"
+DOCKER_USER="${DOCKER_UID}:${DOCKER_GID}"
+
+DOCKER_USER=${DOCKER_USER} \
+  DOCKER_UID=${DOCKER_UID} \
+  DOCKER_GID=${DOCKER_GID} \
+  docker compose exec \
+    prefect \
+    pipenv run prefect "$@"

bin/pytest

@@ -9,10 +9,10 @@ DOCKER_UID="$(id -u)"
 DOCKER_GID="$(id -g)"
 DOCKER_USER="${DOCKER_UID}:${DOCKER_GID}"
 
-if [ "${service}" == "api" ]; then
-  cmd="pipenv run pytest";
-else
+if [ "${service}" == "client" ]; then
   cmd="pytest";
+else
+  cmd="pipenv run pytest";
 fi
 
 DOCKER_USER=${DOCKER_USER} \

docker-compose.yml

@@ -5,6 +5,7 @@ services:
       - env.d/postgresql
       - env.d/api
       - env.d/metabase
+      - env.d/prefect
     healthcheck:
       test:
         - "CMD-SHELL"
@@ -123,6 +124,46 @@ services:
     depends_on:
       - postgresql
 
+  prefect:
+    build:
+      context: ./src/prefect
+      args:
+        DOCKER_UID: ${DOCKER_UID:-1000}
+        DOCKER_GID: ${DOCKER_GID:-1000}
+    user: ${DOCKER_USER:-1000}
+    image: "qualicharge:prefect"
+    ports:
+      - 4200:4200
+    env_file:
+      - env.d/prefect
+      - env.d/prefect-qc
+    volumes:
+      - ./src/prefect:/app
+    depends_on:
+      - postgresql
+
+  prefect-worker:
+    image: "qualicharge:prefect"
+    command: ["pipenv", "run", "honcho", "start", "worker"]
+    user: ${DOCKER_USER:-1000}
+    env_file:
+      - env.d/prefect
+      - env.d/prefect-qc
+    volumes:
+      - ./src/prefect:/app
+    depends_on:
+      - prefect
+
+  prefect-test:
+    image: "qualicharge:prefect"
+    user: ${DOCKER_USER:-1000}
+    env_file:
+      - env.d/prefect-qc
+    volumes:
+      - ./src/prefect:/app
+    depends_on:
+      - postgresql
+
   # -- tools
   curl:
     image: curlimages/curl:8.10.1
@@ -135,6 +176,9 @@ services:
   dockerize:
     image: jwilder/dockerize
 
+  httpd:
+    image: httpd:alpine
+
   jq:
     # There is no other tags than latest
     image: ddev/ddev-utilities:latest