From b6ac88b1a4e8c11a0569887dda0fb76bdac816e3 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 2 Aug 2024 19:35:37 +0100 Subject: [PATCH 1/6] Adding prefect --- docker/prefect/profiles.toml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docker/prefect/profiles.toml diff --git a/docker/prefect/profiles.toml b/docker/prefect/profiles.toml new file mode 100644 index 00000000..c709a055 --- /dev/null +++ b/docker/prefect/profiles.toml @@ -0,0 +1,22 @@ +[profiles] + + +[profiles.arpav-ppcv-server-dev] +PREFECT_API_URL = "http://0.0.0.0:4200/api" +PREFECT_CLI_PROMPT = false +PREFECT_DEBUG_MODE = true +PREFECT_HOME = "/prefect_home" +PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = false +PREFECT_SERVER_API_HOST = "0.0.0.0" +PREFECT_SERVER_API_PORT = "4200" +PREFECT_SERVER_CSRF_PROTECTION_ENABLED = true +PREFECT_UI_API_URL = "http://localhost:8877/prefect/api" +PREFECT_UI_URL = "http://localhost:8877/prefect" +PREFECT_UI_SERVE_BASE = "/prefect" + + +[profiles.arpav-ppcv-worker] +PREFECT_API_URL = "http://prefect-server:4200/api" +PREFECT_DEBUG_MODE = true +PREFECT_HOME = "/prefect_home" +PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = false From 0440ed02b0c62ed972ac8669928779d9d125965a Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Tue, 13 Aug 2024 19:00:01 +0100 Subject: [PATCH 2/6] Tweaking prefect setup so that it works in the stack --- docker/prefect/profiles.toml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/docker/prefect/profiles.toml b/docker/prefect/profiles.toml index c709a055..ed68bc0b 100644 --- a/docker/prefect/profiles.toml +++ b/docker/prefect/profiles.toml @@ -1,22 +1,7 @@ -[profiles] - +active = "ephemeral" [profiles.arpav-ppcv-server-dev] PREFECT_API_URL = "http://0.0.0.0:4200/api" -PREFECT_CLI_PROMPT = false -PREFECT_DEBUG_MODE = true -PREFECT_HOME = "/prefect_home" -PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = false -PREFECT_SERVER_API_HOST = "0.0.0.0" -PREFECT_SERVER_API_PORT = "4200" -PREFECT_SERVER_CSRF_PROTECTION_ENABLED = true -PREFECT_UI_API_URL = "http://localhost:8877/prefect/api" -PREFECT_UI_URL = "http://localhost:8877/prefect" -PREFECT_UI_SERVE_BASE = "/prefect" - [profiles.arpav-ppcv-worker] PREFECT_API_URL = "http://prefect-server:4200/api" -PREFECT_DEBUG_MODE = true -PREFECT_HOME = "/prefect_home" -PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = false From e0f25220822ead8b9a13842020c7dc0f3f1c106c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Wed, 14 Aug 2024 18:47:42 +0100 Subject: [PATCH 3/6] Added prefect flows for ingesting observations data --- docker/compose.dev.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml index 305c40e6..d4f35973 100644 --- a/docker/compose.dev.yaml +++ b/docker/compose.dev.yaml @@ -189,7 +189,6 @@ services: ] environment: PREFECT_API_URL: "http://prefect-server:4200/api" - ARPAV_PPCV__PREFECT__OBSERVATION_STATIONS_REFRESHER_FLOW_CRON_SCHEDULE: "*/5 * * * *" PREFECT_DEBUG_MODE: true <<: *common-env volumes: From f13456d6d347fb0fe9467561c5f8638e55cc4dac Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 15 Aug 2024 13:51:03 +0100 Subject: [PATCH 4/6] Removed prefect profiles configuration This is no longer needed thanks to prefect 3rc17 --- docker/compose.dev.yaml | 1 + docker/prefect/profiles.toml | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 docker/prefect/profiles.toml diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml index d4f35973..305c40e6 100644 --- a/docker/compose.dev.yaml +++ b/docker/compose.dev.yaml @@ -189,6 +189,7 @@ services: ] environment: PREFECT_API_URL: "http://prefect-server:4200/api" + ARPAV_PPCV__PREFECT__OBSERVATION_STATIONS_REFRESHER_FLOW_CRON_SCHEDULE: "*/5 * * * *" PREFECT_DEBUG_MODE: true <<: *common-env volumes: diff --git a/docker/prefect/profiles.toml b/docker/prefect/profiles.toml deleted file mode 100644 index ed68bc0b..00000000 --- a/docker/prefect/profiles.toml +++ /dev/null @@ -1,7 +0,0 @@ -active = "ephemeral" - -[profiles.arpav-ppcv-server-dev] -PREFECT_API_URL = "http://0.0.0.0:4200/api" - -[profiles.arpav-ppcv-worker] -PREFECT_API_URL = "http://prefect-server:4200/api" From 637a1103f37c8fe4d2f46db7b6eab4f2d4182f83 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 15 Aug 2024 17:07:36 +0100 Subject: [PATCH 5/6] Added HTTP basic auth to protect access to prefect --- docker/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/compose.yaml b/docker/compose.yaml index 2b8b0a2a..7ad3b810 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -64,7 +64,7 @@ services: source: /var/run/docker.sock target: /var/run/docker.sock - type: bind - source: ${ARPAV_PPCV_TRAEFIK_BASIC_AUTH_USERS_FILE:-/$PWD/docker/traefik/basicauth-users.txt} + source: ${ARPAV_PPCV_TRAEFIK_BASIC_AUTH_USERS_FILE:-$PWD/docker/traefik/basicauth-users.txt} target: /traefikauth/usersfile frontend: From e540c7d1e7a0b68e020fcb673ae35d2b2fd37b4e Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Wed, 21 Aug 2024 14:54:20 +0100 Subject: [PATCH 6/6] Added missing docker-related config for prefect-based services --- docker/compose.dev.yaml | 8 -------- docker/compose.staging.yaml | 29 +++++++++++++++++++++++++++++ docker/compose.yaml | 7 +++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml index 305c40e6..ea225b52 100644 --- a/docker/compose.dev.yaml +++ b/docker/compose.dev.yaml @@ -180,16 +180,8 @@ services: prefect-static-worker: image: *webapp-image - command: [ - "start-periodic-tasks", - "--refresh-stations", - "--refresh-monthly-measurements", - "--refresh-seasonal-measurements", - "--refresh-yearly-measurements", - ] environment: PREFECT_API_URL: "http://prefect-server:4200/api" - ARPAV_PPCV__PREFECT__OBSERVATION_STATIONS_REFRESHER_FLOW_CRON_SCHEDULE: "*/5 * * * *" PREFECT_DEBUG_MODE: true <<: *common-env volumes: diff --git a/docker/compose.staging.yaml b/docker/compose.staging.yaml index 96377e5e..23e10a10 100644 --- a/docker/compose.staging.yaml +++ b/docker/compose.staging.yaml @@ -19,6 +19,9 @@ x-env-file-db: &env-file-db ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_DB_SERVICE:-/home/a x-env-file-webapp: &env-file-webapp ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_WEBAPP_SERVICE:-/home/arpav/environment-files/webapp-service.env} x-env-file-martin: &env-file-martin ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_MARTIN_SERVICE:-/home/arpav/environment-files/martin-service.env} x-env-file-frontend: &env-file-frontend ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_FRONTEND_SERVICE:-/home/arpav/environment-files/frontend-service.env} +x-env-file-prefect-db: &env-file-prefect-db ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_PREFECT_DB_SERVICE:-/home/arpav/environment-files/prefect-db-service.env} +x-env-file-prefect-server: &env-file-prefect-server ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_PREFECT_SERVER_SERVICE:-/home/arpav/environment-files/prefect-server-service.env} +x-env-file-prefect-static-worker: &env-file-prefect-static-worker ${ARPAV_PPCV_DEPLOYMENT_ENV_FILE_PREFECT_STATIC_WORKER_SERVICE:-/home/arpav/environment-files/prefect-static-worker-service.env} name: arpav-ppcv-staging @@ -126,5 +129,31 @@ services: target: /additional restart: unless-stopped + prefect-server: + networks: + - back + labels: + - "traefik.http.routers.prefect-router.entrypoints=web" + - "exposed.inside=true" + env_file: + - *env-file-prefect-server + + prefect-static-worker: + networks: + - back + env_file: + - *env-file-prefect-static-worker + restart: unless-stopped + + prefect-db: + networks: + - back + env_file: + - *env-file-prefect-db + volumes: + - prefect-db-data:/var/lib/postgresql/data + restart: unless-stopped + volumes: db-data: + prefect-db-data: diff --git a/docker/compose.yaml b/docker/compose.yaml index 7ad3b810..20ca47a6 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -121,6 +121,13 @@ services: prefect-static-worker: image: *backend-image entrypoint: ["tini", "-g", "--", "poetry", "run", "arpav-ppcv", "prefect"] + command: [ + "start-periodic-tasks", + "--refresh-stations", + "--refresh-monthly-measurements", + "--refresh-seasonal-measurements", + "--refresh-yearly-measurements", + ] depends_on: prefect-server: condition: service_healthy