From 152bcd7d87074d04d14c8c796e67950538e01af3 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 21 Nov 2024 18:19:59 +0000 Subject: [PATCH 1/4] Adding production-related deployment files --- deployments/deploy.py | 566 ++++++++++++++++++++++++ docker/compose.production.template.yaml | 168 +++++++ 2 files changed, 734 insertions(+) create mode 100644 deployments/deploy.py create mode 100644 docker/compose.production.template.yaml diff --git a/deployments/deploy.py b/deployments/deploy.py new file mode 100644 index 00000000..94231e69 --- /dev/null +++ b/deployments/deploy.py @@ -0,0 +1,566 @@ +"""Deployment script + +This is used for managing semi-automated deployments of the system. +""" + +# NOTE: IN ORDER TO SIMPLIFY DEPLOYMENT, THIS SCRIPT SHALL ONLY USE STUFF FROM THE +# PYTHON STANDARD LIBRARY + +import argparse +import dataclasses +import configparser +import json +import logging +import os +import shlex +import shutil +import socket +import subprocess +import sys +import urllib.request +from pathlib import Path +from string import Template +from typing import Protocol +from urllib.error import HTTPError + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class DeploymentConfiguration: + backend_image: str + db_image_tag: str + db_name: str + db_password: str + db_user: str + deployment_files_repo: str + deployment_root: Path + discord_notification_urls: list[str] + frontend_image: str + martin_conf_path: Path + martin_env_database_url: str + martin_image_tag: str + prefect_db_name: str + prefect_db_password: str + prefect_db_user: str + prefect_server_env_allow_ephemeral_mode: bool + prefect_server_env_api_database_connection_url: str + prefect_server_env_api_host: str + prefect_server_env_api_port: int + prefect_server_env_api_url: str + prefect_server_env_cli_prompt: bool + prefect_server_env_csrf_protection_enabled: bool + prefect_server_env_debug_mode: bool + prefect_server_env_home: Path + prefect_server_env_serve_base: str + prefect_server_env_ui_api_url: str + prefect_server_env_ui_url: str + prefect_server_image_tag: str + prefect_static_worker_env_api_url: str + prefect_static_worker_env_db_dsn: str + prefect_static_worker_env_debug: bool + prefect_static_worker_env_debug_mode: bool + reverse_proxy_image_tag: str + tls_cert_path: Path + tls_cert_key_path: Path + tolgee_app_env_server_port: int + tolgee_app_env_server_spring_datasource_url: str + tolgee_app_env_spring_datasource_password: str + tolgee_app_env_spring_datasource_username: str + tolgee_app_env_tolgee_authentication_create_demo_for_initial_user: bool + tolgee_app_env_tolgee_authentication_enabled: bool + tolgee_app_env_tolgee_authentication_initial_password: str + tolgee_app_env_tolgee_authentication_jwt_secret: str + tolgee_app_env_tolgee_file_storage_fs_data_path: Path + tolgee_app_env_tolgee_frontend_url: str + tolgee_app_env_tolgee_postgres_autostart_enabled: bool + tolgee_app_env_tolgee_telemetry_enabled: bool + tolgee_app_image_tag: str + tolgee_db_name: str + tolgee_db_password: str + tolgee_db_user: str + traefik_conf_path: Path + webapp_env_admin_user_password: str + webapp_env_admin_user_username: str + webapp_env_allow_cors_credentials: bool + webapp_env_bind_host: str + webapp_env_bind_port: int + webapp_env_cors_methods: list[str] + webapp_env_cors_origins: list[str] + webapp_env_db_dsn: str + webapp_env_debug: bool + webapp_env_num_uvicorn_worker_processes: int + webapp_env_public_url: str + webapp_env_session_secret_key: str + webapp_env_thredds_server_base_url: str + webapp_env_uvicorn_log_config_file: Path + compose_project_name: str = "arpav-cline" + executable_webapp_service_name: str = "arpav-cline-webapp-1" + git_repo_clone_destination: Path = Path("/tmp/arpav-cline") + + @classmethod + def from_config_parser(cls, config_parser: configparser.ConfigParser): + return cls( + backend_image=config_parser["main"]["backend_image"], + db_image_tag=config_parser["main"]["db_image_tag"], + db_name=config_parser["db"]["name"], + db_password=config_parser["db"]["password"], + db_user=config_parser["db"]["user"], + deployment_files_repo=config_parser["main"]["deployment_files_repo"], + deployment_root=Path(config_parser["main"]["deployment_root"]), + discord_notification_urls=[ + i.strip() + for i in config_parser["main"]["discord_notification_urls"].split(",") + ], + frontend_image=config_parser["main"]["frontend_image"], + martin_conf_path=Path(config_parser["martin"]["conf_path"]), + martin_env_database_url=config_parser["martin"]["env_database_url"], + martin_image_tag=config_parser["martin"]["image_tag"], + prefect_db_name=config_parser["prefect_db"]["name"], + prefect_db_password=config_parser["prefect_db"]["password"], + prefect_db_user=config_parser["prefect_db"]["user"], + prefect_server_env_allow_ephemeral_mode=config_parser.getboolean( + "prefect_server", "env_allow_ephemeral_mode" + ), + prefect_server_env_api_database_connection_url=config_parser[ + "prefect_server" + ]["env_api_database_connection_url"], + prefect_server_env_api_host=config_parser["prefect_server"]["env_api_host"], + prefect_server_env_api_port=config_parser.getint( + "prefect_server", "env_api_port" + ), + prefect_server_env_api_url=config_parser["prefect_server"]["env_api_url"], + prefect_server_env_cli_prompt=config_parser.getboolean( + "prefect_server", "env_cli_prompt" + ), + prefect_server_env_csrf_protection_enabled=config_parser.getboolean( + "prefect_server", "env_csrf_protection_enabled" + ), + prefect_server_env_debug_mode=config_parser.getboolean( + "prefect_server", "env_debug_mode" + ), + prefect_server_env_home=Path(config_parser["prefect_server"]["env_home"]), + prefect_server_env_serve_base=config_parser["prefect_server"][ + "env_serve_base" + ], + prefect_server_env_ui_api_url=config_parser["prefect_server"][ + "env_ui_api_url" + ], + prefect_server_env_ui_url=config_parser["prefect_server"]["env_ui_url"], + prefect_server_image_tag=config_parser["prefect_server"]["image_tag"], + prefect_static_worker_env_api_url=config_parser["prefect_static_worker"][ + "env_api_url" + ], + prefect_static_worker_env_db_dsn=config_parser["prefect_static_worker"][ + "env_db_dsn" + ], + prefect_static_worker_env_debug=config_parser.getboolean( + "prefect_static_worker", "env_debug" + ), + prefect_static_worker_env_debug_mode=config_parser.getboolean( + "prefect_static_worker", "env_debug_mode" + ), + reverse_proxy_image_tag=config_parser["reverse_proxy"]["image_tag"], + tls_cert_path=Path(config_parser["reverse_proxy"]["tls_cert_path"]), + tls_cert_key_path=Path(config_parser["reverse_proxy"]["tls_cert_key_path"]), + tolgee_app_env_server_port=config_parser.getint( + "tolgee_app", "env_server_port" + ), + tolgee_app_env_server_spring_datasource_url=config_parser["tolgee_app"][ + "env_server_spring_datasource_url" + ], + tolgee_app_env_spring_datasource_password=config_parser["tolgee_app"][ + "env_spring_datasource_password" + ], + tolgee_app_env_spring_datasource_username=config_parser["tolgee_app"][ + "env_spring_datasource_username" + ], + tolgee_app_env_tolgee_authentication_create_demo_for_initial_user=config_parser.getboolean( + "tolgee_app", "env_tolgee_authentication_create_demo_for_initial_user" + ), + tolgee_app_env_tolgee_authentication_enabled=config_parser.getboolean( + "tolgee_app", "env_tolgee_authentication_enabled" + ), + tolgee_app_env_tolgee_authentication_initial_password=config_parser[ + "tolgee_app" + ]["env_tolgee_authentication_initial_password"], + tolgee_app_env_tolgee_authentication_jwt_secret=config_parser["tolgee_app"][ + "env_tolgee_authentication_jwt_secret" + ], + tolgee_app_env_tolgee_file_storage_fs_data_path=Path( + config_parser["tolgee_app"]["env_tolgee_file_storage_fs_path"] + ), + tolgee_app_env_tolgee_frontend_url=config_parser["tolgee_app"][ + "env_tolgee_frontend_url" + ], + tolgee_app_env_tolgee_postgres_autostart_enabled=config_parser.getboolean( + "tolgee_app", "env_tolgee_postgres_autostart_enabled" + ), + tolgee_app_env_tolgee_telemetry_enabled=config_parser.getboolean( + "tolgee_app", "env_tolgee_telemetry_enabled" + ), + tolgee_app_image_tag=config_parser["tolgee_app"]["image_tag"], + tolgee_db_name=config_parser["tolgee_db"]["name"], + tolgee_db_password=config_parser["tolgee_db"]["password"], + tolgee_db_user=config_parser["tolgee_db"]["user"], + traefik_conf_path=Path(config_parser["reverse_proxy"]["traefik_conf_path"]), + webapp_env_admin_user_password=config_parser["webapp"][ + "env_admin_user_password" + ], + webapp_env_admin_user_username=config_parser["webapp"][ + "env_admin_user_username" + ], + webapp_env_allow_cors_credentials=config_parser.getboolean( + "webapp", "env_allow_cors_credentials" + ), + webapp_env_bind_host=config_parser["webapp"]["env_bind_host"], + webapp_env_bind_port=config_parser.getint("webapp", "env_bind_port"), + webapp_env_cors_methods=[ + m.strip() + for m in config_parser["webapp"]["env_cors_methods"].split(",") + ], + webapp_env_cors_origins=[ + o.strip() + for o in config_parser["webapp"]["env_cors_origins"].split(",") + ], + webapp_env_db_dsn=config_parser["webapp"]["env_db_dsn"], + webapp_env_debug=config_parser.getboolean("webapp", "env_debug"), + webapp_env_num_uvicorn_worker_processes=config_parser.getint( + "webapp", "env_num_uvicorn_worker_processes" + ), + webapp_env_public_url=config_parser["webapp"]["env_public_url"], + webapp_env_session_secret_key=config_parser["webapp"][ + "env_session_secret_key" + ], + webapp_env_thredds_server_base_url=config_parser["webapp"][ + "env_thredds_server_base_url" + ], + webapp_env_uvicorn_log_config_file=Path( + config_parser["webapp"]["env_uvicorn_log_config_file"] + ), + ) + + def ensure_paths_exist(self): + paths_to_test = ( + self.deployment_root, + self.martin_conf_path, + self.prefect_server_env_home, + self.tls_cert_path, + self.tls_cert_key_path, + self.tolgee_app_env_tolgee_file_storage_fs_data_path, + self.traefik_conf_path, + self.webapp_env_uvicorn_log_config_file, + ) + for path in paths_to_test: + if not path.exists(): + raise RuntimeError( + f"Could not find referenced configuration file {path!r}" + ) + + +class DeployStepProtocol(Protocol): + name: str + config: DeploymentConfiguration + + def handle(self) -> None: + ... + + +@dataclasses.dataclass +class _CloneRepo: + config: DeploymentConfiguration + name: str = "clone git repository to a temporary directory" + + def handle(self) -> None: + print("Cloning repo...") + if self.config.git_repo_clone_destination.exists(): + shutil.rmtree(self.config.git_repo_clone_destination) + subprocess.run( + shlex.split( + f"git clone {self.config.deployment_files_repo} " + f"{self.config.git_repo_clone_destination}" + ), + check=True, + ) + + +@dataclasses.dataclass +class _CopyRelevantRepoFiles: + config: DeploymentConfiguration + name: str = ( + "Copy files relevant to the deployment from temporary git clone " + "to target location" + ) + + def handle(self) -> None: + to_copy_paths = ( + self.config.git_repo_clone_destination / "deployments/deploy.py", + self.config.git_repo_clone_destination / "docker/compose.yaml", + self.config.git_repo_clone_destination + / "docker/compose.production.template.yaml", + ) + for to_copy_path in to_copy_paths: + if not to_copy_path.exists(): + raise RuntimeError(f"Could not find expected file {to_copy_path!r}") + else: + shutil.copyfile( + to_copy_path, self.config.deployment_root / to_copy_path.name + ) + + +@dataclasses.dataclass +class _CreateDeploymentReadme: + config: DeploymentConfiguration + name: str = "Create deployment README file" + + def handle(self) -> None: + contents = """ + ## Deployment README + + This directory contains the following deployment-related files: + + - compose.yaml - Base docker compose file, to be used together with `compose.production.yaml` + - compose.production.yaml - Production-specific compose file, to be used together with the base `compose.yaml` file + - deploy.py - Deployment script, which can be used to trigger a deployment + + Relevant actions that can be taken using the files in this directory: + + - Stand up/bring down the system - make use of the existing docker compose files, like this + + ```shell + # stand up the system + docker compose -f compose.yaml -f compose.production.yaml up --detach --force-recreate + + # bring down the system + docker compose -f compose.yaml -f compose.production.yaml down + ``` + + - (Re)deploy the system - Call the `deploy.py` python module, like this: + + ```shell + # get help on how to call the command + python3 deploy --help + + # redeploy the system + python3 deploy --configuration-file + ``` + """.strip() + target_path = Path(self.config.deployment_root) / "README.md" + target_path.write_text(contents) + + +@dataclasses.dataclass +class _RelaunchDeploymentScript: + config: DeploymentConfiguration + original_call_args: list[str] + name: str = "Relaunch the updated deployment script" + + def handle(self) -> None: + call_args = self.original_call_args[:] + if (update_flag_index := args.index("--auto-update")) != -1: + call_args.pop(update_flag_index) + os.execv(sys.executable, call_args) + + +@dataclasses.dataclass +class _GenerateComposeFile: + config: DeploymentConfiguration + name: str = "generate docker compose file" + + def handle(self) -> None: + compose_teplate_path = ( + self.config.deployment_root / "compose.production.template.yaml" + ) + compose_template = Template(compose_teplate_path.read_text()) + rendered = compose_template.substitute(dataclasses.asdict(self.config)) + target_path = Path( + self.config.deployment_root / "docker/compose.production.yaml" + ) + target_path.write_text(rendered) + compose_teplate_path.unlink(missing_ok=True) + + +@dataclasses.dataclass +class _ComposeCommandExecutor: + config: DeploymentConfiguration + environment: dict[str, str] | None = None + + def handle(self) -> None: + raise NotImplementedError + + def _run_compose_command(self, suffix: str) -> subprocess.CompletedProcess: + compose_files = [ + self.config.deployment_root / "compose.yaml", + self.config.deployment_root / "compose.production.yaml", + ] + compose_files_fragment = " ".join(f"-f {p}" for p in compose_files) + return subprocess.run( + shlex.split(f"docker compose {compose_files_fragment} {suffix}"), + cwd=self.config.deployment_root, + env=self.environment or os.environ, + check=True, + ) + + +class _StartCompose(_ComposeCommandExecutor): + name: str = "start docker compose" + + def handle(self) -> None: + print("Restarting the docker compose stack...") + self._run_compose_command("up --detach --force-recreate") + + +class _StopCompose(_ComposeCommandExecutor): + name: str = "stop docker compose" + + def handle(self) -> None: + print("Stopping docker compose stack...") + run_result = self._run_compose_command("down") + if run_result.returncode == 14: + logger.info("docker compose stack was not running, no need to stop") + else: + run_result.check_returncode() + + +@dataclasses.dataclass +class _PullImages(_ComposeCommandExecutor): + name: str = "pull new docker images from their respective container registries" + + def handle(self) -> None: + self._run_compose_command("pull") + + +@dataclasses.dataclass +class _CompileTranslations: + config: DeploymentConfiguration + name: str = "compile static translations" + + def handle(self) -> None: + print("Compiling translations...") + subprocess.run( + shlex.split( + f"docker exec {self.config.executable_webapp_service_name} poetry run " + f"arpav-ppcv translations compile" + ), + check=True, + ) + + +@dataclasses.dataclass +class _RunMigrations: + config: DeploymentConfiguration + name: str = "run DB migrations" + + def handle(self) -> None: + print("Upgrading database...") + subprocess.run( + shlex.split( + f"docker exec {self.config.executable_webapp_service_name} poetry run " + f"arpav-ppcv db upgrade" + ), + check=True, + ) + + +@dataclasses.dataclass +class _SendDiscordChannelNotification: + config: DeploymentConfiguration + content: str + name: str = "send a notification to a discord channel" + + def handle(self) -> None: + for webhook_url in self.config.discord_notification_urls: + request = urllib.request.Request(webhook_url, method="POST") + request.add_header("Content-Type", "application/json") + + # the discord server blocks the default user-agent sent by urllib, the + # one sent by httpx works, so we just use that + request.add_header("User-Agent", "python-httpx/0.27.0") + try: + print(f"Sending notification to {webhook_url!r}...") + with urllib.request.urlopen( + request, data=json.dumps({"content": self.content}).encode("utf-8") + ) as response: + if 200 <= response.status <= 299: + print("notification sent") + else: + print( + f"notification response was not successful: {response.status}" + ) + except HTTPError: + print("sending notification failed") + + +def get_configuration(config_file: Path) -> DeploymentConfiguration: + config_parser = configparser.ConfigParser() + config_parser.read(config_file) + return DeploymentConfiguration.from_config_parser(config_parser) + + +def perform_deployment( + *, + configuration: DeploymentConfiguration, + confirmed: bool = False, +): + logger.info(f"{configuration=}") + deployment_steps = [ + _CreateDeploymentReadme(config=configuration), + _CloneRepo(config=configuration), + _CopyRelevantRepoFiles(config=configuration), + _RelaunchDeploymentScript(config=configuration, original_call_args=sys.argv), + _StopCompose(config=configuration), + _GenerateComposeFile(config=configuration), + _PullImages(config=configuration), + _StartCompose(config=configuration), + _RunMigrations(config=configuration), + _CompileTranslations(config=configuration), + ] + this_host = socket.gethostname() + if len(configuration.discord_notification_urls) > 0: + deployment_steps.append( + _SendDiscordChannelNotification( + config=configuration, + content=( + f"A new deployment of ARPAV-Cline to {this_host!r} has finished" + ), + ) + ) + if not confirmed: + print("Performing a dry-run") + for step in deployment_steps: + print(f"Running step: {step.name!r}...") + if confirmed: + step.handle() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--config-file", + default=f"{Path.home()}/arpav-cline/production-deployment.cfg", + help="Path to configuration file", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Turn on debug logging level", + ) + parser.add_argument( + "--confirm", + action="store_true", + help=( + "Perform the actual deployment. If this is not provided the script runs " + "in dry-run mode, just showing what steps would be performed" + ), + ) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING) + deployment_config = get_configuration(args.config_file) + deployment_config.ensure_paths_exist() + try: + perform_deployment( + configuration=deployment_config, + confirmed=args.confirm, + ) + except RuntimeError as err: + raise SystemExit(err) from err diff --git a/docker/compose.production.template.yaml b/docker/compose.production.template.yaml new file mode 100644 index 00000000..36690273 --- /dev/null +++ b/docker/compose.production.template.yaml @@ -0,0 +1,168 @@ +# docker compose file template that can be used to create a suitable production deployment file. +# This template file is meant to be processed via Python - the processor shall replace all +# template strings, which are denoted like this: ${name} with their respective values. +# +# - do not mount source code inside any container - keep volume binds to the +# minimum, only for relevant configuration file(s) and data collections + +name: arpav-cline-production + +services: + + reverse-proxy: + image: "traefik:${reverse_proxy_image_tag}" + command: --configFile /traefik.toml + labels: + - "traefik.enable=true" + - "traefik.http.routers.router-arpav-backend.entrypoints=webSecure" + - "traefik.http.routers.router-arpav-backend.tls=true" + - "traefik.http.routers.router-arpav-backend.tls.certificates.certFile=/run/secrets/tls-cert" + - "traefik.http.routers.router-arpav-backend.tls.certificates.keyFile=/run/secrets/tls-key" + - 'traefik.http.routers.router-arpav-backend.rule=HostRegexp(`^(.+\.)?clima\.arpa\.veneto.it$`)' + configs: + - source: traefik-conf + target: /traefik.toml + secrets: + - tls-cert + - tls-key + restart: unless-stopped + + frontend: + image: "${frontend_image}" + labels: + - "traefik.http.routers.arpav-frontend.entrypoints=web" + - "traefik.http.routers.arpav-frontend-router.rule=Host(`clima.arpa.veneto.it`) && !PathRegexp(`^/(api|admin|prefect|vector-tiles)`)" + restart: unless-stopped + + webapp: + image: "${backend_image}" + environment: + ARPAV_PPCV__DEBUG: "${webapp_env_debug}" + ARPAV_PPCV__BIND_HOST: "${webapp_env_bind_host}" + ARPAV_PPCV__BIND_PORT: "${webapp_env_bind_port}" + ARPAV_PPCV__PUBLIC_URL: "${webapp_env_public_url}" + ARPAV_PPCV__NUM_UVICORN_WORKER_PROCESSES: "${webapp_env_num_uvicorn_worker_processes}" + ARPAV_PPCV__DB_DSN: "${webapp_env_db_dsn}" + ARPAV_PPCV__UVICORN_LOG_CONFIG_FILE: "${webapp_env_uvicorn_log_config_file}" + ARPAV_PPCV__SESSION_SECRET_KEY: "${webapp_env_session_secret_key}" + ARPAV_PPCV__ADMIN_USER__USERNAME: "${webapp_env_admin_user_username}" + ARPAV_PPCV__ADMIN_USER__PASSWORD: "${webapp_env_admin_user_password}" + ARPAV_PPCV__THREDDS_SERVER__BASE_URL: "${webapp_env_thredds_server_base_url}" + ARPAV_PPCV__CORS_ORIGINS: "${webapp_env_cors_origins}" + ARPAV_PPCV__CORS_METHODS: "${webapp_env_cors_methods}" + ARPAV_PPCV__ALLOW_CORS_CREDENTIALS: "${webapp_env_allow_cors_credentials}" + labels: + - "traefik.http.routers.arpav-backend.entrypoints=web" + - "traefik.http.routers.arpav-backend-router.rule=Host(`clima.arpa.veneto.it`) && PathRegexp(`^/(api|admin)`)" + restart: unless-stopped + + db: + image: "postgis/postgis:${db_image_tag}" + environment: + POSTGRES_PASSWORD: "${db_password}" + POSTGRES_USER: "${db_user}" + POSTGRES_DB: "${db_name}" + volumes: + - db-data:/var/lib/postgresql/data + restart: unless-stopped + + martin: + image: "ghcr.io/maplibre/martin:${martin_image_tag}" + environment: + DATABASE_URL: "${martin_env_database_url}" + configs: + - martin-conf + labels: + - "traefik.http.routers.martin-router.entrypoints=web" + - "traefik.http.routers.martin-router.rule=Host(`clima.arpa.veneto.it`) && PathPrefix(`/vector-tiles`)" + restart: unless-stopped + + prefect-server: + image: "prefecthq/prefect:${prefect_server_image_tag}" + labels: + - "traefik.http.routers.prefect-router.entrypoints=web" + - "traefik.http.routers.prefect-router.rule=Host(`clima.arpa.veneto.it`) && PathPrefix(`/prefect`)" + environment: + PREFECT_API_DATABASE_CONNECTION_URL: "${prefect_server_env_api_database_connection_url}" + PREFECT_API_URL: "${prefect_server_env_api_url}" + PREFECT_CLI_PROMPT: "${prefect_server_env_cli_prompt}" + PREFECT_DEBUG_MODE: "${prefect_server_env_debug_mode}" + PREFECT_HOME: "${prefect_server_env_home}" + PREFECT_SERVER_ALLOW_EPHEMERAL_MODE: "${prefect_server_env_allow_ephemeral_mode}" + PREFECT_SERVER_API_HOST: "${prefect_server_env_api_host}" + PREFECT_SERVER_API_PORT: "${prefect_server_env_api_port}" + PREFECT_SERVER_CSRF_PROTECTION_ENABLED: "${prefect_server_env_csrf_protection_enabled}" + PREFECT_UI_API_URL: "${prefect_server_env_ui_api_url}" + PREFECT_UI_URL: "${prefect_server_env_ui_url}" + PREFECT_UI_SERVE_BASE: "${prefect_server_env_serve_base}" + restart: unless-stopped + + prefect-static-worker: + image: "${backend_image}" + environment: + ARPAV_PPCV__DEBUG: "${prefect_static_worker_env_debug}" + ARPAV_PPCV__DB_DSN: "${prefect_static_worker_env_db_dsn}" + PREFECT_API_URL: "${prefect_static_worker_env_api_url}" + PREFECT_DEBUG_MODE: "${prefect_static_worker_env_debug_mode}" + restart: unless-stopped + + prefect-db: + image: "postgis/postgis:${db_image_tag}" + environment: + POSTGRES_PASSWORD: "${prefect_db_password}" + POSTGRES_USER: "${prefect_db_user}" + POSTGRES_DB: "${prefect_db_name}" + volumes: + - prefect-db-data:/var/lib/postgresql/data + restart: unless-stopped + + tolgee-app: + image: "tolgee/tolgee:${tolgee_app_image_tag}" + labels: + - "traefik.http.routers.arpav-frontend.entrypoints=web" + - "traefik.http.routers.tolgee-app-router.rule=Host(`tolgee.clima.arpa.veneto.it`) && PathPrefix(`/`)" + environment: + SERVER_PORT: "${tolgee_app_env_server_port}" + SPRING_DATASOURCE_URL: "${tolgee_app_env_server_spring_datasource_url}" + SPRING_DATASOURCE_USERNAME: "${tolgee_app_env_spring_datasource_username}" + SPRING_DATASOURCE_PASSWORD: "${tolgee_app_env_spring_datasource_password}" + TOLGEE_AUTHENTICATION_CREATE_DEMO_FOR_INITIAL_USER: "${tolgee_app_env_tolgee_authentication_create_demo_for_initial_user}" + TOLGEE_AUTHENTICATION_ENABLED: "${tolgee_app_env_tolgee_authentication_enabled}" + TOLGEE_AUTHENTICATION_INITIAL_PASSWORD: "${tolgee_app_env_tolgee_authentication_initial_password}" + TOLGEE_AUTHENTICATION_JWT_SECRET: "${tolgee_app_env_tolgee_authentication_jwt_secret}" + TOLGEE_FILE_STORAGE_FS_DATA_PATH: "${tolgee_app_env_tolgee_file_storage_fs_data_path}" + TOLGEE_FRONTEND_URL: "${tolgee_app_env_tolgee_frontend_url}" + TOLGEE_POSTGRES_AUTOSTART_ENABLED: "${tolgee_app_env_tolgee_postgres_autostart_enabled}" + TOLGEE_TELEMETRY_ENABLED: "${tolgee_app_env_tolgee_telemetry_enabled}" + restart: unless-stopped + + tolgee-db: + image: "postgis/postgis:${db_image_tag}" + environment: + POSTGRES_PASSWORD: "${tolgee_db_password}" + POSTGRES_USER: "${tolgee_db_user}" + POSTGRES_DB: "${tolgee_db_name}" + volumes: + - tolgee-db-data:/var/lib/postgresql/data + restart: unless-stopped + +volumes: + db-data: + prefect-db-data: + tolgee-db-data: + + +configs: + traefik-conf: + file: "${traefik_conf_path}" + + martin-conf: + file: "${martin_conf_path}" + + +secrets: + tls-cert: + file: "${tls_cert_path}" + + tls-key: + file: "${tls_cert_key_path}" From df3026942bc90a9fa533a95e6d96d546d2380fa1 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 21 Nov 2024 18:22:12 +0000 Subject: [PATCH 2/4] Adding production-related deployment files --- deployments/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/deploy.py b/deployments/deploy.py index 94231e69..433759b3 100644 --- a/deployments/deploy.py +++ b/deployments/deploy.py @@ -504,10 +504,10 @@ def perform_deployment( ): logger.info(f"{configuration=}") deployment_steps = [ - _CreateDeploymentReadme(config=configuration), _CloneRepo(config=configuration), _CopyRelevantRepoFiles(config=configuration), _RelaunchDeploymentScript(config=configuration, original_call_args=sys.argv), + _CreateDeploymentReadme(config=configuration), _StopCompose(config=configuration), _GenerateComposeFile(config=configuration), _PullImages(config=configuration), From fc2f1dc9a274b93a458e75b2decb073a2143eb1b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 22 Nov 2024 16:07:18 +0000 Subject: [PATCH 3/4] Adding production-related deployment files --- .gitignore | 2 + deployments/deploy.py | 351 +++++++++++------------- deployments/sample-prod-deployment.cfg | 131 +++++++++ docker/compose.production.template.yaml | 6 +- docker/traefik/production-config.toml | 29 ++ 5 files changed, 329 insertions(+), 190 deletions(-) create mode 100644 deployments/sample-prod-deployment.cfg create mode 100644 docker/traefik/production-config.toml diff --git a/.gitignore b/.gitignore index de66422c..60040f62 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,5 @@ celerybeat-schedule docker/traefik/basicauth-users.txt arpav-cache + +prod-deployment.cfg diff --git a/deployments/deploy.py b/deployments/deploy.py index 433759b3..68b420ae 100644 --- a/deployments/deploy.py +++ b/deployments/deploy.py @@ -29,6 +29,7 @@ @dataclasses.dataclass class DeploymentConfiguration: backend_image: str + compose_project_name: str = dataclasses.field(init=False) db_image_tag: str db_name: str db_password: str @@ -36,67 +37,133 @@ class DeploymentConfiguration: deployment_files_repo: str deployment_root: Path discord_notification_urls: list[str] + executable_webapp_service_name: str = dataclasses.field(init=False) frontend_image: str - martin_conf_path: Path - martin_env_database_url: str + git_repo_clone_destination: Path = dataclasses.field(init=False) + martin_conf_path: Path = dataclasses.field( + init=False + ) # is copied to inside the deployment_root dir + martin_env_database_url: str = dataclasses.field(init=False) martin_image_tag: str prefect_db_name: str prefect_db_password: str prefect_db_user: str - prefect_server_env_allow_ephemeral_mode: bool - prefect_server_env_api_database_connection_url: str - prefect_server_env_api_host: str - prefect_server_env_api_port: int - prefect_server_env_api_url: str - prefect_server_env_cli_prompt: bool - prefect_server_env_csrf_protection_enabled: bool - prefect_server_env_debug_mode: bool - prefect_server_env_home: Path - prefect_server_env_serve_base: str - prefect_server_env_ui_api_url: str - prefect_server_env_ui_url: str + prefect_server_env_allow_ephemeral_mode: bool = dataclasses.field(init=False) + prefect_server_env_api_database_connection_url: str = dataclasses.field(init=False) + prefect_server_env_api_host: str = dataclasses.field(init=False) + prefect_server_env_api_port: int = dataclasses.field(init=False) + prefect_server_env_api_url: str = dataclasses.field(init=False) + prefect_server_env_cli_prompt: bool = dataclasses.field(init=False) + prefect_server_env_csrf_protection_enabled: bool = dataclasses.field(init=False) + prefect_server_env_debug_mode: bool = dataclasses.field(init=False) + prefect_server_env_home: str = dataclasses.field(init=False) + prefect_server_env_ui_api_url: str = dataclasses.field(init=False) + prefect_server_env_ui_serve_base: str = dataclasses.field(init=False) + prefect_server_env_ui_url: str = dataclasses.field(init=False) prefect_server_image_tag: str - prefect_static_worker_env_api_url: str - prefect_static_worker_env_db_dsn: str - prefect_static_worker_env_debug: bool - prefect_static_worker_env_debug_mode: bool + prefect_static_worker_env_arpav_ppcv_db_dsn: str = dataclasses.field(init=False) + prefect_static_worker_env_arpav_ppcv_debug: bool = dataclasses.field(init=False) + prefect_static_worker_env_prefect_api_url: str = dataclasses.field(init=False) + prefect_static_worker_env_prefect_debug_mode: bool = dataclasses.field(init=False) reverse_proxy_image_tag: str tls_cert_path: Path tls_cert_key_path: Path - tolgee_app_env_server_port: int - tolgee_app_env_server_spring_datasource_url: str - tolgee_app_env_spring_datasource_password: str - tolgee_app_env_spring_datasource_username: str - tolgee_app_env_tolgee_authentication_create_demo_for_initial_user: bool - tolgee_app_env_tolgee_authentication_enabled: bool + tolgee_app_env_server_port: int = dataclasses.field(init=False) + tolgee_app_env_server_spring_datasource_url: str = dataclasses.field(init=False) + tolgee_app_env_spring_datasource_password: str = dataclasses.field(init=False) + tolgee_app_env_spring_datasource_username: str = dataclasses.field(init=False) + tolgee_app_env_tolgee_authentication_create_demo_for_initial_user: bool = ( + dataclasses.field(init=False) + ) + tolgee_app_env_tolgee_authentication_enabled: bool = dataclasses.field(init=False) tolgee_app_env_tolgee_authentication_initial_password: str tolgee_app_env_tolgee_authentication_jwt_secret: str - tolgee_app_env_tolgee_file_storage_fs_data_path: Path + tolgee_app_env_tolgee_file_storage_fs_data_path: str = dataclasses.field(init=False) tolgee_app_env_tolgee_frontend_url: str - tolgee_app_env_tolgee_postgres_autostart_enabled: bool - tolgee_app_env_tolgee_telemetry_enabled: bool + tolgee_app_env_tolgee_postgres_autostart_enabled: bool = dataclasses.field( + init=False + ) + tolgee_app_env_tolgee_telemetry_enabled: bool = dataclasses.field(init=False) tolgee_app_image_tag: str tolgee_db_name: str tolgee_db_password: str tolgee_db_user: str - traefik_conf_path: Path + traefik_conf_path: Path = dataclasses.field( + init=False + ) # is copied to inside the deployment_root dir + traefik_users_file_path: Path webapp_env_admin_user_password: str webapp_env_admin_user_username: str - webapp_env_allow_cors_credentials: bool - webapp_env_bind_host: str - webapp_env_bind_port: int - webapp_env_cors_methods: list[str] + webapp_env_allow_cors_credentials: bool = dataclasses.field(init=False) + webapp_env_bind_host: str = dataclasses.field(init=False) + webapp_env_bind_port: int = dataclasses.field(init=False) + webapp_env_cors_methods: list[str] = dataclasses.field(init=False) webapp_env_cors_origins: list[str] - webapp_env_db_dsn: str - webapp_env_debug: bool + webapp_env_db_dsn: str = dataclasses.field(init=False) + webapp_env_debug: bool = dataclasses.field(init=False) webapp_env_num_uvicorn_worker_processes: int webapp_env_public_url: str webapp_env_session_secret_key: str webapp_env_thredds_server_base_url: str webapp_env_uvicorn_log_config_file: Path - compose_project_name: str = "arpav-cline" - executable_webapp_service_name: str = "arpav-cline-webapp-1" - git_repo_clone_destination: Path = Path("/tmp/arpav-cline") + + def __post_init__(self): + _debug = False + self.compose_project_name = "arpav-cline" + self.executable_webapp_service_name = "arpav-cline-webapp-1" + self.git_repo_clone_destination = Path("/tmp/arpav-cline") + self.martin_conf_path = self.deployment_root / "martin-config.yaml" + self.traefik_conf_path = self.deployment_root / "traefik-config.toml" + self.martin_env_database_url = ( + f"postgresql://{self.db_user}:{self.db_password}@db:5432/{self.db_name}" + ) + self.prefect_server_env_api_database_connection_url = ( + f"postgresql+asyncpg://{self.prefect_db_user}:{self.prefect_db_password}@" + f"prefect_db/{self.prefect_db_name}" + ) + self.prefect_server_env_api_host = "0.0.0.0" + self.prefect_server_env_api_port = 4200 + self.prefect_server_env_allow_ephemeral_mode = False + self.prefect_server_env_api_url = ( + f"http://{self.prefect_server_env_api_host}:" + f"{self.prefect_server_env_api_port}/api" + ) + self.prefect_server_env_cli_prompt = False + self.prefect_server_env_csrf_protection_enabled = True + self.prefect_server_env_debug_mode = _debug + self.prefect_server_env_home = "/prefect_home" + self.prefect_server_env_ui_api_url = f"{self.webapp_env_public_url}/prefect/api" + self.prefect_server_env_ui_serve_base = "/prefect/ui" + self.prefect_server_env_ui_url = ( + f"{self.webapp_env_public_url}{self.prefect_server_env_ui_serve_base}" + ) + self.prefect_static_worker_env_arpav_ppcv_db_dsn = ( + f"postgresql://{self.db_user}:{self.db_password}@db:5432/{self.db_name}" + ) + self.prefect_static_worker_env_arpav_ppcv_debug = _debug + self.prefect_static_worker_env_prefect_api_url = ( + f"http://prefect-server:{self.prefect_server_env_api_port}/api" + ) + self.prefect_static_worker_env_prefect_debug_mode = _debug + self.tolgee_app_env_server_port = 8080 + self.tolgee_app_env_server_spring_datasource_url = ( + f"jdbc:postgresql://tolgee-db:5432/{self.tolgee_db_name}" + ) + self.tolgee_app_env_spring_datasource_password = self.tolgee_db_password + self.tolgee_app_env_spring_datasource_username = self.tolgee_db_user + self.tolgee_app_env_tolgee_authentication_create_demo_for_initial_user = False + self.tolgee_app_env_tolgee_authentication_enabled = True + self.tolgee_app_env_tolgee_file_storage_fs_data_path = "/data" + self.tolgee_app_env_tolgee_postgres_autostart_enabled = False + self.tolgee_app_env_tolgee_telemetry_enabled = False + self.webapp_env_allow_cors_credentials = True + self.webapp_env_bind_host = "0.0.0.0" + self.webapp_env_bind_port = 5001 + self.webapp_env_cors_methods = ["*"] + self.webapp_env_db_dsn = ( + f"postgresql://{self.db_user}:{self.db_password}@db:5432/{self.db_name}" + ) + self.webapp_env_debug = _debug @classmethod def from_config_parser(cls, config_parser: configparser.ConfigParser): @@ -113,118 +180,40 @@ def from_config_parser(cls, config_parser: configparser.ConfigParser): for i in config_parser["main"]["discord_notification_urls"].split(",") ], frontend_image=config_parser["main"]["frontend_image"], - martin_conf_path=Path(config_parser["martin"]["conf_path"]), - martin_env_database_url=config_parser["martin"]["env_database_url"], - martin_image_tag=config_parser["martin"]["image_tag"], + martin_image_tag=config_parser["main"]["martin_image_tag"], prefect_db_name=config_parser["prefect_db"]["name"], prefect_db_password=config_parser["prefect_db"]["password"], prefect_db_user=config_parser["prefect_db"]["user"], - prefect_server_env_allow_ephemeral_mode=config_parser.getboolean( - "prefect_server", "env_allow_ephemeral_mode" - ), - prefect_server_env_api_database_connection_url=config_parser[ - "prefect_server" - ]["env_api_database_connection_url"], - prefect_server_env_api_host=config_parser["prefect_server"]["env_api_host"], - prefect_server_env_api_port=config_parser.getint( - "prefect_server", "env_api_port" - ), - prefect_server_env_api_url=config_parser["prefect_server"]["env_api_url"], - prefect_server_env_cli_prompt=config_parser.getboolean( - "prefect_server", "env_cli_prompt" - ), - prefect_server_env_csrf_protection_enabled=config_parser.getboolean( - "prefect_server", "env_csrf_protection_enabled" - ), - prefect_server_env_debug_mode=config_parser.getboolean( - "prefect_server", "env_debug_mode" - ), - prefect_server_env_home=Path(config_parser["prefect_server"]["env_home"]), - prefect_server_env_serve_base=config_parser["prefect_server"][ - "env_serve_base" - ], - prefect_server_env_ui_api_url=config_parser["prefect_server"][ - "env_ui_api_url" - ], - prefect_server_env_ui_url=config_parser["prefect_server"]["env_ui_url"], - prefect_server_image_tag=config_parser["prefect_server"]["image_tag"], - prefect_static_worker_env_api_url=config_parser["prefect_static_worker"][ - "env_api_url" - ], - prefect_static_worker_env_db_dsn=config_parser["prefect_static_worker"][ - "env_db_dsn" - ], - prefect_static_worker_env_debug=config_parser.getboolean( - "prefect_static_worker", "env_debug" - ), - prefect_static_worker_env_debug_mode=config_parser.getboolean( - "prefect_static_worker", "env_debug_mode" - ), + prefect_server_image_tag=config_parser["main"]["prefect_server_image_tag"], reverse_proxy_image_tag=config_parser["reverse_proxy"]["image_tag"], tls_cert_path=Path(config_parser["reverse_proxy"]["tls_cert_path"]), tls_cert_key_path=Path(config_parser["reverse_proxy"]["tls_cert_key_path"]), - tolgee_app_env_server_port=config_parser.getint( - "tolgee_app", "env_server_port" - ), - tolgee_app_env_server_spring_datasource_url=config_parser["tolgee_app"][ - "env_server_spring_datasource_url" - ], - tolgee_app_env_spring_datasource_password=config_parser["tolgee_app"][ - "env_spring_datasource_password" - ], - tolgee_app_env_spring_datasource_username=config_parser["tolgee_app"][ - "env_spring_datasource_username" - ], - tolgee_app_env_tolgee_authentication_create_demo_for_initial_user=config_parser.getboolean( - "tolgee_app", "env_tolgee_authentication_create_demo_for_initial_user" - ), - tolgee_app_env_tolgee_authentication_enabled=config_parser.getboolean( - "tolgee_app", "env_tolgee_authentication_enabled" - ), tolgee_app_env_tolgee_authentication_initial_password=config_parser[ "tolgee_app" ]["env_tolgee_authentication_initial_password"], tolgee_app_env_tolgee_authentication_jwt_secret=config_parser["tolgee_app"][ "env_tolgee_authentication_jwt_secret" ], - tolgee_app_env_tolgee_file_storage_fs_data_path=Path( - config_parser["tolgee_app"]["env_tolgee_file_storage_fs_path"] - ), tolgee_app_env_tolgee_frontend_url=config_parser["tolgee_app"][ "env_tolgee_frontend_url" ], - tolgee_app_env_tolgee_postgres_autostart_enabled=config_parser.getboolean( - "tolgee_app", "env_tolgee_postgres_autostart_enabled" - ), - tolgee_app_env_tolgee_telemetry_enabled=config_parser.getboolean( - "tolgee_app", "env_tolgee_telemetry_enabled" - ), tolgee_app_image_tag=config_parser["tolgee_app"]["image_tag"], tolgee_db_name=config_parser["tolgee_db"]["name"], tolgee_db_password=config_parser["tolgee_db"]["password"], tolgee_db_user=config_parser["tolgee_db"]["user"], - traefik_conf_path=Path(config_parser["reverse_proxy"]["traefik_conf_path"]), + traefik_users_file_path=Path( + config_parser["reverse_proxy"]["traefik_users_file_path"] + ), webapp_env_admin_user_password=config_parser["webapp"][ "env_admin_user_password" ], webapp_env_admin_user_username=config_parser["webapp"][ "env_admin_user_username" ], - webapp_env_allow_cors_credentials=config_parser.getboolean( - "webapp", "env_allow_cors_credentials" - ), - webapp_env_bind_host=config_parser["webapp"]["env_bind_host"], - webapp_env_bind_port=config_parser.getint("webapp", "env_bind_port"), - webapp_env_cors_methods=[ - m.strip() - for m in config_parser["webapp"]["env_cors_methods"].split(",") - ], webapp_env_cors_origins=[ o.strip() for o in config_parser["webapp"]["env_cors_origins"].split(",") ], - webapp_env_db_dsn=config_parser["webapp"]["env_db_dsn"], - webapp_env_debug=config_parser.getboolean("webapp", "env_debug"), webapp_env_num_uvicorn_worker_processes=config_parser.getint( "webapp", "env_num_uvicorn_worker_processes" ), @@ -243,13 +232,8 @@ def from_config_parser(cls, config_parser: configparser.ConfigParser): def ensure_paths_exist(self): paths_to_test = ( self.deployment_root, - self.martin_conf_path, - self.prefect_server_env_home, self.tls_cert_path, self.tls_cert_key_path, - self.tolgee_app_env_tolgee_file_storage_fs_data_path, - self.traefik_conf_path, - self.webapp_env_uvicorn_log_config_file, ) for path in paths_to_test: if not path.exists(): @@ -291,62 +275,42 @@ class _CopyRelevantRepoFiles: "Copy files relevant to the deployment from temporary git clone " "to target location" ) + deployment_related_files = ( + "deployments/deploy.py", + "docker/compose.yaml", + "docker/compose.production.template.yaml", + ) + martin_conf_file = "docker/martin/config.yaml" + traefik_conf_file = "docker/traefik/production-config.toml" def handle(self) -> None: - to_copy_paths = ( - self.config.git_repo_clone_destination / "deployments/deploy.py", - self.config.git_repo_clone_destination / "docker/compose.yaml", - self.config.git_repo_clone_destination - / "docker/compose.production.template.yaml", + to_copy_martin_conf_file_path = ( + self.config.git_repo_clone_destination / self.martin_conf_file ) - for to_copy_path in to_copy_paths: + to_copy_traefik_conf_file_path = ( + self.config.git_repo_clone_destination / self.traefik_conf_file + ) + to_copy_deployment_related_file_paths = [ + self.config.git_repo_clone_destination / i + for i in self.deployment_related_files + ] + all_files_to_copy = ( + *to_copy_deployment_related_file_paths, + to_copy_martin_conf_file_path, + to_copy_traefik_conf_file_path, + ) + for to_copy_path in all_files_to_copy: if not to_copy_path.exists(): - raise RuntimeError(f"Could not find expected file {to_copy_path!r}") - else: - shutil.copyfile( - to_copy_path, self.config.deployment_root / to_copy_path.name + raise RuntimeError( + f"Could not find expected file in the previously cloned " + f"git repo: {to_copy_path!r}" ) - - -@dataclasses.dataclass -class _CreateDeploymentReadme: - config: DeploymentConfiguration - name: str = "Create deployment README file" - - def handle(self) -> None: - contents = """ - ## Deployment README - - This directory contains the following deployment-related files: - - - compose.yaml - Base docker compose file, to be used together with `compose.production.yaml` - - compose.production.yaml - Production-specific compose file, to be used together with the base `compose.yaml` file - - deploy.py - Deployment script, which can be used to trigger a deployment - - Relevant actions that can be taken using the files in this directory: - - - Stand up/bring down the system - make use of the existing docker compose files, like this - - ```shell - # stand up the system - docker compose -f compose.yaml -f compose.production.yaml up --detach --force-recreate - - # bring down the system - docker compose -f compose.yaml -f compose.production.yaml down - ``` - - - (Re)deploy the system - Call the `deploy.py` python module, like this: - - ```shell - # get help on how to call the command - python3 deploy --help - - # redeploy the system - python3 deploy --configuration-file - ``` - """.strip() - target_path = Path(self.config.deployment_root) / "README.md" - target_path.write_text(contents) + for to_copy_path in to_copy_deployment_related_file_paths: + shutil.copyfile( + to_copy_path, self.config.deployment_root / to_copy_path.name + ) + shutil.copyfile(to_copy_martin_conf_file_path, self.config.martin_conf_path) + shutil.copyfile(to_copy_traefik_conf_file_path, self.config.traefik_conf_path) @dataclasses.dataclass @@ -502,12 +466,10 @@ def perform_deployment( configuration: DeploymentConfiguration, confirmed: bool = False, ): - logger.info(f"{configuration=}") deployment_steps = [ _CloneRepo(config=configuration), _CopyRelevantRepoFiles(config=configuration), _RelaunchDeploymentScript(config=configuration, original_call_args=sys.argv), - _CreateDeploymentReadme(config=configuration), _StopCompose(config=configuration), _GenerateComposeFile(config=configuration), _PullImages(config=configuration), @@ -534,11 +496,14 @@ def perform_deployment( if __name__ == "__main__": - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) parser.add_argument( "--config-file", - default=f"{Path.home()}/arpav-cline/production-deployment.cfg", + default=Path.home() / "arpav-cline/production-deployment.cfg", help="Path to configuration file", + type=Path, ) parser.add_argument( "--verbose", @@ -555,12 +520,20 @@ def perform_deployment( ) args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING) - deployment_config = get_configuration(args.config_file) - deployment_config.ensure_paths_exist() - try: - perform_deployment( - configuration=deployment_config, - confirmed=args.confirm, - ) - except RuntimeError as err: - raise SystemExit(err) from err + config_file = args.config_file.resolve() + logger.debug(f"{config_file=}") + if config_file.exists(): + deployment_config = get_configuration(config_file) + deployment_config.ensure_paths_exist() + logger.debug("Configuration:") + for k, v in dataclasses.asdict(deployment_config).items(): + logger.debug(f"{k}: {v}") + try: + perform_deployment( + configuration=deployment_config, + confirmed=args.confirm, + ) + except RuntimeError as err: + raise SystemExit(err) from err + else: + raise SystemExit(f"Configuration file {str(config_file)!r} not found") diff --git a/deployments/sample-prod-deployment.cfg b/deployments/sample-prod-deployment.cfg new file mode 100644 index 00000000..bf7fb694 --- /dev/null +++ b/deployments/sample-prod-deployment.cfg @@ -0,0 +1,131 @@ +# Sample production deployment configuration file +# +# Be sure to modify all samples - these all look like +# +# The contents of this file is used by the deployment script to generate the +# compose file which is used for production + +[main] + +# full docker registry URL of the backend image +backend_image = ghcr.io/geobeyond/arpav-ppcv-backend/arpav-ppcv-backend: + +# tag name of the `postgis/postgis` image to use for various DB containers +db_image_tag = 16-3.4 + +# path to the directory where the deployment files reside +deployment_root = /opt/cline + +# full docker registry URL of the frontend image +frontend_image = ghcr.io/geobeyond/arpav-ppcv/arpav-ppcv: + +# comma-separated list of discord-webhook URLs which will be notified when a deployment is done +discord_notification_urls = + , + + +# URL of the git repository where the deployment-related files reside +deployment_files_repo = https://github.com/geobeyond/Arpav-PPCV-backend.git + +# tag name of the `ghcr.io/maplibre/martin` image to use +martin_image_tag = v0.13.0 + +# tag name of the `prefecthq/prefect` image to use +prefect_server_image_tag = 3.0.0rc17-python3.10 + + +[db] + +# name of the backend db +name = arpavppcv + +# password of the backend db +password = + +# username of user that connects to the backend db +user = + + +[prefect_db] + +# name of the prefect db +name = prefect + +# password of the prefect db +password = + +# username of user that connects to the prefect db +user = + + +[reverse_proxy] + +# tag name of the `traefik` image to use +image_tag = 3.0.2 + +# Local path to TLS certificate +# /opt/arpav_ppcv_tls_certs/cert.crt +tls_cert_path = + +# Local path to TLS certificate key +# /opt/arpav_ppcv_tls_certs/cert.key +tls_cert_key_path = + +# Local path to traefik basicauth users file +traefik_users_file_path = + + +[tolgee_app] + +# initial password for the tolgee user +env_tolgee_authentication_initial_password = + +# JWT secrete key for auth-related tasks +env_tolgee_authentication_jwt_secret = + +# public URL of the tolgee service +env_tolgee_frontend_url = + +# tag name of the `tolgee/tolgee` image to use +image_tag = v3.71.4 + + +[tolgee_db] + +# name of the tolgee db +name = tolgee + +# password of the tolgee db +password = + +# username of user that connects to the tolgee db +user = + + +[webapp] + +# password for the user that is able to access the admin section +env_admin_user_password = + +# username for the user that is able to access the admin section +env_admin_user_username = + +# comma-separated list of allowed CORS origins +env_cors_origins = + , + + +# how many worker processes should the uvicorn server use +env_num_uvicorn_worker_processes = 1 + +# public URL of the system +env_public_url = + +# secret key used in sessions +env_session_secret_key = + +# base URL of the THREDDS server +env_thredds_server_base_url = + +# Local path to the uvicorn log configuration +env_uvicorn_log_config_file = diff --git a/docker/compose.production.template.yaml b/docker/compose.production.template.yaml index 36690273..028842c3 100644 --- a/docker/compose.production.template.yaml +++ b/docker/compose.production.template.yaml @@ -5,7 +5,7 @@ # - do not mount source code inside any container - keep volume binds to the # minimum, only for relevant configuration file(s) and data collections -name: arpav-cline-production +name: "${compose_project_name}" services: @@ -25,6 +25,7 @@ services: secrets: - tls-cert - tls-key + - traefik-users-file restart: unless-stopped frontend: @@ -166,3 +167,6 @@ secrets: tls-key: file: "${tls_cert_key_path}" + + traefik-users-file: + file: "${traefik_users_file_path}" diff --git a/docker/traefik/production-config.toml b/docker/traefik/production-config.toml new file mode 100644 index 00000000..297ada30 --- /dev/null +++ b/docker/traefik/production-config.toml @@ -0,0 +1,29 @@ +# Static configuration file for traefik +# +# In this file we mostly configure providers, entrypoints and security. +# Routers, the other major part of a traefik configuration, form the +# so-called 'dynamic configuration' and in this case are gotten from +# the labels associated with the docker provider +# +# More info: +# +# https://doc.traefik.io/traefik/ + +[accessLog] + +[entryPoints] + +[entryPoints.web] +address = ":80" +[entryPoints.web.forwardedHeaders] +insecure = true + +[entryPoints.webSecure] +address = ":443" +[entryPoints.web.forwardedHeaders] +insecure = true + +[providers] + +[providers.docker] +exposedByDefault = false From 630b66a7da86da5aaee5dfd8bde97e9e5027773a Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 25 Nov 2024 15:47:21 +0000 Subject: [PATCH 4/4] Added optional backend and frontend image names to deployment script --- .github/workflows/ci.yaml | 2 +- deployments/deploy.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d0cd6d49..3c7a1b7e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: Continuous Integration on: - push: + push: # will run both when pushing a branch and when pushing a tag pull_request: diff --git a/deployments/deploy.py b/deployments/deploy.py index 68b420ae..9989b506 100644 --- a/deployments/deploy.py +++ b/deployments/deploy.py @@ -518,12 +518,35 @@ def perform_deployment( "in dry-run mode, just showing what steps would be performed" ), ) + parser.add_argument( + "-b", + "--backend-image", + help=( + "Full name of the docker image to be used for the backend. " + "Example: " + "'ghcr.io/geobeyond/arpav-ppcv-backend/arpav-ppcv-backend:v1.0.0'. " + "Defaults to whatever is specified in the configuration file." + ), + ) + parser.add_argument( + "-f", + "--frontend-image", + help=( + "Full name of the docker image to be used for the frontend. " + "Example: 'ghcr.io/geobeyond/arpav-ppcv/arpav-ppcv:v1.0.0'. " + "Defaults to whatever is specified in the configuration file." + ), + ) args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING) config_file = args.config_file.resolve() logger.debug(f"{config_file=}") if config_file.exists(): deployment_config = get_configuration(config_file) + if (backend_image_name := args.backend_image) is not None: + deployment_config.backend_image = backend_image_name + if (frontend_image_name := args.frontend_image) is not None: + deployment_config.frontend_image = frontend_image_name deployment_config.ensure_paths_exist() logger.debug("Configuration:") for k, v in dataclasses.asdict(deployment_config).items():