From d215a20a18d9f47509a82ec6035a3f505fc54c2f Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 23 Jul 2024 14:56:30 +0200 Subject: [PATCH 1/5] feat(Backend):delete exisitng backends with same upstream --- .../app/main/model/serializers.py | 5 ++ .../app/main/service/backend.py | 52 ++++++++++++++----- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/FastapiOpenRestyConfigurator/app/main/model/serializers.py b/FastapiOpenRestyConfigurator/app/main/model/serializers.py index 65661d2..74f8530 100644 --- a/FastapiOpenRestyConfigurator/app/main/model/serializers.py +++ b/FastapiOpenRestyConfigurator/app/main/model/serializers.py @@ -125,6 +125,11 @@ class BackendOut(BackendBase): description="Protected reverse-proxy path which leads to specific backend.", example="myRstudio_103" ) + file_path:str=Field( + ..., + title="File path of Backend", + description=" Local File path of the backend", + ) class BackendTemp(BackendIn, BackendOut): diff --git a/FastapiOpenRestyConfigurator/app/main/service/backend.py b/FastapiOpenRestyConfigurator/app/main/service/backend.py index 2abdd9a..e439e6b 100644 --- a/FastapiOpenRestyConfigurator/app/main/service/backend.py +++ b/FastapiOpenRestyConfigurator/app/main/service/backend.py @@ -5,7 +5,7 @@ import os import re from random import randint -from typing import List +from typing import List, Dict from werkzeug.exceptions import NotFound, InternalServerError @@ -19,9 +19,10 @@ file_regex = r"(\d*)%([a-z0-9\-\@.]*?)%([^%]*)%([^%]*)%([^%]*)\.conf" + async def random_with_n_digits(n): - range_start = 10**(n-1) - range_end = (10**n)-1 + range_start = 10 ** (n - 1) + range_end = (10 ** n) - 1 return randint(range_start, range_end) @@ -44,12 +45,39 @@ async def get_backends() -> List[BackendOut]: owner=match.group(2), location_url=match.group(3), template=match.group(4), - template_version=match.group(5) + template_version=match.group(5), + file_path=os.path.join(settings.FORC_BACKEND_PATH, file) ) valid_backends.append(backend) return valid_backends +async def get_backends_upstream_urls() -> Dict[str, List[str]]: + valid_backends: List[BackendOut] = await get_backends() + upstream_urls = {} + + for backend in valid_backends: + upstream_url = extract_proxy_pass(backend.file_path) + if upstream_url: + if upstream_url not in upstream_urls: + upstream_urls[upstream_url] = [] + upstream_urls[upstream_url].append(backend.id) + + return upstream_urls + + +def extract_proxy_pass(file_path): + with open(file_path, 'r') as file: + content = file.read() + + match = re.search(r'proxy_pass\s+(http[^\s;]+);', content) + + if match: + return match.group(1) + else: + return None + + async def generate_suffix_number(user_key_url): current_backends: List[BackendOut] = await get_backends() same_name_backend_ids = [] @@ -82,16 +110,12 @@ async def create_backend(payload: BackendIn): payload.location_url = f"{payload.user_key_url}_{suffix_number}" - # check for duplicates in filesystem - backend_path_files = os.listdir(settings.FORC_BACKEND_PATH) - for file in backend_path_files: - match = re.fullmatch(file_regex, file) - if not match: - logger.warning("Found a backend file with wrong naming, skipping it: " + str(file)) - continue - if match.group(1) == payload.id or match.group(3) == payload.location_url: - logger.error("Tried to create duplicate backend: " + payload.id + " and " + payload.location_url) - raise InternalServerError("Server tried to create duplicate backend.") + # check for duplicated in ip and port: + upstream_urls: Dict[str, List[BackendOut]] = await get_backends_upstream_urls() + matching_urls_backends: List[BackendOut] = upstream_urls.get(payload.upstream_url, []) + for backend in matching_urls_backends: + logger.info(f"Deleting existing Backend with same Upstream Url - {payload.upstream_url}") + await delete_backend(backend.id) # create backend file in filesystem filename = f"{payload.id}%{payload.owner}%{payload.location_url}%{payload.template}%{payload.template_version}.conf" From bad72f3b66fbd3024b6eef6e686c6a77fb199609 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 23 Jul 2024 15:16:38 +0200 Subject: [PATCH 2/5] updateda ansible play --- .../app/main/service/backend.py | 2 +- ansible/roles/forc_api/tasks/main.yml | 39 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/FastapiOpenRestyConfigurator/app/main/service/backend.py b/FastapiOpenRestyConfigurator/app/main/service/backend.py index e439e6b..4c4efbc 100644 --- a/FastapiOpenRestyConfigurator/app/main/service/backend.py +++ b/FastapiOpenRestyConfigurator/app/main/service/backend.py @@ -41,7 +41,7 @@ async def get_backends() -> List[BackendOut]: logger.warning("Found a backend file with wrong naming, skipping it: " + str(file)) continue backend: BackendOut = BackendOut( - id=match.group(1), + id=match.group(1), owner=match.group(2), location_url=match.group(3), template=match.group(4), diff --git a/ansible/roles/forc_api/tasks/main.yml b/ansible/roles/forc_api/tasks/main.yml index 897c03b..e1c7398 100644 --- a/ansible/roles/forc_api/tasks/main.yml +++ b/ansible/roles/forc_api/tasks/main.yml @@ -1,23 +1,34 @@ --- # tasks file for forc_api -- include: "000-preparation.yml" - tags: ['forc_preparation'] +- name: Preparation tasks + import_tasks: "000-preparation.yml" + tags: + - forc_preparation -- include: "005-packages.yml" - tags: ['forc_packages'] - -- include: "010-config.yml" - tags: ['forc_config'] +- name: Package installation tasks + import_tasks: "005-packages.yml" + tags: + - forc_packages -- include: "015-forc.yml" - tags: ['forc_forc'] +- name: Configuration tasks + import_tasks: "010-config.yml" + tags: + - forc_config -- include: "020-openresty.yml" - tags: ['forc_openresty'] +- name: FORC tasks + import_tasks: "015-forc.yml" + tags: + - forc_forc + +- name: OpenResty installation tasks + import_tasks: "020-openresty.yml" when: INSTALL_OPENRESTY + tags: + - forc_openresty -- include: "025-backup.yml" - tags: ["forc-backup"] +- name: Backup tasks + import_tasks: "025-backup.yml" when: FORC_BACKUP_ENABLED - + tags: + - forc_backup From 05efe3f52f889a86db0ab28e979a78adfc10bb36 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 23 Jul 2024 15:25:12 +0200 Subject: [PATCH 3/5] updateda ansible play --- ansible/roles/forc_api/tasks/025-backup.yml | 38 +++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/ansible/roles/forc_api/tasks/025-backup.yml b/ansible/roles/forc_api/tasks/025-backup.yml index 4a4f8b3..4b36cb0 100644 --- a/ansible/roles/forc_api/tasks/025-backup.yml +++ b/ansible/roles/forc_api/tasks/025-backup.yml @@ -34,24 +34,26 @@ src: "{{ role_path }}/files" dest: "/persistent" -- docker_container: - name: "backup-forc" - image: quay.io/denbicloud/cron-backup:0.13.5 - - state: started - restart: yes - volumes: - - "{{FORC_BACKEND_PATH}}:/backends" - - "{{FORC_TEMPLATE_PATH}}:/templates" - - "/persistent/files/install-packages.sh:/install-packages.sh" - - "/persistent/files/backup.sh:/etc/cronscripts/backup.sh" - - "/persistent/files/backup-cron:/etc/crontabs/dockercron/backup-cron" - - "{{FORC_BACKUP_HOST_PATH}}:/etc/backup" - env: - BACKUP_ROTATION_ENABLED={{FORC_BACKUP_ROTATION_ENABLED}} - BACKUP_ROTATION_MAX_SIZE={{FORC_BACKUP_ROTATION_MAX_SIZE}} - BACKUP_ROTATION_CUT_SIZE={{FORC_BACKUP_ROTATION_CUT_SIZE}} - BACKUP_ROTATION_SIZE_TYP={{FORC_BACKUP_ROTATION_SIZE_TYP}} +- name: Run the backup-forc container + shell: | + docker run -d \ + --name backup-forc \ + --restart always \ + -v "{{ FORC_BACKEND_PATH }}:/backends" \ + -v "{{ FORC_TEMPLATE_PATH }}:/templates" \ + -v "/persistent/files/install-packages.sh:/install-packages.sh" \ + -v "/persistent/files/backup.sh:/etc/cronscripts/backup.sh" \ + -v "/persistent/files/backup-cron:/etc/crontabs/dockercron/backup-cron" \ + -v "{{ FORC_BACKUP_HOST_PATH }}:/etc/backup" \ + -e BACKUP_ROTATION_ENABLED={{ FORC_BACKUP_ROTATION_ENABLED }} \ + -e BACKUP_ROTATION_MAX_SIZE={{ FORC_BACKUP_ROTATION_MAX_SIZE }} \ + -e BACKUP_ROTATION_CUT_SIZE={{ FORC_BACKUP_ROTATION_CUT_SIZE }} \ + -e BACKUP_ROTATION_SIZE_TYP={{ FORC_BACKUP_ROTATION_SIZE_TYP }} \ + quay.io/denbicloud/cron-backup:0.13.5 + args: + executable: /bin/bash + + From abb3297e66df84166bbbbbde8e6e6dd63017c0fa Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 23 Jul 2024 15:43:32 +0200 Subject: [PATCH 4/5] updated default --- FastapiOpenRestyConfigurator/app/main/model/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FastapiOpenRestyConfigurator/app/main/model/serializers.py b/FastapiOpenRestyConfigurator/app/main/model/serializers.py index 74f8530..084c263 100644 --- a/FastapiOpenRestyConfigurator/app/main/model/serializers.py +++ b/FastapiOpenRestyConfigurator/app/main/model/serializers.py @@ -126,7 +126,7 @@ class BackendOut(BackendBase): example="myRstudio_103" ) file_path:str=Field( - ..., + None, title="File path of Backend", description=" Local File path of the backend", ) From 0b863b8c206023067dd264b1217b9dfd51987f76 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 23 Jul 2024 15:50:01 +0200 Subject: [PATCH 5/5] updated default --- FastapiOpenRestyConfigurator/app/main/service/backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FastapiOpenRestyConfigurator/app/main/service/backend.py b/FastapiOpenRestyConfigurator/app/main/service/backend.py index 4c4efbc..51e425f 100644 --- a/FastapiOpenRestyConfigurator/app/main/service/backend.py +++ b/FastapiOpenRestyConfigurator/app/main/service/backend.py @@ -52,7 +52,7 @@ async def get_backends() -> List[BackendOut]: return valid_backends -async def get_backends_upstream_urls() -> Dict[str, List[str]]: +async def get_backends_upstream_urls() -> Dict[str, List[BackendOut]]: valid_backends: List[BackendOut] = await get_backends() upstream_urls = {} @@ -61,7 +61,7 @@ async def get_backends_upstream_urls() -> Dict[str, List[str]]: if upstream_url: if upstream_url not in upstream_urls: upstream_urls[upstream_url] = [] - upstream_urls[upstream_url].append(backend.id) + upstream_urls[upstream_url].append(backend) return upstream_urls