From 0d90e384c0b5275fc006d4ca414dc3b2cde84158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= Date: Wed, 27 Nov 2024 09:01:47 +0100 Subject: [PATCH] feat: add backups_sync management command --- weblate_web/hetzner.py | 33 +++++-- .../management/commands/backups_sync.py | 99 +++++++++++++++++++ weblate_web/models.py | 13 +-- 3 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 weblate_web/management/commands/backups_sync.py diff --git a/weblate_web/hetzner.py b/weblate_web/hetzner.py index 7451b074a..8549c54d5 100644 --- a/weblate_web/hetzner.py +++ b/weblate_web/hetzner.py @@ -42,6 +42,17 @@ def create_storage_folder( handle.write(last_report.ssh_key) +def generate_subaccount_data( + dirname: str, service: Service, customer: Customer +) -> dict[str, str]: + return { + "homedirectory": f"weblate/{dirname}", + "ssh": "1", + "external_reachability": "1", + "comment": f"Weblate backup service {service.pk} ({customer.name})", + } + + def create_storage_subaccount( dirname: str, service: Service, customer: Customer ) -> dict: @@ -49,14 +60,24 @@ def create_storage_subaccount( url = SUBACCOUNTS_API.format(settings.STORAGE_BOX) response = requests.post( url, - data={ - "homedirectory": f"weblate/{dirname}", - "ssh": "1", - "external_reachability": "1", - "comment": f"Weblate backup service {service.pk} ({customer.name})", - }, + data=generate_subaccount_data(dirname, service, customer), auth=(settings.STORAGE_USER, settings.STORAGE_PASSWORD), timeout=720, ) response.raise_for_status() return response.json() + + +def generate_ssh_url(data: dict) -> str: + return "ssh://{}@{}:23/./backups".format( + data["subaccount"]["username"], data["subaccount"]["server"] + ) + + +def get_storage_subaccounts() -> list[dict]: + url = SUBACCOUNTS_API.format(settings.STORAGE_BOX) + response = requests.get( + url, auth=(settings.STORAGE_USER, settings.STORAGE_PASSWORD), timeout=720 + ) + response.raise_for_status() + return response.json() diff --git a/weblate_web/management/commands/backups_sync.py b/weblate_web/management/commands/backups_sync.py new file mode 100644 index 000000000..a95aa6163 --- /dev/null +++ b/weblate_web/management/commands/backups_sync.py @@ -0,0 +1,99 @@ +# +# Copyright © Michal Čihař +# +# This file is part of Weblate +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from django.conf import settings +from django.core.management.base import BaseCommand + +from weblate_web.hetzner import ( + generate_ssh_url, + generate_subaccount_data, + get_storage_subaccounts, +) +from weblate_web.models import Service + + +class Command(BaseCommand): + help = "syncrhonizes backup API" + + def add_arguments(self, parser): + parser.add_argument( + "--delete", + default=False, + action="store_true", + help="Delete stale backup repositories", + ) + + def handle(self, *args, **options): + backup_services: dict[str, Service] = { + service.backup_repository: service + for service in Service.objects.exclude(backup_repository="") + } + processed_repositories = set() + backup_storages = get_storage_subaccounts() + + for storage in backup_storages: + # Skip non-weblate subaccounts + homedirectory: str = storage["subaccount"]["homedirectory"] + if not homedirectory.startswith("weblate/"): + continue + # Generate SSH URL used for borg + ssh_url = generate_ssh_url(storage) + processed_repositories.add(ssh_url) + + # Fetch matching service + try: + service = backup_services[ssh_url] + except KeyError: + self.stderr.write(f"unused URL: {ssh_url}") + continue + + # Validate service + customer = service.customer + if customer is None: + self.stderr.write(f"missing customer: {service.pk}") + continue + + # Sync our data + update = False + if not service.backup_box: + service.backup_box = settings.STORAGE_BOX + update = True + dirname = homedirectory.removeprefix("weblate/") + if service.backup_directory != dirname: + service.backup_directory = dirname + update = True + if update: + self.stdout.write("Updating data for {service.pk} ({customer.name})") + service.save(update_fields=["backup_box", "backup_directory"]) + + # Sync Hetzner data + storage_data = generate_subaccount_data(dirname, service, customer) + if storage["subaccount"]["comment"] != storage_data["comment"]: + username: str = storage["subaccount"]["username"] + self.stdout.write( + f"Updating Hetzner data for {username} for {service.pk} ({customer.name})" + ) + + for service in backup_services.values(): + if not service.has_paid_backup(): + self.stderr.write(f"not paid: {service.pk} ({service.customer})") + + for extra in set(backup_services) - processed_repositories: + self.stderr.write(f"unused: {extra}") diff --git a/weblate_web/models.py b/weblate_web/models.py index ae16ff1c8..d57651235 100644 --- a/weblate_web/models.py +++ b/weblate_web/models.py @@ -48,7 +48,7 @@ from weblate_web.payments.utils import send_notification from weblate_web.zammad import create_dedicated_hosting_ticket -from .hetzner import create_storage_folder, create_storage_subaccount +from .hetzner import create_storage_folder, create_storage_subaccount, generate_ssh_url if TYPE_CHECKING: from fakturace.invoices import Invoice @@ -794,11 +794,14 @@ def update_status(self): self.limit_projects = package.limit_projects self.save() - def create_backup(self): + def has_paid_backup(self) -> bool: subscriptions = self.hosted_subscriptions | self.backup_subscriptions + return subscriptions.filter(expires__gt=timezone.now()).exists() + + def create_backup(self): if ( not self.backup_repository - and subscriptions.filter(expires__gt=timezone.now()).exists() + and self.has_paid_backup() and (last_report := self.last_report) ): self.create_backup_repository(last_report) @@ -821,9 +824,7 @@ def create_backup_repository(self, last_report: Report): # Create account on the service data = create_storage_subaccount(dirname, self, self.customer) - self.backup_repository = "ssh://{}@{}:23/./backups".format( - data["subaccount"]["username"], data["subaccount"]["server"] - ) + self.backup_repository = generate_ssh_url(data) self.backup_box = settings.STORAGE_BOX self.backup_directory = dirname self.save(update_fields=["backup_repository", "backup_box", "backup_directory"])