From 34e1eabd16338853f08e3e1dec0c7450f13162ed Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Wed, 1 Nov 2023 17:36:17 +0800 Subject: [PATCH] =?UTF-8?q?fix(backend):=20=E4=BF=AE=E5=A4=8Dmedium?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?bug=20#1619?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0002_auto_20231101_1746.py | 24 +++++++++++++++++++ dbm-ui/backend/db_package/models.py | 5 ++++ dbm-ui/backend/db_package/serializers.py | 10 ++++++++ dbm-ui/backend/db_package/views.py | 6 ++--- dbm-ui/backend/dbm_init/medium/handlers.py | 21 ++++++++++++---- 5 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 dbm-ui/backend/db_package/migrations/0002_auto_20231101_1746.py diff --git a/dbm-ui/backend/db_package/migrations/0002_auto_20231101_1746.py b/dbm-ui/backend/db_package/migrations/0002_auto_20231101_1746.py new file mode 100644 index 0000000000..ecd8c3cfd6 --- /dev/null +++ b/dbm-ui/backend/db_package/migrations/0002_auto_20231101_1746.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.19 on 2023-11-01 09:46 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("db_package", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="package", + name="create_at", + field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="创建时间"), + ), + migrations.AlterField( + model_name="package", + name="update_at", + field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="更新时间"), + ), + ] diff --git a/dbm-ui/backend/db_package/models.py b/dbm-ui/backend/db_package/models.py index d0b1ceb1dd..08034891b5 100644 --- a/dbm-ui/backend/db_package/models.py +++ b/dbm-ui/backend/db_package/models.py @@ -8,8 +8,10 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import datetime from typing import Optional +import django.utils.timezone as timezone from django.db import models from django.db.models import Q from django.utils.translation import ugettext_lazy as _ @@ -35,6 +37,9 @@ class Package(AuditedModel): # allow_biz_ids 主要用于灰度场景，部分业务先用，不配置/为空 代表全业务可用 allow_biz_ids = models.JSONField(_("允许的业务列表"), null=True) mode = models.CharField(_("安装包模式"), choices=PackageMode.get_choices(), max_length=LEN_SHORT, default="system") + # package独立出时间字段 + create_at = models.DateTimeField(_("创建时间"), default=timezone.now) + update_at = models.DateTimeField(_("更新时间"), default=timezone.now) class Meta: verbose_name = _("介质包（Package）") diff --git a/dbm-ui/backend/db_package/serializers.py b/dbm-ui/backend/db_package/serializers.py index 0f0b74d13a..717ffeccc5 100644 --- a/dbm-ui/backend/db_package/serializers.py +++ b/dbm-ui/backend/db_package/serializers.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from datetime import datetime from django.utils.translation import gettext_lazy as _ from rest_framework import serializers @@ -29,3 +30,12 @@ class UploadPackageSerializer(serializers.Serializer): version = serializers.CharField(help_text=_("数据库版本"), required=False, allow_blank=True) pkg_type = serializers.ChoiceField(help_text=_("包类型"), choices=PackageType.get_choices()) db_type = serializers.ChoiceField(help_text=_("存储类型"), choices=DBType.get_choices()) + + +class UpdateOrCreateSerializer(serializers.ModelSerializer): + create_at = serializers.DateTimeField(required=False, default=datetime.now()) + update_at = serializers.DateTimeField(required=False, default=datetime.now()) + + class Meta: + model = Package + fields = "__all__" diff --git a/dbm-ui/backend/db_package/views.py b/dbm-ui/backend/db_package/views.py index 1a74602084..a9ffb0193c 100644 --- a/dbm-ui/backend/db_package/views.py +++ b/dbm-ui/backend/db_package/views.py @@ -22,7 +22,7 @@ from backend.core.storages.storage import get_storage from backend.db_package.filters import PackageListFilter from backend.db_package.models import Package -from backend.db_package.serializers import PackageSerializer, UploadPackageSerializer +from backend.db_package.serializers import PackageSerializer, UpdateOrCreateSerializer, UploadPackageSerializer from backend.flow.consts import MediumEnum from backend.iam_app.handlers.drf_perm import GlobalManageIAMPermission from backend.utils.files import md5sum @@ -47,10 +47,10 @@ def create(self, request, *args, **kwargs): return super().create(request, *args, **kwargs) @common_swagger_auto_schema( - operation_summary=_("新建或者更新版本文件"), + operation_summary=_("新建或者更新版本文件(适用于medium初始化)"), tags=[DB_PACKAGE_TAG], ) - @action(methods=["POST"], detail=False) + @action(methods=["POST"], detail=False, serializer_class=UpdateOrCreateSerializer) def update_or_create(self, request, *args, **kwargs): data = self.params_validate(self.get_serializer_class()) Package.objects.update_or_create(**data) diff --git a/dbm-ui/backend/dbm_init/medium/handlers.py b/dbm-ui/backend/dbm_init/medium/handlers.py index 4721737403..78a6443da8 100644 --- a/dbm-ui/backend/dbm_init/medium/handlers.py +++ b/dbm-ui/backend/dbm_init/medium/handlers.py @@ -9,6 +9,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import hashlib import json import logging import os @@ -73,7 +74,7 @@ def __init__(self, storage=None): project_id=os.getenv("BKREPO_PROJECT"), bucket=os.getenv("BKREPO_PUBLIC_BUCKET"), endpoint_url=os.getenv("BKREPO_ENDPOINT_URL"), - file_overwrite=os.getenv("FILE_OVERWRITE", False), + file_overwrite=os.getenv("FILE_OVERWRITE", True), ) self.storage.client = MediumBKGenericRepoClient( bucket=os.getenv("BKREPO_PUBLIC_BUCKET"), @@ -147,10 +148,15 @@ def upload_medium(self, path, bkrepo_tmp_dir): # 分割路径，保留制品路径(db_type/name/version/file) file_path = os.path.join(root, file) file_path_bkrepo = file_path.split(file_path.rsplit("/", 4)[0])[1] - # 如果文件已存在，则不进行上传更新 logger.info("upload file: %s -> %s", file_path, file_path_bkrepo) - if not self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1]: - with open(file_path, "rb") as f: + with open(file_path, "rb") as f: + # 如果当前版本不存在，则更新介质 + if not self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1]: + self.storage.save(file_path_bkrepo, f) + # 如果文件md5不相等，则更新介质 + bkrepo_file_md5 = self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1][0]["md5"] + pkg_file_md5 = hashlib.md5(f.read()).hexdigest() + if bkrepo_file_md5 != pkg_file_md5: self.storage.save(file_path_bkrepo, f) def sync_from_bkrepo(self, db_type): @@ -159,6 +165,10 @@ def sync_from_bkrepo(self, db_type): http = HttpHandler() for pkg_type in self.storage.listdir(f"/{db_type}")[0]: + # 排除非介质文件 + if pkg_type["name"] in ["keyfiles", "db-remote-service", "sqlfile"]: + continue + for version in self.storage.listdir(pkg_type["fullPath"])[0]: for media in self.storage.listdir(version["fullPath"])[1]: package_params = { @@ -170,6 +180,9 @@ def sync_from_bkrepo(self, db_type): "size": media["size"], "md5": media["md5"], "create_at": str(datetime.strptime(media["createdDate"], "%Y-%m-%dT%H:%M:%S.%f")), + "creator": "system", + "update_at": str(datetime.strptime(media["lastModifiedDate"], "%Y-%m-%dT%H:%M:%S.%f")), + "updater": "system", } logger.info("sync info %s", json.dumps(package_params, indent=4)) http.post(url="apis/packages/update_or_create/", data=package_params)