fix(backend): 修复medium初始化上传文件bug #1619

dbm-ui/backend/db_package/migrations/0002_auto_20231101_1746.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.19 on 2023-11-01 09:46
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("db_package", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="package",
+            name="create_at",
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="创建时间"),
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="update_at",
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="更新时间"),
+        ),
+    ]

dbm-ui/backend/db_package/models.py

@@ -8,8 +8,10 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import datetime
 from typing import Optional
 
+import django.utils.timezone as timezone
 from django.db import models
 from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
@@ -35,6 +37,9 @@ class Package(AuditedModel):
     # allow_biz_ids 主要用于灰度场景，部分业务先用，不配置/为空 代表全业务可用
     allow_biz_ids = models.JSONField(_("允许的业务列表"), null=True)
     mode = models.CharField(_("安装包模式"), choices=PackageMode.get_choices(), max_length=LEN_SHORT, default="system")
+    # package独立出时间字段
+    create_at = models.DateTimeField(_("创建时间"), default=timezone.now)
+    update_at = models.DateTimeField(_("更新时间"), default=timezone.now)
 
     class Meta:
         verbose_name = _("介质包（Package）")

dbm-ui/backend/db_package/serializers.py

@@ -8,6 +8,7 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+from datetime import datetime
 
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
@@ -29,3 +30,12 @@ class UploadPackageSerializer(serializers.Serializer):
     version = serializers.CharField(help_text=_("数据库版本"), required=False, allow_blank=True)
     pkg_type = serializers.ChoiceField(help_text=_("包类型"), choices=PackageType.get_choices())
     db_type = serializers.ChoiceField(help_text=_("存储类型"), choices=DBType.get_choices())
+
+
+class UpdateOrCreateSerializer(serializers.ModelSerializer):
+    create_at = serializers.DateTimeField(required=False, default=datetime.now())
+    update_at = serializers.DateTimeField(required=False, default=datetime.now())
+
+    class Meta:
+        model = Package
+        fields = "__all__"

dbm-ui/backend/db_package/views.py

@@ -22,7 +22,7 @@
 from backend.core.storages.storage import get_storage
 from backend.db_package.filters import PackageListFilter
 from backend.db_package.models import Package
-from backend.db_package.serializers import PackageSerializer, UploadPackageSerializer
+from backend.db_package.serializers import PackageSerializer, UpdateOrCreateSerializer, UploadPackageSerializer
 from backend.flow.consts import MediumEnum
 from backend.iam_app.handlers.drf_perm import GlobalManageIAMPermission
 from backend.utils.files import md5sum
@@ -47,10 +47,10 @@ def create(self, request, *args, **kwargs):
         return super().create(request, *args, **kwargs)
 
     @common_swagger_auto_schema(
-        operation_summary=_("新建或者更新版本文件"),
+        operation_summary=_("新建或者更新版本文件(适用于medium初始化)"),
         tags=[DB_PACKAGE_TAG],
     )
-    @action(methods=["POST"], detail=False)
+    @action(methods=["POST"], detail=False, serializer_class=UpdateOrCreateSerializer)
     def update_or_create(self, request, *args, **kwargs):
         data = self.params_validate(self.get_serializer_class())
         Package.objects.update_or_create(**data)

dbm-ui/backend/dbm_init/medium/handlers.py

@@ -9,6 +9,7 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import hashlib
 import json
 import logging
 import os
@@ -73,7 +74,7 @@ def __init__(self, storage=None):
                 project_id=os.getenv("BKREPO_PROJECT"),
                 bucket=os.getenv("BKREPO_PUBLIC_BUCKET"),
                 endpoint_url=os.getenv("BKREPO_ENDPOINT_URL"),
-                file_overwrite=os.getenv("FILE_OVERWRITE", False),
+                file_overwrite=os.getenv("FILE_OVERWRITE", True),
             )
             self.storage.client = MediumBKGenericRepoClient(
                 bucket=os.getenv("BKREPO_PUBLIC_BUCKET"),
@@ -147,10 +148,15 @@ def upload_medium(self, path, bkrepo_tmp_dir):
                     # 分割路径，保留制品路径(db_type/name/version/file)
                     file_path = os.path.join(root, file)
                     file_path_bkrepo = file_path.split(file_path.rsplit("/", 4)[0])[1]
-                    # 如果文件已存在，则不进行上传更新
                     logger.info("upload file: %s -> %s", file_path, file_path_bkrepo)
-                    if not self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1]:
-                        with open(file_path, "rb") as f:
+                    with open(file_path, "rb") as f:
+                        # 如果当前版本不存在，则更新介质
+                        if not self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1]:
+                            self.storage.save(file_path_bkrepo, f)
+                        # 如果文件md5不相等，则更新介质
+                        bkrepo_file_md5 = self.storage.listdir(file_path_bkrepo.rsplit("/", 1)[0])[1][0]["md5"]
+                        pkg_file_md5 = hashlib.md5(f.read()).hexdigest()
+                        if bkrepo_file_md5 != pkg_file_md5:
                             self.storage.save(file_path_bkrepo, f)
 
     def sync_from_bkrepo(self, db_type):
@@ -159,6 +165,10 @@ def sync_from_bkrepo(self, db_type):
 
         http = HttpHandler()
         for pkg_type in self.storage.listdir(f"/{db_type}")[0]:
+            # 排除非介质文件
+            if pkg_type["name"] in ["keyfiles", "db-remote-service", "sqlfile"]:
+                continue
+
             for version in self.storage.listdir(pkg_type["fullPath"])[0]:
                 for media in self.storage.listdir(version["fullPath"])[1]:
                     package_params = {
@@ -170,6 +180,9 @@ def sync_from_bkrepo(self, db_type):
                         "size": media["size"],
                         "md5": media["md5"],
                         "create_at": str(datetime.strptime(media["createdDate"], "%Y-%m-%dT%H:%M:%S.%f")),
+                        "creator": "system",
+                        "update_at": str(datetime.strptime(media["lastModifiedDate"], "%Y-%m-%dT%H:%M:%S.%f")),
+                        "updater": "system",
                     }
                     logger.info("sync info %s", json.dumps(package_params, indent=4))
                     http.post(url="apis/packages/update_or_create/", data=package_params)