fix(backend): riak单据开发 & 补充redis介质镜像打包 #1560

.dockerignore

@@ -4,4 +4,3 @@
 .vscode
 tmp*
 
-**/mod/

dbm-services/.gitignore

@@ -1,6 +1,3 @@
 .codecc
 .idea
 .vscode
-
-pkg/
-

dbm-ui/backend/configuration/constants.py

@@ -47,6 +47,7 @@ class SystemSettingsEnum(str, StructuredEnum):
     SPEC_OFFSET = EnumField("SPEC_OFFSET", _("默认的规格参数偏移量"))
     DEVICE_CLASSES = EnumField("DEVICE_CLASSES", _("机型列表"))
     BKM_DUTY_NOTICE = EnumField("BKM_DUTY_NOTICE", _("轮值通知设置"))
+    DBM_MIGRATE_USER = EnumField("DBM_MIGRATE_USER", _("具备迁移权限的人员名单"))
 
 
 class BizSettingsEnum(str, StructuredEnum):
@@ -116,6 +117,9 @@ class BizSettingsEnum(str, StructuredEnum):
     },
 }
 
+# 默认具备迁移权限的人员
+DBM_DEFAULT_MIGRATE_USER = ["admin"]
+
 DEFAULT_SETTINGS = [
     # [key, 类型，初始值, 描述]
     [SystemSettingsEnum.BKM_DBM_TOKEN.value, "str", "", _("监控数据源token")],
@@ -124,6 +128,7 @@ class BizSettingsEnum(str, StructuredEnum):
     [SystemSettingsEnum.INDEPENDENT_HOSTING_BIZS.value, "list", [], _("独立托管机器的业务列表")],
     [SystemSettingsEnum.SPEC_OFFSET.value, "dict", SPEC_OFFSET_VALUE, _("默认的规格参数偏移量")],
     [SystemSettingsEnum.BKM_DUTY_NOTICE.value, "dict", BKM_DUTY_NOTICE_VALUE, _("默认通知配置")],
+    [SystemSettingsEnum.DBM_MIGRATE_USER, "list", DBM_DEFAULT_MIGRATE_USER, _("具备迁移权限的人员名单")],
 ]
 
 # 环境配置项 是否支持DNS解析 pulsar flow used

dbm-ui/backend/db_services/meta_import/views.py

@@ -18,6 +18,8 @@
 
 from backend.bk_web import viewsets
 from backend.bk_web.swagger import common_swagger_auto_schema
+from backend.configuration.constants import SystemSettingsEnum
+from backend.configuration.models import SystemSettings
 from backend.db_services.meta_import.constants import SWAGGER_TAG
 from backend.db_services.meta_import.serializers import MySQLHaMetadataImportSerializer
 from backend.iam_app.handlers.drf_perm import RejectPermission
@@ -31,7 +33,8 @@ class DBMetadataImportViewSet(viewsets.SystemViewSet):
     pagination_class = None
 
     def _get_custom_permissions(self):
-        if not self.request.user.is_superuser:
+        migrate_users = SystemSettings.get_setting_value(key=SystemSettingsEnum.DBM_MIGRATE_USER, default=[])
+        if not self.request.user.is_superuser and self.request.user.username not in migrate_users:
             return [RejectPermission()]
         return []
 

dbm-ui/backend/dbm_init/medium/Dockerfile

@@ -32,8 +32,8 @@ FROM go-mod-builder as redis-medium-builder
 
 ## 构建redis介质
 RUN set -ex && \
-    cd /blueking-dbm/dbm-services/redis/db-tools/dbactuator && make -j4
-#    cd /blueking-dbm/dbm-services/redis/db-tools/dbmon/ && sh package.sh TODO: 该命令还有问题
+    cd /blueking-dbm/dbm-services/redis/db-tools/dbactuator && make -j4 && \
+    cd /blueking-dbm/dbm-services/redis/db-tools/dbmon && sh package.sh
 
 
 FROM go-mod-builder as bigdata-medium-builder

dbm-ui/backend/dbm_init/medium/medium.lock

@@ -65,3 +65,8 @@ redis:
     commitId: xxx
     name: dbactuator_redis
     version: 1.0.1
+- dbmon:
+    buildPath: /blueking-dbm/dbm-services/redis/db-tools/dbmon/build/bk-dbmon-*.tar.gz
+    commitId: xxx
+    name: dbmon
+    version: 1.0.1

dbm-ui/backend/dbm_init/medium/readme.md

@@ -1,5 +1,22 @@
 # 版本文件初始化镜像
 
+## medium.lock文件
+lock文件记录了每种db类型打包的介质文件信息，以redis为例：
+```yaml
+redis: # dbtype
+- actuator:
+    buildPath: /blueking-dbm/dbm-services/redis/db-tools/dbactuator/build/dbactuator_redis # 介质路径
+    commitId: xxx # 最新编译代码的commit id，这个一般自动生成
+    name: dbactuator_redis # 介质文件名称
+    version: 1.0.1 # 介质文件版本
+- dbmon:
+    buildPath: /blueking-dbm/dbm-services/redis/db-tools/dbmon/build/bk-dbmon-*.tar.gz
+    commitId: xxx
+    name: dbmon
+    version: 1.0.1
+```
+如果我们有新加的打包文件，只需按照上面格式填写即可，commit id可以为xxx，version按照x.y.z格式填写
+
 ## 打包
 
 在blueking目录下执行

dbm-ui/backend/flow/engine/bamboo/scene/riak/riak_cluster_apply_flow.py

@@ -189,8 +189,8 @@ def _get_riak_config(self):
                 "bk_biz_id": str(self.data["bk_biz_id"]),
                 "level_name": LevelName.CLUSTER,
                 "level_value": self.data["domain"],
-                "level_info": {"module": LevelInfoEnum.TendataModuleDefault},
-                "conf_file": "riak-{}-{}".format(self.data["db_version"], self.data["module_name"]),
+                "level_info": {"module": "riak-{}-{}".format(self.data["db_version"], self.data["db_module_name"])},
+                "conf_file": "riak-{}-{}".format(self.data["db_version"], self.data["db_module_name"]),
                 "conf_type": ConfType.DBCONF,
                 "namespace": NameSpaceEnum.Riak,
                 "format": FormatType.MAP_LEVEL,

dbm-ui/backend/flow/urls.py

@@ -23,7 +23,6 @@
 from backend.flow.views.es_destroy import DestroyEsSceneApiView
 from backend.flow.views.es_disable import DisableEsSceneApiView
 from backend.flow.views.es_enable import EnableEsSceneApiView
-from backend.flow.views.es_fake_apply import FakeInstallEsSceneApiView
 from backend.flow.views.es_reboot import RebootEsSceneApiView
 from backend.flow.views.es_replace import ReplaceEsSceneApiView
 from backend.flow.views.es_scale_up import ScaleUpEsSceneApiView
@@ -32,7 +31,6 @@
 from backend.flow.views.hdfs_destroy import DestroyHdfsSceneApiView
 from backend.flow.views.hdfs_disable import DisableHdfsSceneApiView
 from backend.flow.views.hdfs_enable import EnableHdfsSceneApiView
-from backend.flow.views.hdfs_fake_apply import FakeInstallHdfsSceneApiView
 from backend.flow.views.hdfs_reboot import RebootHdfsSceneApiView
 from backend.flow.views.hdfs_replace import ReplaceHdfsSceneApiView
 from backend.flow.views.hdfs_scale_up import ScaleUpHdfsSceneApiView
@@ -43,18 +41,26 @@
 from backend.flow.views.influxdb_destroy import DestroyInfluxdbSceneApiView
 from backend.flow.views.influxdb_disable import DisableInfluxdbSceneApiView
 from backend.flow.views.influxdb_enable import EnableInfluxdbSceneApiView
-from backend.flow.views.influxdb_fake_apply import FakeInstallInfluxdbSceneApiView
 from backend.flow.views.influxdb_reboot import RebootInfluxdbSceneApiView
 from backend.flow.views.influxdb_replace import ReplaceInfluxdbSceneApiView
 from backend.flow.views.kafka_apply import InstallKafkaSceneApiView
 from backend.flow.views.kafka_destroy import DestroyKafkaSceneApiView
 from backend.flow.views.kafka_disable import DisableKafkaSceneApiView
 from backend.flow.views.kafka_enable import EnableKafkaSceneApiView
-from backend.flow.views.kafka_fake_apply import FakeInstallKafkaSceneApiView
 from backend.flow.views.kafka_reboot import RebootKafkaSceneApiView
 from backend.flow.views.kafka_replace import ReplaceKafkaSceneApiView
 from backend.flow.views.kafka_scale_up import ScaleUpKafkaSceneApiView
 from backend.flow.views.kafka_shrink import ShrinkKafkaSceneApiView
+from backend.flow.views.migrate_views.es_fake_apply import FakeInstallEsSceneApiView
+from backend.flow.views.migrate_views.hdfs_fake_apply import FakeInstallHdfsSceneApiView
+from backend.flow.views.migrate_views.influxdb_fake_apply import FakeInstallInfluxdbSceneApiView
+from backend.flow.views.migrate_views.kafka_fake_apply import FakeInstallKafkaSceneApiView
+from backend.flow.views.migrate_views.pulsar_fake_apply import FakeInstallPulsarSceneApiView
+from backend.flow.views.migrate_views.redis_migrate import (
+    RedisClusterMigrateCompair,
+    RedisClusterMigrateLoad,
+    RedisClusterMigratePrecheck,
+)
 from backend.flow.views.mysql_add_slave import AddMysqlSlaveSceneApiView
 from backend.flow.views.mysql_add_slave_remote import AddMysqlSlaveRemoteSceneApiView
 from backend.flow.views.mysql_checksum import MysqlChecksumSceneApiView
@@ -104,7 +110,6 @@
 from backend.flow.views.pulsar_destroy import DestroyPulsarSceneApiView
 from backend.flow.views.pulsar_disable import DisablePulsarSceneApiView
 from backend.flow.views.pulsar_enable import EnablePulsarSceneApiView
-from backend.flow.views.pulsar_fake_apply import FakeInstallPulsarSceneApiView
 from backend.flow.views.pulsar_reboot import RebootPulsarSceneApiView
 from backend.flow.views.pulsar_scale_up import ScaleUpPulsarSceneApiView
 from backend.flow.views.redis_cluster import (
@@ -116,9 +121,6 @@
     RedisClusterBackupSceneApiView,
     RedisClusterDataCheckRepairApiView,
     RedisClusterDataCopySceneApiView,
-    RedisClusterMigrateCompair,
-    RedisClusterMigrateLoad,
-    RedisClusterMigratePrecheck,
     RedisClusterOpenCloseSceneApiView,
     RedisClusterShardNumUpdateSceneApiView,
     RedisClusterShutdownSceneApiView,

dbm-ui/backend/flow/utils/riak/riak_db_meta.py

@@ -14,6 +14,7 @@
 
 from backend.db_meta import api
 from backend.db_meta.enums import InstanceRole, MachineType
+from backend.db_meta.models import Spec
 from backend.flow.consts import DEFAULT_DB_MODULE_ID, DEFAULT_RIAK_PORT
 
 logger = logging.getLogger("flow")
@@ -35,8 +36,16 @@ def __init__(self, ticket_data: dict, cluster: dict):
 
     def riak_cluster_apply(self) -> bool:
         ips = self.cluster.nodes
+        spec_id = self.ticket_data["resource_spec"]["riak"]["spec_id"]
+        spec_config = Spec.objects.get(spec_id=spec_id).get_spec_info()
         machines = [
-            {"ip": ip, "bk_biz_id": int(self.ticket_data["bk_biz_id"]), "machine_type": MachineType.RIAK.value}
+            {
+                "ip": ip,
+                "bk_biz_id": int(self.ticket_data["bk_biz_id"]),
+                "machine_type": MachineType.RIAK.value,
+                "spec_id": spec_id,
+                "spec_config": spec_config,
+            }
             for ip in ips
         ]
         instances = [
@@ -48,7 +57,7 @@ def riak_cluster_apply(self) -> bool:
             "name": self.ticket_data["cluster_name"],
             "alias": self.ticket_data["cluster_alias"],
             "immute_domain": self.ticket_data["domain"],
-            "db_module_id": DEFAULT_DB_MODULE_ID,
+            "db_module_id": self.ticket_data["db_module_id"],
             "storages": instances,
             "creator": self.ticket_data["created_by"],
             "major_version": self.ticket_data["db_version"],

dbm-ui/backend/flow/views/base.py

@@ -18,6 +18,9 @@
 from rest_framework.permissions import AllowAny
 from rest_framework.views import APIView
 
+from backend.configuration.constants import SystemSettingsEnum
+from backend.configuration.models import SystemSettings
+
 logger = logging.getLogger("root")
 
 
@@ -35,3 +38,13 @@ def get_permissions(self):
         if not self.request.user.is_superuser and not settings.DEBUG:
             raise PermissionDenied(_("权限不足，无法访问!"))
         return []
+
+
+class MigrateFlowView(FlowTestView):
+    """迁移流程的view视图"""
+
+    def get_permissions(self):
+        migrate_users = SystemSettings.get_setting_value(key=SystemSettingsEnum.DBM_MIGRATE_USER, default=[])
+        if self.request.user.username in migrate_users:
+            return []
+        return super().get_permissions()

dbm-ui/backend/flow/views/es_fake_apply.py → dbm-ui/backend/flow/views/migrate_views/es_fake_apply.py

@@ -15,12 +15,12 @@
 from rest_framework.response import Response
 
 from backend.flow.engine.controller.es import EsController
-from backend.flow.views.base import FlowTestView
+from backend.flow.views.base import FlowTestView, MigrateFlowView
 
 logger = logging.getLogger("root")
 
 
-class FakeInstallEsSceneApiView(FlowTestView):
+class FakeInstallEsSceneApiView(MigrateFlowView):
     """
     api: /apis/v1/flow/scene/fake_install_es
     params:

dbm-ui/backend/flow/views/hdfs_fake_apply.py → dbm-ui/backend/flow/views/migrate_views/hdfs_fake_apply.py

@@ -15,12 +15,12 @@
 from rest_framework.response import Response
 
 from backend.flow.engine.controller.hdfs import HdfsController
-from backend.flow.views.base import FlowTestView
+from backend.flow.views.base import FlowTestView, MigrateFlowView
 
 logger = logging.getLogger("root")
 
 
-class FakeInstallHdfsSceneApiView(FlowTestView):
+class FakeInstallHdfsSceneApiView(MigrateFlowView):
     """
     api: /apis/v1/flow/scene/fake_install_hdfs
     params:

dbm-ui/backend/flow/views/influxdb_fake_apply.py → dbm-ui/backend/flow/views/migrate_views/influxdb_fake_apply.py

@@ -15,12 +15,12 @@
 from rest_framework.response import Response
 
 from backend.flow.engine.controller.influxdb import InfluxdbController
-from backend.flow.views.base import FlowTestView
+from backend.flow.views.base import FlowTestView, MigrateFlowView
 
 logger = logging.getLogger("root")
 
 
-class FakeInstallInfluxdbSceneApiView(FlowTestView):
+class FakeInstallInfluxdbSceneApiView(MigrateFlowView):
     """
     api: /apis/v1/flow/scene/fake_install_influxdb
     params:

dbm-ui/backend/flow/views/kafka_fake_apply.py → dbm-ui/backend/flow/views/migrate_views/kafka_fake_apply.py

@@ -15,12 +15,12 @@
 from rest_framework.response import Response
 
 from backend.flow.engine.controller.kafka import KafkaController
-from backend.flow.views.base import FlowTestView
+from backend.flow.views.base import FlowTestView, MigrateFlowView
 
 logger = logging.getLogger("root")
 
 
-class FakeInstallKafkaSceneApiView(FlowTestView):
+class FakeInstallKafkaSceneApiView(MigrateFlowView):
     """
     api: /apis/v1/flow/scene/fake_install_kafka
     params:

dbm-ui/backend/flow/views/pulsar_fake_apply.py → dbm-ui/backend/flow/views/migrate_views/pulsar_fake_apply.py

@@ -14,12 +14,12 @@
 from rest_framework.response import Response
 
 from backend.flow.engine.controller.pulsar import PulsarController
-from backend.flow.views.base import FlowTestView
+from backend.flow.views.base import FlowTestView, MigrateFlowView
 
 logger = logging.getLogger("root")
 
 
-class FakeInstallPulsarSceneApiView(FlowTestView):
+class FakeInstallPulsarSceneApiView(MigrateFlowView):
     """
     api: /apis/v1/flow/scene/fake_install_pulsar
     params:

dbm-ui/backend/flow/views/migrate_views/redis_migrate.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+import uuid
+
+from rest_framework.response import Response
+
+from backend.flow.engine.controller.redis import RedisController
+from backend.flow.views.base import MigrateFlowView
+
+logger = logging.getLogger("root")
+
+
+class RedisClusterMigratePrecheck(MigrateFlowView):
+    """
+    集群迁移前置检查
+    """
+
+    @staticmethod
+    def post(request):
+        root_id = uuid.uuid1().hex
+        RedisController(root_id=root_id, ticket_data=request.data).redis_cluster_migrate_precheck()
+        return Response({"root_id": root_id})
+
+
+class RedisClusterMigrateLoad(MigrateFlowView):
+    """
+    集群迁移
+    """
+
+    @staticmethod
+    def post(request):
+        root_id = uuid.uuid1().hex
+        RedisController(root_id=root_id, ticket_data=request.data).redis_cluster_migrate_load()
+        return Response({"root_id": root_id})
+
+
+class RedisClusterMigrateCompair(MigrateFlowView):
+    """
+    集群迁移数据对比
+    """
+
+    @staticmethod
+    def post(request):
+        root_id = uuid.uuid1().hex
+        RedisController(root_id=root_id, ticket_data=request.data).redis_cluster_migrate_compair()
+        return Response({"root_id": root_id})