feat(dbm-services): dbresource auto migrate TencentBlueKing#7291

ymakedaq · Oct 16, 2024 · a38d11b · a38d11b
1 parent 577873e
commit a38d11b
Show file tree

Hide file tree

Showing 6 changed files with 222 additions and 10 deletions.
diff --git a/dbm-services/common/db-resource/assets/assets.go b/dbm-services/common/db-resource/assets/assets.go
@@ -0,0 +1,61 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package assets
+
+import (
+	"embed"
+	"fmt"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/mysql" // mysql TODO
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/pkg/errors"
+
+	"dbm-services/common/go-pubpkg/logger"
+)
+
+// Migrations embed migrations sqlfile
+
+//go:embed migrations/*.sql
+var fs embed.FS
+
+// DoMigrateFromEmbed  do migrate from embed
+func DoMigrateFromEmbed(user, addr, password, dbname string, port int) (err error) {
+	var mig *migrate.Migrate
+	var d source.Driver
+	if d, err = iofs.New(fs, "migrations"); err != nil {
+		return err
+	}
+	dbURL := fmt.Sprintf(
+		"mysql://%s:%s@tcp(%s)/%s?charset=%s&parseTime=true&loc=Local&multiStatements=true&interpolateParams=true",
+		user,
+		password,
+		addr,
+		dbname,
+		"utf8",
+	)
+	mig, err = migrate.NewWithSourceInstance("iofs", d, dbURL)
+	if err != nil {
+		return errors.WithMessage(err, "migrate from embed")
+	}
+	defer mig.Close()
+	if err = mig.Up(); err != nil {
+		if err == migrate.ErrNoChange {
+			logger.Info("migrate source from embed success with", "msg", err.Error())
+			return nil
+		}
+		logger.Error("migrate source from embed failed", err)
+		return err
+	}
+	logger.Info("migrate source from embed success")
+	return nil
+}
diff --git a/dbm-services/common/db-resource/assets/migrations/000001_init.down.sql b/dbm-services/common/db-resource/assets/migrations/000001_init.down.sql
@@ -0,0 +1 @@
+SET names utf8;
diff --git a/dbm-services/common/db-resource/assets/migrations/000001_init.up.sql b/dbm-services/common/db-resource/assets/migrations/000001_init.up.sql
@@ -0,0 +1,130 @@
+SET names utf8;
+CREATE TABLE IF NOT EXISTS `tb_request_log` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `request_id` varchar(64) NOT NULL,
+    `request_user` varchar(32) NOT NULL,
+    `request_body` json DEFAULT NULL,
+    `request_url` varchar(32) NOT NULL,
+    `source_ip` varchar(32) NOT NULL,
+    `respone_body` json DEFAULT NULL,
+    `respone_code` int(11) NOT NULL,
+    `respone_messsage` text,
+    `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `request_id` (`request_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+CREATE TABLE IF NOT EXISTS `tb_rp_apply_detail_log` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `request_id` varchar(64) NOT NULL,
+    `item` varchar(64) NOT NULL,
+    `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID',
+    `ip` varchar(20) NOT NULL,
+    `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID',
+    `update_time` timestamp NULL DEFAULT NULL,
+    `create_time` datetime DEFAULT NULL,
+    PRIMARY KEY (`id`),
+    KEY `idx_request_id` (`request_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+CREATE TABLE IF NOT EXISTS `tb_rp_detail` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID',
+    `bk_biz_id` int(11) NOT NULL COMMENT '机器当前所属业务',
+    `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务',
+    `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型',
+    `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID',
+    `ip` varchar(20) NOT NULL,
+    `asset_id` varchar(64) NOT NULL COMMENT '固定资产编号',
+    `device_class` varchar(64) NOT NULL,
+    `svr_type_name` varchar(64) NOT NULL COMMENT '服务器型号,判断是否是云机器',
+    `cpu_num` int(11) NOT NULL COMMENT 'cpu核数',
+    `dram_cap` int(11) NOT NULL COMMENT '内存大小',
+    `storage_device` json DEFAULT NULL COMMENT '磁盘设备',
+    `total_storage_cap` int(11) DEFAULT NULL COMMENT '磁盘总容量',
+    `raid` varchar(20) NOT NULL,
+    `city_id` varchar(64) NOT NULL,
+    `city` varchar(128) NOT NULL,
+    `sub_zone` varchar(32) NOT NULL,
+    `sub_zone_id` varchar(64) NOT NULL,
+    `rack_id` varchar(64) NOT NULL,
+    `net_device_id` varchar(128) DEFAULT NULL,
+    `label` json DEFAULT NULL,
+    `is_init` int(11) DEFAULT NULL COMMENT '是否初始化过',
+    `is_idle` int(11) DEFAULT NULL COMMENT '是否空闲检查过',
+    `status` varchar(20) NOT NULL,
+    `bk_agent_id` varchar(64) NOT NULL,
+    `gse_agent_status_code` int(11) NOT NULL,
+    `agent_status_update_time` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01',
+    `consume_time` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01',
+    `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `os_type` varchar(32) NOT NULL COMMENT '操作系统类型',
+    `os_bit` varchar(32) NOT NULL COMMENT '操作系统位数',
+    `os_version` varchar(64) NOT NULL COMMENT '操作系统版本',
+    `os_name` varchar(64) NOT NULL COMMENT '操作系统名称',
+    `dedicated_bizs` json DEFAULT NULL COMMENT '专属业务,可属于多个',
+    `rs_types` json DEFAULT NULL COMMENT '资源类型标签',
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `ip` (`bk_cloud_id`, `ip`),
+    KEY `idx_host_id` (`bk_host_id`),
+    KEY `idx_bk_agent_id` (`bk_agent_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+CREATE TABLE IF NOT EXISTS `tb_rp_detail_archive` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID',
+    `bk_biz_id` int(11) NOT NULL COMMENT '机器当前所属业务',
+    `dedicated_bizs` json DEFAULT NULL COMMENT '专属业务,可属于多个',
+    `rs_types` json DEFAULT NULL COMMENT '资源类型标签',
+    `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID',
+    `ip` varchar(20) NOT NULL,
+    `asset_id` varchar(64) NOT NULL COMMENT '固定资产编号',
+    `device_class` varchar(64) NOT NULL,
+    `svr_type_name` varchar(64) NOT NULL COMMENT '服务器型号,判断是否是云机器',
+    `cpu_num` int(11) NOT NULL COMMENT 'cpu核数',
+    `dram_cap` int(11) NOT NULL COMMENT '内存大小',
+    `storage_device` json DEFAULT NULL COMMENT '磁盘设备',
+    `total_storage_cap` int(11) DEFAULT NULL COMMENT '磁盘总容量',
+    `os_type` varchar(32) NOT NULL COMMENT '操作系统类型',
+    `os_bit` varchar(32) NOT NULL COMMENT '操作系统位数',
+    `os_version` varchar(64) NOT NULL COMMENT '操作系统版本',
+    `raid` varchar(20) NOT NULL,
+    `city_id` varchar(64) NOT NULL,
+    `city` varchar(128) NOT NULL,
+    `sub_zone` varchar(32) NOT NULL,
+    `sub_zone_id` varchar(64) NOT NULL,
+    `rack_id` varchar(64) NOT NULL,
+    `net_device_id` varchar(128) DEFAULT NULL,
+    `label` json DEFAULT NULL,
+    `is_init` int(11) DEFAULT NULL COMMENT '是否初始化过',
+    `is_idle` int(11) DEFAULT NULL COMMENT '是否空闲检查过',
+    `status` varchar(20) NOT NULL,
+    `bk_agent_id` varchar(64) NOT NULL,
+    `gse_agent_status_code` int(11) NOT NULL,
+    `agent_status_update_time` timestamp NULL DEFAULT '1970-01-01 00:00:01',
+    `consume_time` timestamp NULL DEFAULT '1970-01-01 00:00:01',
+    `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `os_name` varchar(64) NOT NULL COMMENT '操作系统名称',
+    `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务',
+    `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型',
+    PRIMARY KEY (`id`),
+    KEY `idx_bk_agent_id` (`bk_agent_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+CREATE TABLE IF NOT EXISTS `tb_rp_operation_info` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `request_id` varchar(64) NOT NULL,
+    `total_count` int(11) DEFAULT NULL COMMENT 'task Id',
+    `bk_host_ids` json DEFAULT NULL COMMENT '主机Id',
+    `ip_list` json DEFAULT NULL COMMENT '主机ip',
+    `operation_type` varchar(64) NOT NULL COMMENT 'operation type',
+    `operator` varchar(64) NOT NULL COMMENT 'operator user',
+    `status` varchar(64) NOT NULL COMMENT ' status',
+    `task_id` varchar(128) NOT NULL COMMENT 'task Id',
+    `bill_id` varchar(128) NOT NULL COMMENT 'bill Id',
+    `bill_type` varchar(128) NOT NULL COMMENT 'bill type',
+    `description` varchar(256) DEFAULT '' COMMENT 'description',
+    `update_time` timestamp NULL DEFAULT NULL,
+    `create_time` datetime DEFAULT NULL,
+    PRIMARY KEY (`id`),
+    KEY `idx_request_id` (`request_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
diff --git a/dbm-services/common/db-resource/assets/migrations/000002_rename_col.down.sql b/dbm-services/common/db-resource/assets/migrations/000002_rename_col.down.sql
diff --git a/dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql b/dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql
@@ -0,0 +1,25 @@
+update tb_rp_detail
+set rs_type = JSON_UNQUOTE(JSON_EXTRACT(rs_types, '$[0]'))
+where JSON_EXTRACT(rs_types, '$[0]') is not null;
+update tb_rp_detail
+set dedicated_biz = JSON_UNQUOTE(JSON_EXTRACT(dedicated_bizs, '$[0]'))
+where JSON_EXTRACT(dedicated_bizs, '$[0]') is not null;
+-- tb_rp_detail add column dedicated_bizs, rs_types;
+alter table tb_rp_detail
+add `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务'
+after bk_biz_id;
+alter table tb_rp_detail
+add `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型'
+after dedicated_biz;
+--  tb_rp_detail_archive add column dedicated_bizs, rs_types;
+alter table tb_rp_detail_archive
+add `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务'
+after bk_biz_id;
+alter table tb_rp_detail_archive
+add `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型'
+after dedicated_biz;
+-- drop old column dedicated_bizs, rs_types;
+alter table tb_rp_detail drop dedicated_bizs,
+    drop rs_types;
+alter table tb_rp_detail_archive drop dedicated_bizs,
+    drop rs_types;
diff --git a/dbm-services/common/db-resource/internal/model/model.go b/dbm-services/common/db-resource/internal/model/model.go
@@ -20,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	"dbm-services/common/db-resource/assets"
 	"dbm-services/common/db-resource/internal/config"
 	"dbm-services/common/go-pubpkg/logger"
 
@@ -53,7 +54,6 @@ func init() {
 		Self:      ormDB,
 		SelfSqlDB: sqlDB,
 	}
-	migration()
 	initarchive()
 	TbRpOperationInfoColumns = []string{}
 	TbRpOperationInfoColumns, err = getTbRpOperationInfoColumns()
@@ -80,6 +80,10 @@ func createSysDb() {
 	if err != nil {
 		log.Fatalf("init create db failed:%s", err.Error())
 	}
+	err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname, 3306)
+	if err != nil {
+		log.Fatalf("init migrate from embed failed:%s", err.Error())
+	}
 	var autoIncrement sql.NullInt64
 	err = testConn.Raw(fmt.Sprintf("select max(id) from `%s`.`%s`", dbname, TbRpDetailArchiveName())).Scan(&autoIncrement).
 		Error
@@ -137,15 +141,6 @@ func initSelfDB() *gorm.DB {
 	)
 }
 
-// migration migration
-func migration() {
-	err := DB.Self.AutoMigrate(&TbRpDetail{}, &TbRequestLog{}, &TbRpDetailArchive{}, &TbRpApplyDetailLog{},
-		&TbRpOperationInfo{})
-	if err != nil {
-		logger.Error("auto migrate failed %v", err)
-	}
-}
-
 // JSONQueryExpression json query expression, implements clause.Expression interface to use as querier
 type JSONQueryExpression struct {
 	column             string