From a38d11b4e0fa6e3bd51a4f729ec423c38af72add Mon Sep 17 00:00:00 2001 From: yuanruji Date: Wed, 16 Oct 2024 10:22:48 +0800 Subject: [PATCH] feat(dbm-services): dbresource auto migrate #7291 --- .../common/db-resource/assets/assets.go | 61 ++++++++ .../assets/migrations/000001_init.down.sql | 1 + .../assets/migrations/000001_init.up.sql | 130 ++++++++++++++++++ .../migrations/000002_rename_col.down.sql | 0 .../migrations/000002_rename_col.up.sql | 25 ++++ .../db-resource/internal/model/model.go | 15 +- 6 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 dbm-services/common/db-resource/assets/assets.go create mode 100644 dbm-services/common/db-resource/assets/migrations/000001_init.down.sql create mode 100644 dbm-services/common/db-resource/assets/migrations/000001_init.up.sql create mode 100644 dbm-services/common/db-resource/assets/migrations/000002_rename_col.down.sql create mode 100644 dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql diff --git a/dbm-services/common/db-resource/assets/assets.go b/dbm-services/common/db-resource/assets/assets.go new file mode 100644 index 0000000000..089862d1db --- /dev/null +++ b/dbm-services/common/db-resource/assets/assets.go @@ -0,0 +1,61 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package assets + +import ( + "embed" + "fmt" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/mysql" // mysql TODO + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/pkg/errors" + + "dbm-services/common/go-pubpkg/logger" +) + +// Migrations embed migrations sqlfile + +//go:embed migrations/*.sql +var fs embed.FS + +// DoMigrateFromEmbed do migrate from embed +func DoMigrateFromEmbed(user, addr, password, dbname string, port int) (err error) { + var mig *migrate.Migrate + var d source.Driver + if d, err = iofs.New(fs, "migrations"); err != nil { + return err + } + dbURL := fmt.Sprintf( + "mysql://%s:%s@tcp(%s)/%s?charset=%s&parseTime=true&loc=Local&multiStatements=true&interpolateParams=true", + user, + password, + addr, + dbname, + "utf8", + ) + mig, err = migrate.NewWithSourceInstance("iofs", d, dbURL) + if err != nil { + return errors.WithMessage(err, "migrate from embed") + } + defer mig.Close() + if err = mig.Up(); err != nil { + if err == migrate.ErrNoChange { + logger.Info("migrate source from embed success with", "msg", err.Error()) + return nil + } + logger.Error("migrate source from embed failed", err) + return err + } + logger.Info("migrate source from embed success") + return nil +} diff --git a/dbm-services/common/db-resource/assets/migrations/000001_init.down.sql b/dbm-services/common/db-resource/assets/migrations/000001_init.down.sql new file mode 100644 index 0000000000..e006a43a7b --- /dev/null +++ b/dbm-services/common/db-resource/assets/migrations/000001_init.down.sql @@ -0,0 +1 @@ +SET names utf8; \ No newline at end of file diff --git a/dbm-services/common/db-resource/assets/migrations/000001_init.up.sql b/dbm-services/common/db-resource/assets/migrations/000001_init.up.sql new file mode 100644 index 0000000000..46ed3dc7f1 --- /dev/null +++ b/dbm-services/common/db-resource/assets/migrations/000001_init.up.sql @@ -0,0 +1,130 @@ +SET names utf8; +CREATE TABLE IF NOT EXISTS `tb_request_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `request_id` varchar(64) NOT NULL, + `request_user` varchar(32) NOT NULL, + `request_body` json DEFAULT NULL, + `request_url` varchar(32) NOT NULL, + `source_ip` varchar(32) NOT NULL, + `respone_body` json DEFAULT NULL, + `respone_code` int(11) NOT NULL, + `respone_messsage` text, + `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `request_id` (`request_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; +CREATE TABLE IF NOT EXISTS `tb_rp_apply_detail_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `request_id` varchar(64) NOT NULL, + `item` varchar(64) NOT NULL, + `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID', + `ip` varchar(20) NOT NULL, + `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID', + `update_time` timestamp NULL DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `idx_request_id` (`request_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; +CREATE TABLE IF NOT EXISTS `tb_rp_detail` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID', + `bk_biz_id` int(11) NOT NULL COMMENT '机器当前所属业务', + `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务', + `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型', + `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID', + `ip` varchar(20) NOT NULL, + `asset_id` varchar(64) NOT NULL COMMENT '固定资产编号', + `device_class` varchar(64) NOT NULL, + `svr_type_name` varchar(64) NOT NULL COMMENT '服务器型号,判断是否是云机器', + `cpu_num` int(11) NOT NULL COMMENT 'cpu核数', + `dram_cap` int(11) NOT NULL COMMENT '内存大小', + `storage_device` json DEFAULT NULL COMMENT '磁盘设备', + `total_storage_cap` int(11) DEFAULT NULL COMMENT '磁盘总容量', + `raid` varchar(20) NOT NULL, + `city_id` varchar(64) NOT NULL, + `city` varchar(128) NOT NULL, + `sub_zone` varchar(32) NOT NULL, + `sub_zone_id` varchar(64) NOT NULL, + `rack_id` varchar(64) NOT NULL, + `net_device_id` varchar(128) DEFAULT NULL, + `label` json DEFAULT NULL, + `is_init` int(11) DEFAULT NULL COMMENT '是否初始化过', + `is_idle` int(11) DEFAULT NULL COMMENT '是否空闲检查过', + `status` varchar(20) NOT NULL, + `bk_agent_id` varchar(64) NOT NULL, + `gse_agent_status_code` int(11) NOT NULL, + `agent_status_update_time` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01', + `consume_time` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01', + `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `os_type` varchar(32) NOT NULL COMMENT '操作系统类型', + `os_bit` varchar(32) NOT NULL COMMENT '操作系统位数', + `os_version` varchar(64) NOT NULL COMMENT '操作系统版本', + `os_name` varchar(64) NOT NULL COMMENT '操作系统名称', + `dedicated_bizs` json DEFAULT NULL COMMENT '专属业务,可属于多个', + `rs_types` json DEFAULT NULL COMMENT '资源类型标签', + PRIMARY KEY (`id`), + UNIQUE KEY `ip` (`bk_cloud_id`, `ip`), + KEY `idx_host_id` (`bk_host_id`), + KEY `idx_bk_agent_id` (`bk_agent_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; +CREATE TABLE IF NOT EXISTS `tb_rp_detail_archive` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `bk_cloud_id` int(11) NOT NULL COMMENT '云区域 ID', + `bk_biz_id` int(11) NOT NULL COMMENT '机器当前所属业务', + `dedicated_bizs` json DEFAULT NULL COMMENT '专属业务,可属于多个', + `rs_types` json DEFAULT NULL COMMENT '资源类型标签', + `bk_host_id` int(11) NOT NULL COMMENT 'bk主机ID', + `ip` varchar(20) NOT NULL, + `asset_id` varchar(64) NOT NULL COMMENT '固定资产编号', + `device_class` varchar(64) NOT NULL, + `svr_type_name` varchar(64) NOT NULL COMMENT '服务器型号,判断是否是云机器', + `cpu_num` int(11) NOT NULL COMMENT 'cpu核数', + `dram_cap` int(11) NOT NULL COMMENT '内存大小', + `storage_device` json DEFAULT NULL COMMENT '磁盘设备', + `total_storage_cap` int(11) DEFAULT NULL COMMENT '磁盘总容量', + `os_type` varchar(32) NOT NULL COMMENT '操作系统类型', + `os_bit` varchar(32) NOT NULL COMMENT '操作系统位数', + `os_version` varchar(64) NOT NULL COMMENT '操作系统版本', + `raid` varchar(20) NOT NULL, + `city_id` varchar(64) NOT NULL, + `city` varchar(128) NOT NULL, + `sub_zone` varchar(32) NOT NULL, + `sub_zone_id` varchar(64) NOT NULL, + `rack_id` varchar(64) NOT NULL, + `net_device_id` varchar(128) DEFAULT NULL, + `label` json DEFAULT NULL, + `is_init` int(11) DEFAULT NULL COMMENT '是否初始化过', + `is_idle` int(11) DEFAULT NULL COMMENT '是否空闲检查过', + `status` varchar(20) NOT NULL, + `bk_agent_id` varchar(64) NOT NULL, + `gse_agent_status_code` int(11) NOT NULL, + `agent_status_update_time` timestamp NULL DEFAULT '1970-01-01 00:00:01', + `consume_time` timestamp NULL DEFAULT '1970-01-01 00:00:01', + `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `os_name` varchar(64) NOT NULL COMMENT '操作系统名称', + `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务', + `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型', + PRIMARY KEY (`id`), + KEY `idx_bk_agent_id` (`bk_agent_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; +CREATE TABLE IF NOT EXISTS `tb_rp_operation_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `request_id` varchar(64) NOT NULL, + `total_count` int(11) DEFAULT NULL COMMENT 'task Id', + `bk_host_ids` json DEFAULT NULL COMMENT '主机Id', + `ip_list` json DEFAULT NULL COMMENT '主机ip', + `operation_type` varchar(64) NOT NULL COMMENT 'operation type', + `operator` varchar(64) NOT NULL COMMENT 'operator user', + `status` varchar(64) NOT NULL COMMENT ' status', + `task_id` varchar(128) NOT NULL COMMENT 'task Id', + `bill_id` varchar(128) NOT NULL COMMENT 'bill Id', + `bill_type` varchar(128) NOT NULL COMMENT 'bill type', + `description` varchar(256) DEFAULT '' COMMENT 'description', + `update_time` timestamp NULL DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `idx_request_id` (`request_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; \ No newline at end of file diff --git a/dbm-services/common/db-resource/assets/migrations/000002_rename_col.down.sql b/dbm-services/common/db-resource/assets/migrations/000002_rename_col.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql b/dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql new file mode 100644 index 0000000000..672c9aa129 --- /dev/null +++ b/dbm-services/common/db-resource/assets/migrations/000002_rename_col.up.sql @@ -0,0 +1,25 @@ +update tb_rp_detail +set rs_type = JSON_UNQUOTE(JSON_EXTRACT(rs_types, '$[0]')) +where JSON_EXTRACT(rs_types, '$[0]') is not null; +update tb_rp_detail +set dedicated_biz = JSON_UNQUOTE(JSON_EXTRACT(dedicated_bizs, '$[0]')) +where JSON_EXTRACT(dedicated_bizs, '$[0]') is not null; +-- tb_rp_detail add column dedicated_bizs, rs_types; +alter table tb_rp_detail +add `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务' +after bk_biz_id; +alter table tb_rp_detail +add `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型' +after dedicated_biz; +-- tb_rp_detail_archive add column dedicated_bizs, rs_types; +alter table tb_rp_detail_archive +add `dedicated_biz` int(11) DEFAULT '0' COMMENT '专属业务' +after bk_biz_id; +alter table tb_rp_detail_archive +add `rs_type` varchar(64) DEFAULT 'PUBLIC' COMMENT '资源专用组件类型' +after dedicated_biz; +-- drop old column dedicated_bizs, rs_types; +alter table tb_rp_detail drop dedicated_bizs, + drop rs_types; +alter table tb_rp_detail_archive drop dedicated_bizs, + drop rs_types; \ No newline at end of file diff --git a/dbm-services/common/db-resource/internal/model/model.go b/dbm-services/common/db-resource/internal/model/model.go index 8cec4d28fe..ccf42afb3c 100644 --- a/dbm-services/common/db-resource/internal/model/model.go +++ b/dbm-services/common/db-resource/internal/model/model.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "dbm-services/common/db-resource/assets" "dbm-services/common/db-resource/internal/config" "dbm-services/common/go-pubpkg/logger" @@ -53,7 +54,6 @@ func init() { Self: ormDB, SelfSqlDB: sqlDB, } - migration() initarchive() TbRpOperationInfoColumns = []string{} TbRpOperationInfoColumns, err = getTbRpOperationInfoColumns() @@ -80,6 +80,10 @@ func createSysDb() { if err != nil { log.Fatalf("init create db failed:%s", err.Error()) } + err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname, 3306) + if err != nil { + log.Fatalf("init migrate from embed failed:%s", err.Error()) + } var autoIncrement sql.NullInt64 err = testConn.Raw(fmt.Sprintf("select max(id) from `%s`.`%s`", dbname, TbRpDetailArchiveName())).Scan(&autoIncrement). Error @@ -137,15 +141,6 @@ func initSelfDB() *gorm.DB { ) } -// migration migration -func migration() { - err := DB.Self.AutoMigrate(&TbRpDetail{}, &TbRequestLog{}, &TbRpDetailArchive{}, &TbRpApplyDetailLog{}, - &TbRpOperationInfo{}) - if err != nil { - logger.Error("auto migrate failed %v", err) - } -} - // JSONQueryExpression json query expression, implements clause.Expression interface to use as querier type JSONQueryExpression struct { column string