From d8b11ceea6c6c51f15c5dce400ea0ddcb05ef8ba Mon Sep 17 00:00:00 2001 From: yuanruji Date: Fri, 29 Nov 2024 12:02:54 +0800 Subject: [PATCH] =?UTF-8?q?feat(dbm-services):=20=E6=B8=85=E7=90=86dump?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=E7=9A=84=E8=A1=A8=E7=BB=93=E6=9E=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=20#8281?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/mysql/semantic_dump_schema.go | 50 +++++++++++++++---- .../bamboo/scene/mysql/import_sqlfile_flow.py | 5 +- .../scene/spider/import_sqlfile_flow.py | 6 +-- .../flow/utils/mysql/mysql_act_playload.py | 1 + 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go index de0a00f261..300deebad7 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go @@ -40,10 +40,14 @@ type SemanticDumpSchemaComp struct { // DumpSchemaParam TODO type DumpSchemaParam struct { - Host string `json:"host" validate:"required,ip"` // 当前实例的主机地址 - Port int `json:"port" validate:"required,lt=65536,gte=3306"` // 当前实例的端口 - CharSet string `json:"charset" validate:"required,checkCharset"` // 字符集参数 - CompressPkgName string `json:"com"` + // 当前实例的主机地址 + Host string `json:"host" validate:"required,ip"` + // 当前实例的端口 + Port int `json:"port" validate:"required,lt=65536,gte=3306"` + // 字符集参数 + CharSet string `json:"charset" validate:"required,checkCharset"` + // 备份文件名后缀,清理相关文件 + BackupFileNameSuffix string `json:"backup_file_name_suffix" validate:"required"` UploadBkRepoParam } @@ -74,6 +78,7 @@ type DumpSchemaRunTimeCtx struct { useTmysqldump bool // 使用自研的mysqldump 并发导出 isSpider bool // 是否spider中控 ignoreTables []string // 忽略的表集合 + gtidPurgedOff bool // 对于开启了gtid模式的实例，在导出时设置 --set-gtid-purged=OFF } // Example godoc @@ -88,12 +93,29 @@ func (c *SemanticDumpSchemaComp) Example() interface{} { } return comp } +func (c *SemanticDumpSchemaComp) cleanHistorySchemaFile() { + if c.Params.BackupFileNameSuffix == "" || c.Params.BackupDir == "" { + return + } + cleanCmd := fmt.Sprintf(`find %s -name "*%s.sql*" -type f -mtime +3 -delete `, c.Params.BackupDir, + c.Params.BackupFileNameSuffix) + logger.Warn("delete before 7 days dump schema file") + logger.Warn("will execute: %s", cleanCmd) + out, err := osutil.StandardShellCommand(false, cleanCmd) + if err != nil { + logger.Error("clean schema file failed:%s,out:%s", err.Error(), out) + return + } + logger.Warn("clean schema file success") +} // Init init // // @receiver c // @return err func (c *SemanticDumpSchemaComp) Init() (err error) { + // 1. clean history schema file + c.cleanHistorySchemaFile() conn, err := native.InsObject{ Host: c.Params.Host, Port: c.Params.Port, @@ -120,6 +142,9 @@ func (c *SemanticDumpSchemaComp) Init() (err error) { logger.Error("获取version failed %s", err.Error()) return err } + if cmutil.MySQLVersionParse(version) > cmutil.MySQLVersionParse("5.6.9") { + c.gtidPurgedOff = true + } c.isSpider = strings.Contains(version, "tdbctl") // 临时方案 if cmutil.FileExists("/home/mysql/dbbackup/mysqldump_x86_64") { @@ -135,6 +160,7 @@ func (c *SemanticDumpSchemaComp) Init() (err error) { } finaldbs := []string{} reg := regexp.MustCompile(`^bak_cbs`) + newBackupDbreg := regexp.MustCompile(`^stage_truncate`) ignoreDBs := computil.GetGcsSystemDatabasesIgnoreTest(version) if c.isSpider { // test 库里面的这些表没有主键，导入中控会失败 @@ -144,6 +170,9 @@ func (c *SemanticDumpSchemaComp) Init() (err error) { if reg.MatchString(db) { continue } + if newBackupDbreg.MatchString(db) { + continue + } finaldbs = append(finaldbs, db) } @@ -183,12 +212,13 @@ func (c *SemanticDumpSchemaComp) Precheck() (err error) { func (c *SemanticDumpSchemaComp) DumpSchema() (err error) { var dumper mysqlutil.Dumper dumpOption := mysqlutil.MySQLDumpOption{ - DumpSchema: true, - AddDropTable: true, - DumpRoutine: true, - DumpTrigger: false, - DumpEvent: true, - Quick: true, + DumpSchema: true, + AddDropTable: true, + DumpRoutine: true, + DumpTrigger: false, + DumpEvent: true, + Quick: true, + GtidPurgedOff: c.gtidPurgedOff, } if c.isSpider { dumpOption.GtidPurgedOff = true diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py index 5c8cf2f718..b8d083a47f 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py @@ -54,7 +54,8 @@ def __init__(self, root_id: str, data: Optional[Dict]): self.uid = self.data["uid"] # 定义好每次语义检测的库表备份文件名称 - self.semantic_dump_schema_file_name = f"{self.root_id}_semantic_dump_schema.sql" + self.semantic_dump_schema_file_name_suffix = "_semantic_dump_schema" + self.semantic_dump_schema_file_name = f"{self.root_id}{self.semantic_dump_schema_file_name_suffix}.sql" # 定义SQL文件的下发位置 self.sql_path = os.path.join(consts.BK_PKG_INSTALL_PATH, f"sqlfile_{self.uid}") + "/" @@ -188,7 +189,7 @@ def sql_semantic_check_flow(self): ) ), ) - + template_cluster["semantic_dump_schema_file_name_suffix"] = self.semantic_dump_schema_file_name_suffix semantic_check_pipeline.add_act( act_name=_("备份测试库表结构"), act_component_code=ExecuteDBActuatorScriptComponent.code, diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/import_sqlfile_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/import_sqlfile_flow.py index 2c299f61f8..ba48975509 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/import_sqlfile_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/import_sqlfile_flow.py @@ -56,8 +56,8 @@ def __init__(self, root_id: str, data: Optional[Dict]): self.uid = self.data["uid"] # 定义好每次语义检测的库表备份文件名称 - self.semantic_dump_schema_file_name = f"{self.root_id}_semantic_dump_schema.sql" - + self.semantic_dump_schema_file_name_suffix = "_semantic_dump_schema" + self.semantic_dump_schema_file_name = f"{self.root_id}{self.semantic_dump_schema_file_name_suffix}.sql" # 定义SQL文件的下发位置 self.sql_path = os.path.join(consts.BK_PKG_INSTALL_PATH, f"sqlfile_{self.uid}") + "/" self.data["sql_path"] = self.sql_path @@ -174,7 +174,7 @@ def sql_semantic_check_flow(self): ) ), ) - + cluster["semantic_dump_schema_file_name_suffix"] = self.semantic_dump_schema_file_name_suffix semantic_check_pipeline.add_act( act_name=_("备份测试库表结构"), act_component_code=ExecuteDBActuatorScriptComponent.code, diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py index a12ceab65b..0defd8b915 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py @@ -890,6 +890,7 @@ def get_semantic_dump_schema_payload(self, **kwargs): "port": self.cluster["port"], "charset": self.ticket_data["charset"], "backup_file_name": f"{self.cluster['semantic_dump_schema_file_name']}", + "backup_file_name_suffix": f"{self.cluster['semantic_dump_schema_file_name_suffix']}", "backup_dir": BK_PKG_INSTALL_PATH, "fileserver": { "url": get_bk_repo_url(bk_cloud_id),