feat(dbm-services): 清理dump出来的表结构文件 #8281

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go

@@ -40,10 +40,14 @@ type SemanticDumpSchemaComp struct {
 
 // DumpSchemaParam TODO
 type DumpSchemaParam struct {
-	Host            string `json:"host"  validate:"required,ip"`                // 当前实例的主机地址
-	Port            int    `json:"port"  validate:"required,lt=65536,gte=3306"` // 当前实例的端口
-	CharSet         string `json:"charset" validate:"required,checkCharset"`    // 字符集参数
-	CompressPkgName string `json:"com"`
+	// 当前实例的主机地址
+	Host string `json:"host"  validate:"required,ip"`
+	// 当前实例的端口
+	Port int `json:"port"  validate:"required,lt=65536,gte=3306"`
+	// 字符集参数
+	CharSet string `json:"charset" validate:"required,checkCharset"`
+	// 备份文件名后缀,清理相关文件
+	BackupFileNameSuffix string `json:"backup_file_name_suffix" validate:"required"`
 	UploadBkRepoParam
 }
 
@@ -74,6 +78,7 @@ type DumpSchemaRunTimeCtx struct {
 	useTmysqldump bool     // 使用自研的mysqldump 并发导出
 	isSpider      bool     // 是否spider中控
 	ignoreTables  []string // 忽略的表集合
+	gtidPurgedOff bool     // 对于开启了gtid模式的实例，在导出时设置 --set-gtid-purged=OFF
 }
 
 // Example godoc
@@ -88,12 +93,29 @@ func (c *SemanticDumpSchemaComp) Example() interface{} {
 	}
 	return comp
 }
+func (c *SemanticDumpSchemaComp) cleanHistorySchemaFile() {
+	if c.Params.BackupFileNameSuffix == "" || c.Params.BackupDir == "" {
+		return
+	}
+	cleanCmd := fmt.Sprintf(`find %s -name "*%s.sql*" -type f -mtime +3 -delete `, c.Params.BackupDir,
+		c.Params.BackupFileNameSuffix)
+	logger.Warn("delete before 7 days dump schema file")
+	logger.Warn("will execute: %s", cleanCmd)
+	out, err := osutil.StandardShellCommand(false, cleanCmd)
+	if err != nil {
+		logger.Error("clean schema file failed:%s,out:%s", err.Error(), out)
+		return
+	}
+	logger.Warn("clean schema file success")
+}
 
 // Init init
 //
 //	@receiver c
 //	@return err
 func (c *SemanticDumpSchemaComp) Init() (err error) {
+	// 1. clean history schema file
+	c.cleanHistorySchemaFile()
 	conn, err := native.InsObject{
 		Host: c.Params.Host,
 		Port: c.Params.Port,
@@ -120,6 +142,9 @@ func (c *SemanticDumpSchemaComp) Init() (err error) {
 		logger.Error("获取version failed %s", err.Error())
 		return err
 	}
+	if cmutil.MySQLVersionParse(version) > cmutil.MySQLVersionParse("5.6.9") {
+		c.gtidPurgedOff = true
+	}
 	c.isSpider = strings.Contains(version, "tdbctl")
 	// 临时方案
 	if cmutil.FileExists("/home/mysql/dbbackup/mysqldump_x86_64") {
@@ -135,6 +160,7 @@ func (c *SemanticDumpSchemaComp) Init() (err error) {
 	}
 	finaldbs := []string{}
 	reg := regexp.MustCompile(`^bak_cbs`)
+	newBackupDbreg := regexp.MustCompile(`^stage_truncate`)
 	ignoreDBs := computil.GetGcsSystemDatabasesIgnoreTest(version)
 	if c.isSpider {
 		// test 库里面的这些表没有主键，导入中控会失败
@@ -144,6 +170,9 @@ func (c *SemanticDumpSchemaComp) Init() (err error) {
 		if reg.MatchString(db) {
 			continue
 		}
+		if newBackupDbreg.MatchString(db) {
+			continue
+		}
 		finaldbs = append(finaldbs, db)
 	}
 
@@ -183,12 +212,13 @@ func (c *SemanticDumpSchemaComp) Precheck() (err error) {
 func (c *SemanticDumpSchemaComp) DumpSchema() (err error) {
 	var dumper mysqlutil.Dumper
 	dumpOption := mysqlutil.MySQLDumpOption{
-		DumpSchema:   true,
-		AddDropTable: true,
-		DumpRoutine:  true,
-		DumpTrigger:  false,
-		DumpEvent:    true,
-		Quick:        true,
+		DumpSchema:    true,
+		AddDropTable:  true,
+		DumpRoutine:   true,
+		DumpTrigger:   false,
+		DumpEvent:     true,
+		Quick:         true,
+		GtidPurgedOff: c.gtidPurgedOff,
 	}
 	if c.isSpider {
 		dumpOption.GtidPurgedOff = true

dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py

@@ -54,7 +54,8 @@ def __init__(self, root_id: str, data: Optional[Dict]):
         self.uid = self.data["uid"]
 
         # 定义好每次语义检测的库表备份文件名称
-        self.semantic_dump_schema_file_name = f"{self.root_id}_semantic_dump_schema.sql"
+        self.semantic_dump_schema_file_name_suffix = "_semantic_dump_schema"
+        self.semantic_dump_schema_file_name = f"{self.root_id}{self.semantic_dump_schema_file_name_suffix}.sql"
 
         # 定义SQL文件的下发位置
         self.sql_path = os.path.join(consts.BK_PKG_INSTALL_PATH, f"sqlfile_{self.uid}") + "/"
@@ -188,7 +189,7 @@ def sql_semantic_check_flow(self):
                 )
             ),
         )
-
+        template_cluster["semantic_dump_schema_file_name_suffix"] = self.semantic_dump_schema_file_name_suffix
         semantic_check_pipeline.add_act(
             act_name=_("备份测试库表结构"),
             act_component_code=ExecuteDBActuatorScriptComponent.code,

dbm-ui/backend/flow/engine/bamboo/scene/spider/import_sqlfile_flow.py

@@ -56,8 +56,8 @@ def __init__(self, root_id: str, data: Optional[Dict]):
         self.uid = self.data["uid"]
 
         # 定义好每次语义检测的库表备份文件名称
-        self.semantic_dump_schema_file_name = f"{self.root_id}_semantic_dump_schema.sql"
-
+        self.semantic_dump_schema_file_name_suffix = "_semantic_dump_schema"
+        self.semantic_dump_schema_file_name = f"{self.root_id}{self.semantic_dump_schema_file_name_suffix}.sql"
         # 定义SQL文件的下发位置
         self.sql_path = os.path.join(consts.BK_PKG_INSTALL_PATH, f"sqlfile_{self.uid}") + "/"
         self.data["sql_path"] = self.sql_path
@@ -174,7 +174,7 @@ def sql_semantic_check_flow(self):
                 )
             ),
         )
-
+        cluster["semantic_dump_schema_file_name_suffix"] = self.semantic_dump_schema_file_name_suffix
         semantic_check_pipeline.add_act(
             act_name=_("备份测试库表结构"),
             act_component_code=ExecuteDBActuatorScriptComponent.code,

dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py

@@ -890,6 +890,7 @@ def get_semantic_dump_schema_payload(self, **kwargs):
                     "port": self.cluster["port"],
                     "charset": self.ticket_data["charset"],
                     "backup_file_name": f"{self.cluster['semantic_dump_schema_file_name']}",
+                    "backup_file_name_suffix": f"{self.cluster['semantic_dump_schema_file_name_suffix']}",
                     "backup_dir": BK_PKG_INSTALL_PATH,
                     "fileserver": {
                         "url": get_bk_repo_url(bk_cloud_id),