feat: if enable backup sqle will backup for every sql

sqle/api/controller/v1/backup.go

@@ -6,7 +6,7 @@ import (
 )
 
 type UpdateSqlBackupStrategyReq struct {
-	Strategy string `json:"strategy" enums:"none,manual,reverse_sql,origin_row"`
+	Strategy string `json:"strategy" enums:"none,manual,reverse_sql,original_row"`
 }
 
 // UpdateSqlBackupStrategy
@@ -27,7 +27,7 @@ func UpdateSqlBackupStrategy(c echo.Context) error {
 }
 
 type UpdateTaskBackupStrategyReq struct {
-	Strategy string `json:"strategy" enums:"none,manual,reverse_sql,origin_row"`
+	Strategy string `json:"strategy" enums:"none,manual,reverse_sql,original_row"`
 }
 
 // UpdateTaskBackupStrategy
@@ -73,7 +73,7 @@ type BackupSqlData struct {
 	OriginSQL      string   `json:"origin_sql"`
 	OriginTaskId   uint     `json:"origin_task_id"`
 	BackupSqls     []string `json:"backup_sqls"`
-	BackupStrategy string   `json:"backup_strategy" enums:"none,manual,reverse_sql,origin_row"`
+	BackupStrategy string   `json:"backup_strategy" enums:"none,manual,reverse_sql,original_row"`
 	InstanceName   string   `json:"instance_name"`
 	InstanceId     string   `json:"instance_id"`
 	ExecStatus     string   `json:"exec_status"`

sqle/api/controller/v1/task.go

@@ -484,15 +484,15 @@ type GetAuditTaskSQLsResV1 struct {
 }
 
 type AuditTaskSQLResV1 struct {
-	Number      uint   `json:"number"`
-	ExecSQL     string `json:"exec_sql"`
-	AuditResult string `json:"audit_result"`
-	AuditLevel  string `json:"audit_level"`
-	AuditStatus string `json:"audit_status"`
-	ExecResult  string `json:"exec_result"`
-	ExecStatus  string `json:"exec_status"`
-	RollbackSQL string `json:"rollback_sql,omitempty"`
-	Description string `json:"description"`
+	Number       uint     `json:"number"`
+	ExecSQL      string   `json:"exec_sql"`
+	AuditResult  string   `json:"audit_result"`
+	AuditLevel   string   `json:"audit_level"`
+	AuditStatus  string   `json:"audit_status"`
+	ExecResult   string   `json:"exec_result"`
+	ExecStatus   string   `json:"exec_status"`
+	RollbackSQLs []string `json:"rollback_sqls,omitempty"`
+	Description  string   `json:"description"`
 }
 
 // @Summary 获取指定扫描任务的SQLs信息
@@ -542,19 +542,22 @@ func GetTaskSQLs(c echo.Context) error {
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
 	}
-
+	rollbackSqlMap, err := server.BackupService{}.GetRollbackSqlsMap(task.ID)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
 	taskSQLsRes := make([]*AuditTaskSQLResV1, 0, len(taskSQLs))
 	for _, taskSQL := range taskSQLs {
 		taskSQLRes := &AuditTaskSQLResV1{
-			Number:      taskSQL.Number,
-			Description: taskSQL.Description,
-			ExecSQL:     taskSQL.ExecSQL,
-			AuditResult: taskSQL.GetAuditResults(ctx),
-			AuditLevel:  taskSQL.AuditLevel,
-			AuditStatus: taskSQL.AuditStatus,
-			ExecResult:  taskSQL.ExecResult,
-			ExecStatus:  taskSQL.ExecStatus,
-			RollbackSQL: taskSQL.RollbackSQL.String,
+			Number:       taskSQL.Number,
+			Description:  taskSQL.Description,
+			ExecSQL:      taskSQL.ExecSQL,
+			AuditResult:  taskSQL.GetAuditResults(ctx),
+			AuditLevel:   taskSQL.AuditLevel,
+			AuditStatus:  taskSQL.AuditStatus,
+			ExecResult:   taskSQL.ExecResult,
+			ExecStatus:   taskSQL.ExecStatus,
+			RollbackSQLs: rollbackSqlMap[taskSQL.Id],
 		}
 		taskSQLsRes = append(taskSQLsRes, taskSQLRes)
 	}
@@ -623,6 +626,10 @@ func DownloadTaskSQLReportFile(c echo.Context) error {
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, errors.New(errors.WriteDataToTheFileError, err))
 	}
+	rollbackSqlMap, err := server.BackupService{}.GetRollbackSqlsMap(task.ID)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
 	for _, td := range taskSQLsDetail {
 		taskSql := &model.ExecuteSQL{
 			AuditResults: td.AuditResults,
@@ -636,7 +643,7 @@ func DownloadTaskSQLReportFile(c echo.Context) error {
 			taskSql.GetAuditResultDesc(ctx),
 			taskSql.GetExecStatusDesc(ctx),
 			td.ExecResult,
-			td.RollbackSQL.String,
+			strings.Join(rollbackSqlMap[taskSql.ID], "\n"),
 			td.Description,
 		})
 		if err != nil {

sqle/api/controller/v2/task.go

@@ -7,6 +7,7 @@ import (
 	v1 "github.com/actiontech/sqle/sqle/api/controller/v1"
 	"github.com/actiontech/sqle/sqle/locale"
 	"github.com/actiontech/sqle/sqle/model"
+	"github.com/actiontech/sqle/sqle/server"
 
 	"github.com/labstack/echo/v4"
 )
@@ -109,19 +110,33 @@ func GetTaskSQLs(c echo.Context) error {
 	}
 
 	taskSQLsRes := make([]*AuditTaskSQLResV2, 0, len(taskSQLs))
+	backupService := server.BackupService{}
+	rollbackSqlMap, err := backupService.GetRollbackSqlsMap(task.ID)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
+	backupTaskMap, err := backupService.GetBackupTasksMap(task.ID)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
 	for _, taskSQL := range taskSQLs {
 		taskSQLRes := &AuditTaskSQLResV2{
-			Number:        taskSQL.Number,
-			Description:   taskSQL.Description,
-			ExecSQL:       taskSQL.ExecSQL,
-			SQLSourceFile: taskSQL.SQLSourceFile.String,
-			SQLStartLine:  taskSQL.SQLStartLine,
-			AuditLevel:    taskSQL.AuditLevel,
-			AuditStatus:   taskSQL.AuditStatus,
-			ExecResult:    taskSQL.ExecResult,
-			ExecStatus:    taskSQL.ExecStatus,
-			RollbackSQLs:  []string{taskSQL.RollbackSQL.String},
-			SQLType:       taskSQL.SQLType.String,
+			ExecSqlID:                   taskSQL.Id,
+			Number:                      taskSQL.Number,
+			Description:                 taskSQL.Description,
+			ExecSQL:                     taskSQL.ExecSQL,
+			SQLSourceFile:               taskSQL.SQLSourceFile.String,
+			SQLStartLine:                taskSQL.SQLStartLine,
+			AuditLevel:                  taskSQL.AuditLevel,
+			AuditStatus:                 taskSQL.AuditStatus,
+			ExecResult:                  taskSQL.ExecResult,
+			ExecStatus:                  taskSQL.ExecStatus,
+			RollbackSQLs:                rollbackSqlMap[taskSQL.Id],
+			SQLType:                     taskSQL.SQLType.String,
+			BackupStrategy:              backupTaskMap.GetBackupStrategy(taskSQL.Id),
+			BackupStrategyTip:           backupTaskMap.GetBackupStrategyTip(taskSQL.Id),
 		}
 		for i := range taskSQL.AuditResults {
 			ar := taskSQL.AuditResults[i]

sqle/docs/docs.go

@@ -11735,8 +11735,11 @@ var doc = `{
                 "number": {
                     "type": "integer"
                 },
-                "rollback_sql": {
-                    "type": "string"
+                "rollback_sqls": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
                 }
             }
         },
@@ -11797,7 +11800,7 @@ var doc = `{
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 },
                 "description": {
@@ -17595,7 +17598,7 @@ var doc = `{
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 }
             }
@@ -17684,7 +17687,7 @@ var doc = `{
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 }
             }

sqle/docs/swagger.json

@@ -11719,8 +11719,11 @@
                 "number": {
                     "type": "integer"
                 },
-                "rollback_sql": {
-                    "type": "string"
+                "rollback_sqls": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
                 }
             }
         },
@@ -11781,7 +11784,7 @@
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 },
                 "description": {
@@ -17579,7 +17582,7 @@
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 }
             }
@@ -17668,7 +17671,7 @@
                         "none",
                         "manual",
                         "reverse_sql",
-                        "origin_row"
+                        "original_row"
                     ]
                 }
             }

sqle/docs/swagger.yaml

@@ -456,8 +456,10 @@ definitions:
         type: string
       number:
         type: integer
-      rollback_sql:
-        type: string
+      rollback_sqls:
+        items:
+          type: string
+        type: array
     type: object
   v1.AuditTasksGroupResV1:
     properties:
@@ -497,7 +499,7 @@ definitions:
         - none
         - manual
         - reverse_sql
-        - origin_row
+        - original_row
         type: string
       description:
         type: string
@@ -4452,7 +4454,7 @@ definitions:
         - none
         - manual
         - reverse_sql
-        - origin_row
+        - original_row
         type: string
     type: object
   v1.UpdateSqlFileOrderV1Req:
@@ -4512,7 +4514,7 @@ definitions:
         - none
         - manual
         - reverse_sql
-        - origin_row
+        - original_row
         type: string
     type: object
   v1.UpdateWechatConfigurationReqV1:

sqle/driver/mysql/mysql.go

@@ -99,7 +99,7 @@ func (inspect *MysqlDriverImpl) applyConfig(cfg *driverV2.Config) {
 	inspect.isOfflineAudit = cfg.DSN == nil
 
 	inspect.cnf = &Config{
-		DMLRollbackMaxRows: -1,
+		DMLRollbackMaxRows: 1000,// TODO 暂时将备份影响行数上限设置为1000，后续需要将对应该配置项的规则移除。备份影响行数上限设置将会在备份任务中设置。
 		DDLOSCMinSize:      -1,
 		DDLGhostMinSize:    -1,
 	}

sqle/model/backup_task.go

@@ -0,0 +1,22 @@
+package model
+
+import (
+	"gorm.io/gorm"
+)
+
+func init() {
+	autoMigrateList = append(autoMigrateList, &BackupTask{})
+}
+
+type BackupTask struct {
+	gorm.Model
+	TaskId            uint   `gorm:"column:task_id;not null"`                                                                         // 备份任务关联原始SQL的任务的id
+	InstanceId        uint64 `gorm:"column:instance_id;not null"`                                                                     // 备份任务关联原始SQL对应的数据源
+	ExecuteSqlId      uint   `gorm:"index;column:execute_sql_id;not null"`                                                            // 备份任务关联原始SQL的id
+	BackupStrategy    string `gorm:"type:enum('none','reverse_sql','original_row','manual');column:backup_strategy;size:20;not null"` // 备份任务支持的备份策略
+	BackupStrategyTip string `gorm:"column:backup_strategy_tip;size:255;not null;default:''"`                                         // 推荐备份任务的原因
+	BackupStatus      string `gorm:"type:enum('waiting_for_execution','executing','failed','succeed');column:backup_status;size:20"`  // 备份任务的执行状态
+	BackupExecResult  string `gorm:"column:backup_exec_result;size:255;not null;default:''"`                                          // 备份任务的执行结果
+	SchemaName        string `gorm:"column:schema_name;size:50;not null;default:''"`                                                  // 备份的SQL对应的schema
+	TableName         string `gorm:"column:table_name;size:50;not null;default:''"`                                                   // 备份的SQL对应的table
+}

sqle/model/backup_task_ce.go

@@ -0,0 +1,12 @@
+//go:build !enterprise
+// +build !enterprise
+
+package model
+
+func (s *Storage) BatchCreateBackupTasks(backupTasks []*BackupTask) error {
+	return nil
+}
+
+func (s *Storage) GetBackupTaskByExecuteSqlId(executeSqlId uint) (*BackupTask, error){
+	return &BackupTask{},nil
+}

sqle/model/task.go

@@ -321,6 +321,7 @@ type ExecuteSQL struct {
 	AuditFingerprint string `json:"audit_fingerprint" gorm:"index;type:char(32)"`
 	// AuditLevel has four level: error, warn, notice, normal.
 	AuditLevel string `json:"audit_level" gorm:"type:varchar(255)"`
+	BackupTask *BackupTask `json:"-" gorm:"foreignkey:execute_sql_id"`
 }
 
 func (s ExecuteSQL) TableName() string {
@@ -590,18 +591,19 @@ func (s *Storage) GetTaskByInstanceId(instanceId uint64) ([]Task, error) {
 }
 
 type TaskSQLDetail struct {
-	Number        uint           `json:"number"`
-	Description   string         `json:"description"`
-	ExecSQL       string         `json:"exec_sql"`
-	SQLSourceFile sql.NullString `json:"sql_source_file"`
-	SQLStartLine  uint64         `json:"sql_start_line"`
-	AuditResults  AuditResults   `json:"audit_results"`
-	AuditLevel    string         `json:"audit_level"`
-	AuditStatus   string         `json:"audit_status"`
-	ExecResult    string         `json:"exec_result"`
-	ExecStatus    string         `json:"exec_status"`
-	RollbackSQL   sql.NullString `json:"rollback_sql"`
-	SQLType       sql.NullString `json:"sql_type"`
+	Id                uint           `json:"id"`
+	Number            uint           `json:"number"`
+	Description       string         `json:"description"`
+	ExecSQL           string         `json:"exec_sql"`
+	SQLSourceFile     sql.NullString `json:"sql_source_file"`
+	SQLStartLine      uint64         `json:"sql_start_line"`
+	AuditResults      AuditResults   `json:"audit_results"`
+	AuditLevel        string         `json:"audit_level"`
+	AuditStatus       string         `json:"audit_status"`
+	ExecResult        string         `json:"exec_result"`
+	ExecStatus        string         `json:"exec_status"`
+	RollbackSQL       sql.NullString `json:"rollback_sql"`
+	SQLType           sql.NullString `json:"sql_type"`
 }
 
 func (t *TaskSQLDetail) GetAuditResults(ctx context.Context) string {
@@ -612,7 +614,8 @@ func (t *TaskSQLDetail) GetAuditResults(ctx context.Context) string {
 	return t.AuditResults.String(ctx)
 }
 
-var taskSQLsQueryTpl = `SELECT e_sql.number, e_sql.description, e_sql.content AS exec_sql,  e_sql.source_file AS sql_source_file, e_sql.start_line AS sql_start_line, e_sql.sql_type, r_sql.content AS rollback_sql,
+
+var taskSQLsQueryTpl = `SELECT e_sql.id,e_sql.number, e_sql.description, e_sql.content AS exec_sql,  e_sql.source_file AS sql_source_file, e_sql.start_line AS sql_start_line, e_sql.sql_type,
 e_sql.audit_results, e_sql.audit_level, e_sql.audit_status, e_sql.exec_result, e_sql.exec_status
 
 {{- template "body" . -}}
@@ -634,7 +637,6 @@ FROM execute_sql_detail AS e_sql
 LEFT JOIN audit_files ON audit_files.task_id = e_sql.task_id
 AND audit_files.file_name = e_sql.source_file
 {{- end }}
-LEFT JOIN rollback_sql_detail AS r_sql ON e_sql.id = r_sql.execute_sql_id
 WHERE
 e_sql.deleted_at IS NULL
 AND e_sql.task_id = :task_id