From 2f2b5d5bf9d5f2e746d6e1304e04c656e0e76bfe Mon Sep 17 00:00:00 2001 From: Bekir Pehlivan Date: Thu, 27 Jun 2024 11:24:16 +0300 Subject: [PATCH] it now sends an extra alarm after a successful backup if the one before was a faillure --- backup/backup.go | 38 +++++++++++++++++++++++++++++++++----- backup/rsync.go | 18 ++++++++++++++---- backup/s3.go | 4 +++- backup/sftp.go | 35 ++++++++++++++++++++++------------- 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 592b2f0..1d65088 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -9,6 +9,19 @@ import ( "time" ) +var didItWork bool = true // this variable is for determining if the app should send a notification after a failed backup to inform that it works now +func itWorksNow(message string, worked bool) { + oldOnlyOnError := params.Notify.Webhook.OnlyOnError + if !didItWork && worked && oldOnlyOnError { + if oldOnlyOnError { + params.Notify.Webhook.OnlyOnError = false + } + notify.SendAlarm(message, false) + params.Notify.Webhook.OnlyOnError = oldOnlyOnError + } + didItWork = worked +} + func getDBList() (dbList []string) { switch params.Database { case "postgresql": @@ -90,13 +103,13 @@ func Backup() { } return } - for _, db := range params.Databases { dst := strings.TrimSuffix(params.BackupDestination, "/") + "/" + db if params.BackupAsTables && db != "mysql" { dumpPaths, names, err := dumpDBWithTables(db, dst) if err != nil { notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true) + itWorksNow("", false) } else { logger.Info("Successfully backed up database:" + db + " with its tables separately, at " + params.BackupDestination + "/" + db) notify.SendAlarm("Successfully backed up "+db+" at "+params.BackupDestination+"/"+db, false) @@ -110,6 +123,7 @@ func Backup() { filePath, name, err := dumpDB(db, dst) if err != nil { notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true) + itWorksNow("", false) } else { logger.Info("Successfully backed up database:" + db + " at " + filePath) notify.SendAlarm("Successfully backed up "+db+" at "+filePath, false) @@ -160,6 +174,7 @@ func uploadWhileDumping(db string) { err := dumpAndUpload(db, pipeWriters) if err != nil { notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true) + itWorksNow("", false) } for _, writer := range pipeWriters { writer.Close() @@ -169,24 +184,37 @@ func uploadWhileDumping(db string) { if uploadErr != nil { logger.Error(strconv.Itoa(i+1) + ") " + db + " - " + "Couldn't upload to S3: " + uploaders[i].instance.Endpoint + " - Error: " + err.Error()) notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Couldn't upload to S3: "+uploaders[i].instance.Endpoint+" - Error: "+err.Error(), true) + itWorksNow("", false) } else { logger.Info(strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint) - notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Successfully uploaded to S3: "+uploaders[i].instance.Endpoint, false) + message := strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint + notify.SendAlarm(message, false) + itWorksNow(message, true) } } } func upload(name, db, filePath string) { + var err error switch params.BackupType.Type { case "s3", "minio": - uploadToS3(filePath, name, db) + err = uploadToS3(filePath, name, db) + if err != nil { + itWorksNow("", false) + } case "sftp": for _, target := range params.BackupType.Info[0].Targets { - SendSFTP(filePath, name, db, target) + err = SendSFTP(filePath, name, db, target) + if err != nil { + itWorksNow("", false) + } } case "rsync": for _, target := range params.BackupType.Info[0].Targets { - SendRsync(filePath, name, target) + err = SendRsync(filePath, name, target) + if err != nil { + itWorksNow("", false) + } } } } diff --git a/backup/rsync.go b/backup/rsync.go index f6114d9..ff37611 100644 --- a/backup/rsync.go +++ b/backup/rsync.go @@ -8,7 +8,7 @@ import ( "strings" ) -func SendRsync(srcPath, dstPath string, target config.Target) { +func SendRsync(srcPath, dstPath string, target config.Target) error { var dst string if target.Path != "" { @@ -18,7 +18,11 @@ func SendRsync(srcPath, dstPath string, target config.Target) { } err := sendRsync(srcPath, dst, target) - if err == nil && params.Rotation.Enabled { + if err != nil { + return err + } + + if params.Rotation.Enabled { extension := strings.Split(dstPath, ".") shouldRotate, dstPath := rotate(extension[0]) for i := 1; i < len(extension); i++ { @@ -28,9 +32,13 @@ func SendRsync(srcPath, dstPath string, target config.Target) { dstPath = target.Path + "/" + dstPath } if shouldRotate { - _ = sendRsync(srcPath, dstPath, target) + err = sendRsync(srcPath, dstPath, target) + if err != nil { + return err + } } } + return nil } func sendRsync(srcPath, dstPath string, target config.Target) error { @@ -65,7 +73,9 @@ func sendRsync(srcPath, dstPath string, target config.Target) error { } logger.Info("Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath) - notify.SendAlarm("Successfully uploaded "+srcPath+" to "+target.Host+":"+dstPath, false) + message := "Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath + notify.SendAlarm(message, false) + itWorksNow(message, true) return nil } diff --git a/backup/s3.go b/backup/s3.go index 04640a8..3283139 100644 --- a/backup/s3.go +++ b/backup/s3.go @@ -131,7 +131,9 @@ func uploadFileToS3(src, dst, db string, reader io.Reader, s3Instance *uploaderS return err } logger.Info("Successfully uploaded " + src + " to S3\nBucket: " + bucketName + " path: " + dst) - notify.SendAlarm("Successfully uploaded "+src+" to S3\nBucket: "+bucketName+" path: "+dst, false) + message := "Successfully uploaded " + src + " to S3\nBucket: " + bucketName + " path: " + dst + notify.SendAlarm(message, false) + itWorksNow(message, true) if params.Rotation.Enabled { if db == "mysql" { db = db + "_users" diff --git a/backup/sftp.go b/backup/sftp.go index 6296374..1e6c49e 100644 --- a/backup/sftp.go +++ b/backup/sftp.go @@ -12,14 +12,14 @@ import ( "golang.org/x/crypto/ssh/agent" ) -func SendSFTP(srcPath, dstPath, db string, target config.Target) { +func SendSFTP(srcPath, dstPath, db string, target config.Target) error { dstPath = target.Path + "/" + nameWithPath(dstPath) logger.Info("SFTP transfer started.\n Source: " + srcPath + " - Destination: " + target.Host + ":" + dstPath) sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) if err != nil { logger.Error("Couldn't get environment variable SSH_AUTH_SOCK - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't get environment variable SSH_AUTH_SOCK - Error: "+err.Error(), true) - return + return err } sockAgent := agent.NewClient(sock) @@ -28,7 +28,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) { if err != nil { logger.Error("Couldn't get signers for ssh keys - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't get signers for ssh keys - Error: "+err.Error(), true) - return + return err } auths := []ssh.AuthMethod{ssh.PublicKeys(signers...)} @@ -51,7 +51,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) { if err != nil { logger.Error("Couldn't create an SFTP client - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create an SFTP client - Error: "+err.Error(), true) - return + return err } defer func() { err = sftpCli.Close() @@ -65,7 +65,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) { if err != nil { logger.Error("Couldn't open source file " + srcPath + " for copying - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't open source file "+srcPath+" for copying - Error: "+err.Error(), true) - return + return err } defer func() { err = src.Close() @@ -75,7 +75,10 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) { } }() - sendOverSFTP(srcPath, dstPath, src, target, sftpCli) + err = sendOverSFTP(srcPath, dstPath, src, target, sftpCli) + if err != nil { + return err + } if params.Rotation.Enabled { shouldRotate, newDst := rotate(db) @@ -85,13 +88,16 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) { newDst = newDst + "." + extension[i] } newDst = target.Path + "/" + newDst - sendOverSFTP(srcPath, newDst, src, target, sftpCli) + err = sendOverSFTP(srcPath, newDst, src, target, sftpCli) + if err != nil { + return err + } } } - + return nil } -func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, sftpCli *sftp.Client) { +func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, sftpCli *sftp.Client) error { fullPath := strings.Split(dstPath, "/") newPath := "/" for i := 0; i < len(fullPath)-1; i++ { @@ -101,13 +107,13 @@ func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, s if err != nil { logger.Error("Couldn't create folders " + newPath + " - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create folders "+newPath+" - Error: "+err.Error(), true) - return + return err } dst, err := sftpCli.Create(dstPath) if err != nil { logger.Error("Couldn't create file " + dstPath + " - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create file "+dstPath+" - Error: "+err.Error(), true) - return + return err } defer func() { err = dst.Close() @@ -121,8 +127,11 @@ func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, s if _, err := dst.ReadFrom(src); err != nil { logger.Error("Couldn't read from file " + srcPath + " to write at " + dstPath + " - Error: " + err.Error()) notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't read from file "+srcPath+" to write at "+dstPath+" - Error: "+err.Error(), true) - return + return err } logger.Info("Successfully copied " + srcPath + " to " + target.Host + ":" + dstPath) - notify.SendAlarm("Successfully copied "+srcPath+" to "+target.Host+":"+dstPath, false) + message := "Successfully copied " + srcPath + " to " + target.Host + ":" + dstPath + notify.SendAlarm(message, false) + itWorksNow(message, true) + return nil }