Skip to content

Commit

Permalink
Added rotation
Browse files Browse the repository at this point in the history
  • Loading branch information
@LobbyLobster
LobbyLobster committed Jan 3, 2024
1 parent dd90986 commit ceda2f0
Show file tree
Hide file tree
Showing 7 changed files with 146 additions and 24 deletions.
52 changes: 43 additions & 9 deletions backup/dumper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"monodb-backup/config"
"monodb-backup/notify"
"os"
"time"
)

var hostname, _ = os.Hostname()
Expand All @@ -14,9 +15,42 @@ type Dumper struct {
}

type rightNow struct {
year string
month string
now string
year string
month string
day string
hour string
minute string
now string
}

func dumpName(db string, params config.Rotation) string {
if !params.Enabled {
date := rightNow{
year: time.Now().Format("2006"),
month: time.Now().Format("01"),
now: time.Now().Format("2006-01-02-150405"),
}
name := date.year + "/" + date.month + "/" + db + "-" + date.now
return name
} else {
suffix := params.Suffix
date := rightNow{
day: time.Now().Format("Monday"),
hour: time.Now().Format("January-Monday-15"),
minute: time.Now().Format("January-Monday-15_04"),
now: time.Now().Format("2006-01-02-150405"),
}
switch suffix {
case "day":
return db + "-" + date.day
case "hour":
return db + "-" + date.hour
case "minute":
return db + "-" + date.minute
default:
return db + "-" + date.day
}
}
}

// NewDumper creates a new Dumper instance.
Expand Down Expand Up @@ -80,7 +114,7 @@ func (d *Dumper) Dump() {
}

if d.p.S3.Enabled {
err = d.uploadS3(filePath, name)
err = d.uploadS3(filePath, name, db)
if err != nil {
d.l.Error("Couldn't upload " + filePath + " to S3" + " - Error: " + err.Error())
notify.SendAlarm("Couldn't upload "+filePath+" to S3"+" - Error: "+err.Error(), true)
Expand All @@ -103,7 +137,7 @@ func (d *Dumper) Dump() {
}

if d.p.Minio.Enabled {
err = d.uploadMinIO(filePath, name)
err = d.uploadMinIO(filePath, name, db)
if err != nil {
d.l.Error("Couldn't upload " + filePath + " to MinIO" + " - Error: " + err.Error())
notify.SendAlarm("Couldn't upload "+filePath+" to MinIO"+" - Error: "+err.Error(), true)
Expand Down Expand Up @@ -148,7 +182,7 @@ func (d *Dumper) dumpDB(db string, dst string) (dumpPath string, name string, er
return
}

func (d *Dumper) uploadS3(filePath, name string) error {
func (d *Dumper) uploadS3(filePath, name, db string) error {
uploader, err := newS3Uploader(d.p.S3.Region, d.p.S3.AccessKey, d.p.S3.SecretKey)
if err != nil {
return err
Expand All @@ -157,7 +191,7 @@ func (d *Dumper) uploadS3(filePath, name string) error {
if d.p.S3.Path != "" {
target = d.p.S3.Path + "/" + target
}
err = uploadFileToS3(uploader, filePath, d.p.S3.Bucket, target)
err = uploadFileToS3(uploader, filePath, d.p.S3.Bucket, target, d.p.Rotation, db, d.p.S3.Path)
if err != nil {
return err
}
Expand All @@ -166,7 +200,7 @@ func (d *Dumper) uploadS3(filePath, name string) error {
return nil
}

func (d *Dumper) uploadMinIO(filePath, name string) error {
func (d *Dumper) uploadMinIO(filePath, name, db string) error {
minioClient, err := newMinioClient(
d.p.Minio.Endpoint,
d.p.Minio.AccessKey,
Expand All @@ -180,7 +214,7 @@ func (d *Dumper) uploadMinIO(filePath, name string) error {
if d.p.Minio.Path != "" {
target = d.p.Minio.Path + "/" + target
}
err = uploadFileToMinio(minioClient, filePath, d.p.Minio.Bucket, target)
err = uploadFileToMinio(minioClient, d.p.Rotation, db, filePath, d.p.Minio.Bucket, target, d.p.Minio.Path)
if err != nil {
return err
}
Expand Down
46 changes: 45 additions & 1 deletion backup/minio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ package backup
import (
"context"
"crypto/tls"
"errors"
"monodb-backup/config"
"net/http"
"os"
"strconv"
"strings"
"time"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
Expand All @@ -20,6 +24,27 @@ type MinioClient struct {
InsecureSkipVerify bool
}

func rotate(db, rotation string) (bool, string) {
t := time.Now()
_, week := t.ISOWeek()
date := rightNow{
month: time.Now().Format("January"),
day: time.Now().Format("Monday"),
}
switch rotation {
case "month":
yesterday := t.AddDate(0, 0, -1)
if yesterday.Month() != t.Month() {
return true, db + "-" + date.month
}
case "week":
if date.day == "Monday" {
return true, db + "-week_" + strconv.Itoa(week)
}
}
return false, ""
}

func newMinioClient(endpoint, accessKey, secretKey string, secure, insecureSkipVerify bool) (*MinioClient, error) {
minioOptions := &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Expand Down Expand Up @@ -47,7 +72,7 @@ func newMinioClient(endpoint, accessKey, secretKey string, secure, insecureSkipV
return &c, nil
}

func uploadFileToMinio(minioClient *MinioClient, src string, bucketName string, dst string) error {
func uploadFileToMinio(minioClient *MinioClient, rotation config.Rotation, db string, src string, bucketName string, dst string, minio_dst string) error {
src = strings.TrimSuffix(src, "/")
_, err := os.Open(src)
if err != nil {
Expand All @@ -57,5 +82,24 @@ func uploadFileToMinio(minioClient *MinioClient, src string, bucketName string,
if err != nil {
return err
}
if rotation.Enabled {
shouldRotate, name := rotate(db, rotation.Period)
if shouldRotate {
source := minio.CopySrcOptions{
Bucket: bucketName,
Object: dst,
}
extension := strings.Split(dst, ".")
name = name + "." + extension[len(extension)-1]
dest := minio.CopyDestOptions{
Bucket: bucketName,
Object: minio_dst + "/" + name,
}
_, err := minioClient.CopyObject(context.Background(), dest, source)
if err != nil {
return errors.New("Couldn't create copy for rotation. Error: " + err.Error())
}
}
}
return nil
}
19 changes: 13 additions & 6 deletions backup/mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func getMySQLList(params config.Remote, logger Logger) ([]string, error) {
for i, line := range bytes.Split(out, []byte{'\n'}) {
if len(line) > 0 && i > 0 {
ln := string(line)
if ln == "" || ln == "information_schema" || ln == "performance_schema" || ln == "sys" { // TODO mysql veritabanindan user tablosunu al
if ln == "" || ln == "information_schema" || ln == "performance_schema" || ln == "sys" {
continue
}
dbList = append(dbList, ln)
Expand All @@ -38,6 +38,7 @@ func getMySQLList(params config.Remote, logger Logger) ([]string, error) {
func dumpMySQLDb(db string, dst string, params config.Params, logger Logger) (string, string, error) {
encrypted := params.ArchivePass != ""
var format string
var name string
var cmd *exec.Cmd
var cmd2 *exec.Cmd
if encrypted {
Expand All @@ -54,14 +55,18 @@ func dumpMySQLDb(db string, dst string, params config.Params, logger Logger) (st
} else {
mysqlArgs = append(mysqlArgs, "-u"+params.Remote.User, "-p"+params.Remote.Password, db)
}

date := rightNow{
year: time.Now().Format("2006"),
month: time.Now().Format("01"),
now: time.Now().Format("2006-01-02-150405"),
}
name := date.year + "/" + date.month + "/" + db + "-" + date.now + ".sql.7z"
dumpPath := dst + "/" + name

if db == "mysql" {
mysqlArgs = append(mysqlArgs, "user")
name = dumpName(db+"_users", params.Rotation)
} else {
name = dumpName(db, params.Rotation)
}
var dumpPath string
_ = os.MkdirAll(dst+"/"+date.year+"/"+date.month, os.ModePerm)

cmd = exec.Command("/usr/bin/mysqldump", mysqlArgs...)
Expand All @@ -77,7 +82,7 @@ func dumpMySQLDb(db string, dst string, params config.Params, logger Logger) (st
}

if !encrypted && format == "gzip" {
name = date.year + "/" + date.month + "/" + db + "-" + date.now + ".sql.gz"
name = name + ".sql.gz"
dumpPath = dst + "/" + name
cmd2 = exec.Command("gzip")
cmd2.Stdin = stdout
Expand Down Expand Up @@ -109,6 +114,8 @@ func dumpMySQLDb(db string, dst string, params config.Params, logger Logger) (st
return "", "", err
}
} else {
name = name + ".sql.7z"
dumpPath = dst + "/" + name
if encrypted {
cmd2 = exec.Command("7z", "a", "-t7z", "-ms=on", "-mhe=on", "-p"+params.ArchivePass, "-si", dumpPath)
} else {
Expand Down
13 changes: 7 additions & 6 deletions backup/postgresql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ func getPSQLList(params config.Remote, logger Logger) ([]string, error) {
func dumpPSQLDb(db string, dst string, params config.Params, logger Logger) (string, string, error) {
encrypted := params.ArchivePass != ""
var dumpPath string
var name string
var format string
var cmd *exec.Cmd

name := dumpName(db, params.Rotation)

if params.Format != "" {
format = params.Format
} else {
Expand All @@ -61,13 +63,12 @@ func dumpPSQLDb(db string, dst string, params config.Params, logger Logger) (str
date := rightNow{
year: time.Now().Format("2006"),
month: time.Now().Format("01"),
now: time.Now().Format("2006-01-02-150405"),
}
_ = os.MkdirAll(dst+"/"+date.year+"/"+date.month, os.ModePerm)

if !encrypted {
if format == "gzip" {
name = date.year + "/" + date.month + "/" + db + "-" + date.now + ".dump"
name = name + ".dump"
dumpPath = dst + "/" + name
pgDumpArgs = append(pgDumpArgs, "-Fc", "-f", dumpPath)
cmd = exec.Command("/usr/bin/pg_dump", pgDumpArgs...)
Expand All @@ -77,7 +78,7 @@ func dumpPSQLDb(db string, dst string, params config.Params, logger Logger) (str
return "", "", err
}
} else if format == "7zip" {
name = date.year + "/" + date.month + "/" + db + "-" + date.now + ".sql.7z"
name = name + ".sql.7z"
dumpPath = dst + "/" + name
cmd = exec.Command("/usr/bin/pg_dump", pgDumpArgs...)
stdout, err := cmd.StdoutPipe()
Expand All @@ -97,7 +98,7 @@ func dumpPSQLDb(db string, dst string, params config.Params, logger Logger) (str
}
} else {
if format == "gzip" {
name = date.year + "/" + date.month + "/" + db + "-" + date.now + ".dump.7z"
name = name + ".dump.7z"
dumpPath = dst + "/" + name
pgDumpArgs = append(pgDumpArgs, "-Fc")
cmd = exec.Command("/usr/bin/pg_dump", pgDumpArgs...)
Expand All @@ -116,7 +117,7 @@ func dumpPSQLDb(db string, dst string, params config.Params, logger Logger) (str

err = cmd2.Run()
} else if format == "7zip" {
name = date.year + "/" + date.month + "/" + db + "-" + date.now + ".sql.7z"
name = name + ".sql.7z"
dumpPath = dst + "/" + name
cmd = exec.Command("/usr/bin/pg_dump", pgDumpArgs...)
stdout, err := cmd.StdoutPipe()
Expand Down
29 changes: 27 additions & 2 deletions backup/s3.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package backup

import (
"errors"
"github.com/aws/aws-sdk-go/service/s3"
"monodb-backup/config"
"os"
"strings"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
Expand All @@ -24,7 +28,7 @@ func newS3Uploader(region, accessKey, secretKey string) (*s3manager.Uploader, er
return uploader, nil
}

func uploadFileToS3(uploader *s3manager.Uploader, src string, bucketName string, dst string) error {
func uploadFileToS3(uploader *s3manager.Uploader, src string, bucketName string, dst string, rotation config.Rotation, db, path string) error {
file, err := os.Open(src)
if err != nil {
return err
Expand All @@ -35,5 +39,26 @@ func uploadFileToS3(uploader *s3manager.Uploader, src string, bucketName string,
Key: aws.String(dst),
Body: file,
})
return err
if err != nil {
return err
}
if rotation.Enabled {
shouldRotate, name := rotate(db, rotation.Period)
if path != "" {
name = path + "/" + name
}
extension := strings.Split(dst, ".")
name = name + "." + extension[len(extension)-1]
if shouldRotate {
_, err := uploader.S3.CopyObject(&s3.CopyObjectInput{
Bucket: aws.String(bucketName),
CopySource: aws.String(bucketName + "/" + dst),
Key: aws.String(name),
})
if err != nil {
return errors.New("Couldn't create copy for rotation. Error: " + err.Error())
}
}
}
return nil
}
4 changes: 4 additions & 0 deletions config/config.sample.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ remote:
port: 5432
user: postgres # necessary for mysql, even if isRemote false
password: password # necessary for mysql, even if isRemote false
rotation:
enabled: true
period: week # week or month - db-week_1.sql.7z .. db-week_52.sql.7z - db-january.sql.7z .. db-december.sql.7z
suffix: minute # day db-week-day.sql.7z - hour db-week-day-15.sql.7z - minute db-week-day-15-24.sql.7z
s3:
enabled: false
region: aws region
Expand Down
7 changes: 7 additions & 0 deletions config/params.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type Params struct {
RemoveLocal bool
ArchivePass string
Remote Remote
Rotation Rotation
Notify struct {
Email struct {
Enabled bool
Expand Down Expand Up @@ -47,6 +48,12 @@ type Params struct {
Fqdn string
}

type Rotation struct {
Enabled bool
Period string // week or month
Suffix string // day db-monday.sql.7z - hour db-monday-15.sql.7z - minute db-monday-15-24.sql.7z
}

type Remote struct {
IsRemote bool
Host string
Expand Down

0 comments on commit ceda2f0

Please sign in to comment.