From 634c2dd73ca27701ba2068c24e10fb5297adeee3 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Sat, 2 Mar 2024 08:34:39 -0500 Subject: [PATCH] Adds S3, finishes MC --- cmd/bucket.go | 47 +++++++++++++++++++++++++++++-------------- go.mod | 1 + go.sum | 2 ++ internal/pipes/mc.go | 8 ++++++-- internal/pipes/s3.go | 40 ++++++++++++++++++++++++++++++++++++ internal/vcap/vcap.go | 32 +++++++++++++++++++++-------- 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/cmd/bucket.go b/cmd/bucket.go index 2c6a44f..81cd994 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -14,6 +14,34 @@ import ( vcap "gov.gsa.fac.backups/internal/vcap" ) +func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) { + mc_pipe := pipes.Mc( + pipes.PG_Dump(source_creds), + up, + "LOCAL", + "local_db", + ) + mc_pipe.Wait() + if err := mc_pipe.Error(); err != nil { + logging.Logger.Println("BACKUPS `dump | mc` pipe failed") + os.Exit(-1) + } +} + +func bucket_cgov(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) { + s3_pipe := pipes.S3( + pipes.PG_Dump(source_creds), + up, + "LOCAL", + "local_db", + ) + s3_pipe.Wait() + if err := s3_pipe.Error(); err != nil { + logging.Logger.Println("BACKUPS `dump | s3` pipe failed") + os.Exit(-1) + } +} + // bucketCmd represents the bucket command var bucketCmd = &cobra.Command{ Use: "bucket", @@ -26,25 +54,14 @@ This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { source_creds, _ := vcap.GetRDSCreds(SourceDB, "") - var up vcap.UserProvidedCredentials - if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) { - up, _ = vcap.GetUserProvidedCredentials("mc") + up, _ := vcap.GetUserProvidedCredentials("mc") + bucket_local(source_creds, up) } else { - up = nil + up, _ := vcap.GetS3Credentials(DestinationBucket) + bucket_cgov(source_creds, up) } - mc_pipe := pipes.Mc( - pipes.PG_Dump(source_creds), - up, - "LOCAL", - "local_db", - ) - mc_pipe.Wait() - if err := mc_pipe.Error(); err != nil { - logging.Logger.Println("BACKUPS `dump | mc` pipe failed") - os.Exit(-1) - } }, } diff --git a/go.mod b/go.mod index 065aff9..8a7b047 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require github.com/bitfield/script v0.22.0 require ( github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/gojq v0.12.12 // indirect diff --git a/go.sum b/go.sum index 13204bc..f42d2f7 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/internal/pipes/mc.go b/internal/pipes/mc.go index c68d131..34bc7d0 100644 --- a/internal/pipes/mc.go +++ b/internal/pipes/mc.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/bitfield/script" + "github.com/google/uuid" "gov.gsa.fac.backups/internal/logging" "gov.gsa.fac.backups/internal/vcap" ) @@ -24,8 +25,11 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s // Always set the alias first. os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"]) os.Setenv("AWS_PRIVATE_SECRET_ACCESS_KEY", upc["secret_access_key"]) + + minio_alias := fmt.Sprintf("minio_alias_%s", uuid.New()) + set_alias := []string{ - "mc", "alias", "set", "minio_alias", + "mc", "alias", "set", minio_alias, upc["endpoint"], upc["admin_username"], upc["admin_password"], @@ -38,7 +42,7 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s "mc", "pipe", fmt.Sprintf("%s/%s/backups/%s-%s.dump", - "minio_alias", + minio_alias, upc["bucket"], prefix, source_db), diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go index 8400772..f21f0aa 100644 --- a/internal/pipes/s3.go +++ b/internal/pipes/s3.go @@ -1 +1,41 @@ package pipes + +import ( + "fmt" + "os" + "strings" + + "github.com/bitfield/script" + "gov.gsa.fac.backups/internal/logging" + "gov.gsa.fac.backups/internal/vcap" +) + +// export AWS_PRIVATE_ACCESS_KEY_ID=longtest +// export AWS_PRIVATE_SECRET_ACCESS_KEY=longtest +// export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000" +// mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin +// # Do nothing if the bucket already exists. +// # https: //min.io/docs/minio/linux/reference/minio-mc/mc-mb.html +// mc mb --ignore-existing myminio/gsa-fac-private-s3 + +// https://bitfieldconsulting.com/golang/scripting +func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db string) *script.Pipe { + os.Setenv("ACCESS_KEY_ID", up.AccessKeyId) + os.Setenv("SECRET_ACCESS_KEY", up.SecretAccessKey) + + cmd := []string{ + "s3", + "cp", + fmt.Sprintf("s3://%s/backups/%s-%s.dump", + up.Bucket, + prefix, + source_db), + } + + // Combine the slice for printing and execution. + combined := strings.Join(cmd[:], " ") + // This will log the password... + logging.Logger.Printf("BACKUPS Running `%s`\n", combined) + logging.Logger.Printf("BACKUPS s3 targeting %s", prefix) + return in_pipe.Exec(combined) +} diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go index 663115d..6c521f5 100644 --- a/internal/vcap/vcap.go +++ b/internal/vcap/vcap.go @@ -34,14 +34,15 @@ type CredentialsS3 struct { } type InstanceS3 struct { - Label string `json:"label"` - Plan string `json:"plan"` - Name string `json:"name"` - Tags []string `json:"tags"` - InstanceGuid string `json:"instance_guid"` - InstanceName string `json:"instance_name"` - BindingGuid string `json:"binding_guid"` - BindingName string `json:"binding_name"` + Label string `json:"label"` + Plan string `json:"plan"` + Name string `json:"name"` + Tags []string `json:"tags"` + InstanceGuid string `json:"instance_guid"` + InstanceName string `json:"instance_name"` + BindingGuid string `json:"binding_guid"` + BindingName string `json:"binding_name"` + Credentials CredentialsS3 `json:"credentials"` } type InstanceRDS struct { @@ -101,6 +102,7 @@ func GetLocalRDSCredentials(label string) (*CredentialsRDS, error) { type UserProvidedCredentials = map[string]string +// Returns a map, not a pointer to a structure func GetUserProvidedCredentials(label string) (UserProvidedCredentials, error) { var instanceSlice []UserProvided err := viper.UnmarshalKey("user-provided", &instanceSlice) @@ -115,6 +117,20 @@ func GetUserProvidedCredentials(label string) (UserProvidedCredentials, error) { return nil, errors.Errorf("No credentials found for '%s'", label) } +func GetS3Credentials(label string) (*CredentialsS3, error) { + var instanceSlice []InstanceS3 + err := viper.UnmarshalKey("s3", &instanceSlice) + if err != nil { + logging.Logger.Println("Could not unmarshal aws-rds from VCAP_SERVICES") + } + for _, instance := range instanceSlice { + if instance.Name == label { + return &instance.Credentials, nil + } + } + return nil, errors.Errorf("No credentials found for '%s'", label) +} + func GetRDSCreds(source_db string, dest_db string) (*CredentialsRDS, *CredentialsRDS) { var source *CredentialsRDS var dest *CredentialsRDS