Adds S3, finishes MC

cmd/bucket.go

@@ -14,6 +14,34 @@ import (
 	vcap "gov.gsa.fac.backups/internal/vcap"
 )
 
+func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) {
+	mc_pipe := pipes.Mc(
+		pipes.PG_Dump(source_creds),
+		up,
+		"LOCAL",
+		"local_db",
+	)
+	mc_pipe.Wait()
+	if err := mc_pipe.Error(); err != nil {
+		logging.Logger.Println("BACKUPS `dump | mc` pipe failed")
+		os.Exit(-1)
+	}
+}
+
+func bucket_cgov(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) {
+	s3_pipe := pipes.S3(
+		pipes.PG_Dump(source_creds),
+		up,
+		"LOCAL",
+		"local_db",
+	)
+	s3_pipe.Wait()
+	if err := s3_pipe.Error(); err != nil {
+		logging.Logger.Println("BACKUPS `dump | s3` pipe failed")
+		os.Exit(-1)
+	}
+}
+
 // bucketCmd represents the bucket command
 var bucketCmd = &cobra.Command{
 	Use:   "bucket",
@@ -26,25 +54,14 @@ This application is a tool to generate the needed files
 to quickly create a Cobra application.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		source_creds, _ := vcap.GetRDSCreds(SourceDB, "")
-		var up vcap.UserProvidedCredentials
-
 		if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) {
-			up, _ = vcap.GetUserProvidedCredentials("mc")
+			up, _ := vcap.GetUserProvidedCredentials("mc")
+			bucket_local(source_creds, up)
 		} else {
-			up = nil
+			up, _ := vcap.GetS3Credentials(DestinationBucket)
+			bucket_cgov(source_creds, up)
 		}
 
-		mc_pipe := pipes.Mc(
-			pipes.PG_Dump(source_creds),
-			up,
-			"LOCAL",
-			"local_db",
-		)
-		mc_pipe.Wait()
-		if err := mc_pipe.Error(); err != nil {
-			logging.Logger.Println("BACKUPS `dump | mc` pipe failed")
-			os.Exit(-1)
-		}
 	},
 }
 

go.mod

@@ -6,6 +6,7 @@ require github.com/bitfield/script v0.22.0
 
 require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/itchyny/gojq v0.12.12 // indirect

go.sum

@@ -13,6 +13,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

internal/pipes/mc.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/bitfield/script"
+	"github.com/google/uuid"
 	"gov.gsa.fac.backups/internal/logging"
 	"gov.gsa.fac.backups/internal/vcap"
 )
@@ -24,8 +25,11 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s
 	// Always set the alias first.
 	os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"])
 	os.Setenv("AWS_PRIVATE_SECRET_ACCESS_KEY", upc["secret_access_key"])
+
+	minio_alias := fmt.Sprintf("minio_alias_%s", uuid.New())
+
 	set_alias := []string{
-		"mc", "alias", "set", "minio_alias",
+		"mc", "alias", "set", minio_alias,
 		upc["endpoint"],
 		upc["admin_username"],
 		upc["admin_password"],
@@ -38,7 +42,7 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s
 		"mc",
 		"pipe",
 		fmt.Sprintf("%s/%s/backups/%s-%s.dump",
-			"minio_alias",
+			minio_alias,
 			upc["bucket"],
 			prefix,
 			source_db),

internal/pipes/s3.go

@@ -1 +1,41 @@
 package pipes
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/bitfield/script"
+	"gov.gsa.fac.backups/internal/logging"
+	"gov.gsa.fac.backups/internal/vcap"
+)
+
+// export AWS_PRIVATE_ACCESS_KEY_ID=longtest
+// export AWS_PRIVATE_SECRET_ACCESS_KEY=longtest
+// export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000"
+// mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin
+// # Do nothing if the bucket already exists.
+// # https: //min.io/docs/minio/linux/reference/minio-mc/mc-mb.html
+// mc mb --ignore-existing myminio/gsa-fac-private-s3
+
+// https://bitfieldconsulting.com/golang/scripting
+func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db string) *script.Pipe {
+	os.Setenv("ACCESS_KEY_ID", up.AccessKeyId)
+	os.Setenv("SECRET_ACCESS_KEY", up.SecretAccessKey)
+
+	cmd := []string{
+		"s3",
+		"cp",
+		fmt.Sprintf("s3://%s/backups/%s-%s.dump",
+			up.Bucket,
+			prefix,
+			source_db),
+	}
+
+	// Combine the slice for printing and execution.
+	combined := strings.Join(cmd[:], " ")
+	// This will log the password...
+	logging.Logger.Printf("BACKUPS Running `%s`\n", combined)
+	logging.Logger.Printf("BACKUPS s3 targeting %s", prefix)
+	return in_pipe.Exec(combined)
+}

internal/vcap/vcap.go

@@ -34,14 +34,15 @@ type CredentialsS3 struct {
 }
 
 type InstanceS3 struct {
-	Label        string   `json:"label"`
-	Plan         string   `json:"plan"`
-	Name         string   `json:"name"`
-	Tags         []string `json:"tags"`
-	InstanceGuid string   `json:"instance_guid"`
-	InstanceName string   `json:"instance_name"`
-	BindingGuid  string   `json:"binding_guid"`
-	BindingName  string   `json:"binding_name"`
+	Label        string        `json:"label"`
+	Plan         string        `json:"plan"`
+	Name         string        `json:"name"`
+	Tags         []string      `json:"tags"`
+	InstanceGuid string        `json:"instance_guid"`
+	InstanceName string        `json:"instance_name"`
+	BindingGuid  string        `json:"binding_guid"`
+	BindingName  string        `json:"binding_name"`
+	Credentials  CredentialsS3 `json:"credentials"`
 }
 
 type InstanceRDS struct {
@@ -101,6 +102,7 @@ func GetLocalRDSCredentials(label string) (*CredentialsRDS, error) {
 
 type UserProvidedCredentials = map[string]string
 
+// Returns a map, not a pointer to a structure
 func GetUserProvidedCredentials(label string) (UserProvidedCredentials, error) {
 	var instanceSlice []UserProvided
 	err := viper.UnmarshalKey("user-provided", &instanceSlice)
@@ -115,6 +117,20 @@ func GetUserProvidedCredentials(label string) (UserProvidedCredentials, error) {
 	return nil, errors.Errorf("No credentials found for '%s'", label)
 }
 
+func GetS3Credentials(label string) (*CredentialsS3, error) {
+	var instanceSlice []InstanceS3
+	err := viper.UnmarshalKey("s3", &instanceSlice)
+	if err != nil {
+		logging.Logger.Println("Could not unmarshal aws-rds from VCAP_SERVICES")
+	}
+	for _, instance := range instanceSlice {
+		if instance.Name == label {
+			return &instance.Credentials, nil
+		}
+	}
+	return nil, errors.Errorf("No credentials found for '%s'", label)
+}
+
 func GetRDSCreds(source_db string, dest_db string) (*CredentialsRDS, *CredentialsRDS) {
 	var source *CredentialsRDS
 	var dest *CredentialsRDS