From 2e58e9f43fa2e7448d632340ef7d9524e667cb32 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 4 Mar 2024 23:48:51 -0500 Subject: [PATCH] Trying a map. --- cmd/bucket.go | 2 +- internal/pipes/s3.go | 11 ++++++----- internal/structs/vcap.go | 23 ++++++++++++----------- internal/vcap/example.json | 8 ++++---- internal/vcap/vcap.go | 25 ++++++------------------- internal/vcap/vcap_test.go | 21 +++++++++++++++++++++ 6 files changed, 50 insertions(+), 40 deletions(-) diff --git a/cmd/bucket.go b/cmd/bucket.go index 9ff7169..4a67165 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -51,7 +51,7 @@ func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserPr } } -func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up *structs.CredentialsS3) { +func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up map[string]string) { table_to_schema := util.Get_table_and_schema_names(source_creds) for table, schema := range table_to_schema { s3_pipe := pipes.S3( diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go index 958a776..6cf5e90 100644 --- a/internal/pipes/s3.go +++ b/internal/pipes/s3.go @@ -2,11 +2,11 @@ package pipes import ( "fmt" + "os" "strings" "github.com/bitfield/script" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" "gov.gsa.fac.cgov-util/internal/util" ) @@ -14,20 +14,21 @@ import ( // are coming through from VCAP empty. But, the endpoint is not. // This makes no sense. func S3(in_pipe *script.Pipe, - up *structs.CredentialsS3, + up map[string]string, prefix string, source_db string, schema string, table string) *script.Pipe { + os.Setenv("AWS_SECRET_ACCESS_KEY", up["secret_access_key"]) + os.Setenv("AWS_ACCESS_KEY_ID", up["access_key_id"]) + os.Setenv("AWS_DEFAULT_REGION", up["region"]) // https://serverfault.com/questions/886562/streaming-postgresql-pg-dump-to-s3 cmd := []string{ "aws", "s3", "cp", - "--endpoint-url", - up.Uri, "-", fmt.Sprintf("s3://%s/backups/%s-%s_%s.dump", - up.Bucket, + up["bucket"], prefix, schema, table), } diff --git a/internal/structs/vcap.go b/internal/structs/vcap.go index f874f23..346a2ec 100644 --- a/internal/structs/vcap.go +++ b/internal/structs/vcap.go @@ -22,20 +22,21 @@ type CredentialsS3 struct { Endpoint string `json:"endpoint"` FipsEndpoint string `json:"fips_endpoint"` AdditionalBuckets []string `json:"additional_buckets"` - SyslogDrainUrl string `json:"syslog_drain_url"` - VolumeMounts []string `json:"volume_mounts` } type InstanceS3 struct { - Label string `json:"label"` - Plan string `json:"plan"` - Name string `json:"name"` - Tags []string `json:"tags"` - InstanceGuid string `json:"instance_guid"` - InstanceName string `json:"instance_name"` - BindingGuid string `json:"binding_guid"` - BindingName string `json:"binding_name"` - Credentials CredentialsS3 `json:"credentials"` + Label string `json:"label"` + Provider string `json:"provider"` + Plan string `json:"plan"` + Name string `json:"name"` + Tags []string `json:"tags"` + InstanceGuid string `json:"instance_guid"` + InstanceName string `json:"instance_name"` + BindingGuid string `json:"binding_guid"` + BindingName string `json:"binding_name"` + Credentials map[string]string `json:"credentials"` + SyslogDrainUrl string `json:"syslog_drain_url"` + VolumeMounts []string `json:"volume_mounts"` } type InstanceRDS struct { diff --git a/internal/vcap/example.json b/internal/vcap/example.json index f382075..5013682 100644 --- a/internal/vcap/example.json +++ b/internal/vcap/example.json @@ -2,7 +2,7 @@ "s3": [ { "label": "s3", - "provider": null, + "provider": "alpha-provider", "plan": "basic", "name": "backups", "tags": [ @@ -18,15 +18,15 @@ "uri": "s3://ACCESSKEYIDALPHA:SECRETACCESSKEYALPHA@s3-us-gov-alpha-1.amazonaws.com/BROKEREDBUCKETALPHA", "insecure_skip_verify": false, "access_key_id": "ACCESSKEYIDALPHA", - "secret_access_key": "SECRETACCESSKEYALPHA", + "secret_access_key": "SECRETACCESSKEY+ALPHA", "region": "us-gov-west-1", "bucket": "BROKEREDBUCKETALPHA", "endpoint": "s3-us-gov-alpha-1.amazonaws.com", "fips_endpoint": "s3-fips.us-gov-alpha-1.amazonaws.com", "additional_buckets": [] }, - "syslog_drain_url": null, - "volume_mounts": [] + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": ["no_mounts"] }, { "label": "s3", diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go index 2452d2f..f88b270 100644 --- a/internal/vcap/vcap.go +++ b/internal/vcap/vcap.go @@ -59,7 +59,7 @@ func GetUserProvidedCredentials(label string) (structs.UserProvidedCredentials, return nil, errors.Errorf("No credentials found for '%s'", label) } -func GetS3Credentials(name string) (*structs.CredentialsS3, error) { +func GetS3Credentials(name string) (map[string]string, error) { var instanceSlice []structs.InstanceS3 err := viper.UnmarshalKey("s3", &instanceSlice) if err != nil { @@ -67,25 +67,12 @@ func GetS3Credentials(name string) (*structs.CredentialsS3, error) { } for _, instance := range instanceSlice { if instance.Name == name { - all_looks_good := false - - // We have to have an endpoint, or the two key bits. - if (len(instance.Credentials.AccessKeyId) > 0) && - (len(instance.Credentials.SecretAccessKey) > 0) { - all_looks_good = true - } else if len(instance.Credentials.Uri) > 0 { - all_looks_good = true - } + fmt.Println("INST", instance) + fmt.Println("AKI", instance.Credentials["access_key_id"]) + fmt.Println("SAK", instance.Credentials["secret_access_key"]) + fmt.Println("REG", instance.Credentials["region"]) - if len(instance.Credentials.Region) < 1 { - logging.Logger.Println("BACKUPS region is empty") - os.Exit(-1) - } - if all_looks_good { - return &instance.Credentials, nil - } else { - return nil, fmt.Errorf("BACKUPS no access key or endpoint") - } + return instance.Credentials, nil } } diff --git a/internal/vcap/vcap_test.go b/internal/vcap/vcap_test.go index 9b0e8df..391cf12 100644 --- a/internal/vcap/vcap_test.go +++ b/internal/vcap/vcap_test.go @@ -1,6 +1,7 @@ package vcap import ( + "io/ioutil" "os" "testing" ) @@ -117,3 +118,23 @@ func TestReadUserProvided(t *testing.T) { t.Error("Could not find a username") } } + +func TestReadS3(t *testing.T) { + buffer, err := ioutil.ReadFile("example.json") + if err != nil { + t.Error("Could not read example.json") + } + os.Setenv("VCAP_SERVICES", string(buffer)) + ReadVCAPConfig() + + creds, err := GetS3Credentials("backups") + if err != nil { + t.Error("Could not read backups credentials from s3.") + } + if creds["access_key_id"] != "ACCESSKEYIDALPHA" { + t.Error("Did not get s3 access key ACCESSKEYIDALPHA") + } + if creds["secret_access_key"] != "SECRETACCESSKEY+ALPHA" { + t.Error("Did not get s3 secret key SECRETACCESSKEY+ALPHA") + } +}