Skip to content
This repository has been archived by the owner on Oct 29, 2024. It is now read-only.

Commit

Permalink
Trying a map.
Browse files Browse the repository at this point in the history
  • Loading branch information
jadudm committed Mar 5, 2024
1 parent d158dbd commit 2e58e9f
Show file tree
Hide file tree
Showing 6 changed files with 50 additions and 40 deletions.
2 changes: 1 addition & 1 deletion cmd/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserPr
}
}

func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up *structs.CredentialsS3) {
func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up map[string]string) {
table_to_schema := util.Get_table_and_schema_names(source_creds)
for table, schema := range table_to_schema {
s3_pipe := pipes.S3(
Expand Down
11 changes: 6 additions & 5 deletions internal/pipes/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,33 @@ package pipes

import (
"fmt"
"os"
"strings"

"github.com/bitfield/script"
"gov.gsa.fac.cgov-util/internal/logging"
"gov.gsa.fac.cgov-util/internal/structs"
"gov.gsa.fac.cgov-util/internal/util"
)

// For reasons that are unclear, the access key id and secret key
// are coming through from VCAP empty. But, the endpoint is not.
// This makes no sense.
func S3(in_pipe *script.Pipe,
up *structs.CredentialsS3,
up map[string]string,
prefix string,
source_db string,
schema string, table string) *script.Pipe {
os.Setenv("AWS_SECRET_ACCESS_KEY", up["secret_access_key"])
os.Setenv("AWS_ACCESS_KEY_ID", up["access_key_id"])
os.Setenv("AWS_DEFAULT_REGION", up["region"])
// https://serverfault.com/questions/886562/streaming-postgresql-pg-dump-to-s3
cmd := []string{
"aws",
"s3",
"cp",
"--endpoint-url",
up.Uri,
"-",
fmt.Sprintf("s3://%s/backups/%s-%s_%s.dump",
up.Bucket,
up["bucket"],
prefix,
schema, table),
}
Expand Down
23 changes: 12 additions & 11 deletions internal/structs/vcap.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,21 @@ type CredentialsS3 struct {
Endpoint string `json:"endpoint"`
FipsEndpoint string `json:"fips_endpoint"`
AdditionalBuckets []string `json:"additional_buckets"`
SyslogDrainUrl string `json:"syslog_drain_url"`
VolumeMounts []string `json:"volume_mounts`
}

type InstanceS3 struct {
Label string `json:"label"`
Plan string `json:"plan"`
Name string `json:"name"`
Tags []string `json:"tags"`
InstanceGuid string `json:"instance_guid"`
InstanceName string `json:"instance_name"`
BindingGuid string `json:"binding_guid"`
BindingName string `json:"binding_name"`
Credentials CredentialsS3 `json:"credentials"`
Label string `json:"label"`
Provider string `json:"provider"`
Plan string `json:"plan"`
Name string `json:"name"`
Tags []string `json:"tags"`
InstanceGuid string `json:"instance_guid"`
InstanceName string `json:"instance_name"`
BindingGuid string `json:"binding_guid"`
BindingName string `json:"binding_name"`
Credentials map[string]string `json:"credentials"`
SyslogDrainUrl string `json:"syslog_drain_url"`
VolumeMounts []string `json:"volume_mounts"`
}

type InstanceRDS struct {
Expand Down
8 changes: 4 additions & 4 deletions internal/vcap/example.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"s3": [
{
"label": "s3",
"provider": null,
"provider": "alpha-provider",
"plan": "basic",
"name": "backups",
"tags": [
Expand All @@ -18,15 +18,15 @@
"uri": "s3://ACCESSKEYIDALPHA:[email protected]/BROKEREDBUCKETALPHA",
"insecure_skip_verify": false,
"access_key_id": "ACCESSKEYIDALPHA",
"secret_access_key": "SECRETACCESSKEYALPHA",
"secret_access_key": "SECRETACCESSKEY+ALPHA",
"region": "us-gov-west-1",
"bucket": "BROKEREDBUCKETALPHA",
"endpoint": "s3-us-gov-alpha-1.amazonaws.com",
"fips_endpoint": "s3-fips.us-gov-alpha-1.amazonaws.com",
"additional_buckets": []
},
"syslog_drain_url": null,
"volume_mounts": []
"syslog_drain_url": "https://ALPHA.drain.url",
"volume_mounts": ["no_mounts"]
},
{
"label": "s3",
Expand Down
25 changes: 6 additions & 19 deletions internal/vcap/vcap.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,33 +59,20 @@ func GetUserProvidedCredentials(label string) (structs.UserProvidedCredentials,
return nil, errors.Errorf("No credentials found for '%s'", label)
}

func GetS3Credentials(name string) (*structs.CredentialsS3, error) {
func GetS3Credentials(name string) (map[string]string, error) {
var instanceSlice []structs.InstanceS3
err := viper.UnmarshalKey("s3", &instanceSlice)
if err != nil {
logging.Logger.Println("Could not unmarshal s3 from VCAP_SERVICES")
}
for _, instance := range instanceSlice {
if instance.Name == name {
all_looks_good := false

// We have to have an endpoint, or the two key bits.
if (len(instance.Credentials.AccessKeyId) > 0) &&
(len(instance.Credentials.SecretAccessKey) > 0) {
all_looks_good = true
} else if len(instance.Credentials.Uri) > 0 {
all_looks_good = true
}
fmt.Println("INST", instance)
fmt.Println("AKI", instance.Credentials["access_key_id"])
fmt.Println("SAK", instance.Credentials["secret_access_key"])
fmt.Println("REG", instance.Credentials["region"])

if len(instance.Credentials.Region) < 1 {
logging.Logger.Println("BACKUPS region is empty")
os.Exit(-1)
}
if all_looks_good {
return &instance.Credentials, nil
} else {
return nil, fmt.Errorf("BACKUPS no access key or endpoint")
}
return instance.Credentials, nil
}
}

Expand Down
21 changes: 21 additions & 0 deletions internal/vcap/vcap_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package vcap

import (
"io/ioutil"
"os"
"testing"
)
Expand Down Expand Up @@ -117,3 +118,23 @@ func TestReadUserProvided(t *testing.T) {
t.Error("Could not find a username")
}
}

func TestReadS3(t *testing.T) {
buffer, err := ioutil.ReadFile("example.json")
if err != nil {
t.Error("Could not read example.json")
}
os.Setenv("VCAP_SERVICES", string(buffer))
ReadVCAPConfig()

creds, err := GetS3Credentials("backups")
if err != nil {
t.Error("Could not read backups credentials from s3.")
}
if creds["access_key_id"] != "ACCESSKEYIDALPHA" {
t.Error("Did not get s3 access key ACCESSKEYIDALPHA")
}
if creds["secret_access_key"] != "SECRETACCESSKEY+ALPHA" {
t.Error("Did not get s3 secret key SECRETACCESSKEY+ALPHA")
}
}

0 comments on commit 2e58e9f

Please sign in to comment.