From c13532614c7f6d2b66f6161b5e83f9f0a0768804 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Sat, 20 Apr 2024 14:54:54 -0400 Subject: [PATCH] Better data structures. --- README.md | 2 + cmd/{bucket.go => bucket._go} | 44 +++++++-- cmd/{check.go => check._go} | 25 +++-- cmd/{clone.go => clone._go} | 8 +- cmd/dumpDbToS3.go | 139 ++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 4 + internal/pipes/mc.go | 30 +++--- internal/pipes/pg_dump.go | 32 ++++--- internal/pipes/psql.go | 16 ++-- internal/pipes/s3.go | 23 +++-- internal/structs/vcap.go | 66 ------------- internal/util/db.go | 40 -------- internal/vcap/vcap.go | 168 +++++++++++----------------------- internal/vcap/vcap_test.go | 47 +++++----- main.go | 16 +--- 16 files changed, 335 insertions(+), 328 deletions(-) rename cmd/{bucket.go => bucket._go} (67%) rename cmd/{check.go => check._go} (78%) rename cmd/{clone.go => clone._go} (84%) create mode 100644 cmd/dumpDbToS3.go delete mode 100644 internal/structs/vcap.go delete mode 100644 internal/util/db.go diff --git a/README.md b/README.md index e4030ce..e202778 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,5 @@ You can change the local DB values in `config.yaml` to reflect your config. In a remote environment, the variable `VCAP_SERVICES` is referenced to extract values. +## adding a new command + diff --git a/cmd/bucket.go b/cmd/bucket._go similarity index 67% rename from cmd/bucket.go rename to cmd/bucket._go index 4a67165..c791f18 100644 --- a/cmd/bucket.go +++ b/cmd/bucket._go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "database/sql" "os" "golang.org/x/exp/slices" @@ -11,7 +12,6 @@ import ( "github.com/spf13/cobra" "gov.gsa.fac.cgov-util/internal/logging" "gov.gsa.fac.cgov-util/internal/pipes" - "gov.gsa.fac.cgov-util/internal/structs" "gov.gsa.fac.cgov-util/internal/util" vcap "gov.gsa.fac.cgov-util/internal/vcap" @@ -33,14 +33,44 @@ var backup_tag string // } // } -func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserProvidedCredentials) { - table_to_schema := util.Get_table_and_schema_names(source_creds) +func get_table_and_schema_names(source_creds vcap.Credentials) map[string]string { + // Do this table-by-table for RAM reasons. + db, err := sql.Open("postgres", source_creds.Get("uri").String()) + if err != nil { + logging.Logger.Println("BACKUPS could not connect to DB for table-by-table dump") + logging.Logger.Printf("BACKUPS %s\n", err) + os.Exit(-1) + } + + tables, err := db.Query("SELECT schemaname, tablename FROM pg_tables WHERE schemaname = 'public'") + if err != nil { + logging.Logger.Println("BACKUPS could not get table names for table-by-table dump") + logging.Logger.Printf("BACKUPS %s\n", err) + os.Exit(-1) + } + + table_names := make(map[string]string, 0) + + for tables.Next() { + var table string + var schema string + if err := tables.Scan(&schema, &table); err != nil { + logging.Logger.Println("BACKUPS could not scan table names in SELECT") + os.Exit(-1) + } + table_names[table] = schema + } + + return table_names +} +func bucket_local_tables(source_creds vcap.Credentials, up vcap.Credentials) { + table_to_schema := get_table_and_schema_names(source_creds) for table, schema := range table_to_schema { mc_pipe := pipes.Mc( pipes.PG_Dump_Table(source_creds, schema, table), up, backup_tag, - source_creds.DB_Name, + source_creds.Get("db_name").String(), schema, table, ) mc_pipe.Wait() @@ -51,8 +81,8 @@ func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserPr } } -func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up map[string]string) { - table_to_schema := util.Get_table_and_schema_names(source_creds) +func bucket_cgov_tables(source_creds vcap.Credentials, up vcap.Credentials) { + table_to_schema := get_table_and_schema_names(source_creds) for table, schema := range table_to_schema { s3_pipe := pipes.S3( pipes.PG_Dump_Table(source_creds, schema, table), @@ -80,7 +110,7 @@ Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { - source_creds, _ := vcap.GetRDSCreds(SourceDB, "") + source_creds, _ := vcap.GetRDSCredentials(SourceDB) if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) { up, _ := vcap.GetUserProvidedCredentials("mc") bucket_local_tables(source_creds, up) diff --git a/cmd/check.go b/cmd/check._go similarity index 78% rename from cmd/check.go rename to cmd/check._go index 9eaf654..80cecca 100644 --- a/cmd/check.go +++ b/cmd/check._go @@ -10,19 +10,19 @@ import ( "github.com/spf13/cobra" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" vcap "gov.gsa.fac.cgov-util/internal/vcap" ) -func get_row_count(creds *structs.CredentialsRDS, table string) int { +func get_row_count(creds vcap.Credentials, table string) int { var count int // FIXME: Not sure if `disable` is correct for RDS sslmode. connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", - creds.Username, - creds.Password, - creds.Host, - creds.DB_Name) + creds.Get("username").String(), + creds.Get("password").String(), + creds.Get("host").String(), + creds.Get("db_name").String(), + ) db, _ := sql.Open("postgres", connStr) defer db.Close() row := db.QueryRow(fmt.Sprintf("SELECT count(*) FROM %s", table)) @@ -32,7 +32,7 @@ func get_row_count(creds *structs.CredentialsRDS, table string) int { return count } -func check_results(source *structs.CredentialsRDS, dest *structs.CredentialsRDS, tables []string) { +func check_results(source vcap.Credentials, dest vcap.Credentials, tables []string) { // FIXME: These won't exist in the VCAP_SERVICES version // of the config. We'll have to always... load both? // There needs to be a way to configure this in the remote env. @@ -65,7 +65,16 @@ Expects a space-separated list of table names as arguments. `, Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - source_creds, dest_creds := vcap.GetRDSCreds(SourceDB, DestinationDB) + source_creds, err := vcap.GetRDSCredentials(SourceDB) + if err != nil { + logging.Logger.Printf("CGOVUTIL cannot get source creds") + os.Exit(-1) + } + dest_creds, err := vcap.GetRDSCredentials(DestinationDB) + if err != nil { + logging.Logger.Printf("CGOVUTIL cannot get dest creds") + os.Exit(-1) + } check_results(source_creds, dest_creds, args) }, diff --git a/cmd/clone.go b/cmd/clone._go similarity index 84% rename from cmd/clone.go rename to cmd/clone._go index 51f3602..0641c3b 100644 --- a/cmd/clone.go +++ b/cmd/clone._go @@ -9,14 +9,12 @@ import ( "github.com/spf13/cobra" "gov.gsa.fac.cgov-util/internal/logging" "gov.gsa.fac.cgov-util/internal/pipes" - "gov.gsa.fac.cgov-util/internal/structs" - "gov.gsa.fac.cgov-util/internal/util" vcap "gov.gsa.fac.cgov-util/internal/vcap" _ "github.com/lib/pq" ) -func clone(source *structs.CredentialsRDS, dest *structs.CredentialsRDS) { +func clone(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) { psql_pipe := pipes.Psql(pipes.PG_Dump(source), dest) psql_pipe.Wait() if err := psql_pipe.Error(); err != nil { @@ -25,8 +23,8 @@ func clone(source *structs.CredentialsRDS, dest *structs.CredentialsRDS) { } } -func clone_tables(source *structs.CredentialsRDS, dest *structs.CredentialsRDS) { - table_to_schema := util.Get_table_and_schema_names(source) +func clone_tables(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) { + table_to_schema := get_table_and_schema_names(source) for table, schema := range table_to_schema { psql_pipe := pipes.Psql(pipes.PG_Dump_Table(source, schema, table), dest) psql_pipe.Wait() diff --git a/cmd/dumpDbToS3.go b/cmd/dumpDbToS3.go new file mode 100644 index 0000000..8d6ae02 --- /dev/null +++ b/cmd/dumpDbToS3.go @@ -0,0 +1,139 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "database/sql" + "fmt" + "os" + + _ "github.com/lib/pq" + + "github.com/spf13/cobra" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/pipes" + + "gov.gsa.fac.cgov-util/internal/vcap" +) + +var ( + db string + s3path string +) + +func get_table_and_schema_names(source_creds vcap.Credentials) map[string]string { + // Do this table-by-table for RAM reasons. + db, err := sql.Open("postgres", source_creds.Get("uri").String()) + if err != nil { + logging.Logger.Println("DUMPDBTOS3 could not connect to DB for table-by-table dump") + logging.Logger.Printf("DUMPDBTOS3 %s\n", err) + os.Exit(-1) + } + + tables, err := db.Query("SELECT schemaname, tablename FROM pg_tables WHERE schemaname = 'public'") + if err != nil { + logging.Logger.Println("DUMPDBTOS3 could not get table names for table-by-table dump") + logging.Logger.Printf("DUMPDBTOS3 %s\n", err) + os.Exit(-1) + } + + table_names := make(map[string]string, 0) + + for tables.Next() { + var table string + var schema string + if err := tables.Scan(&schema, &table); err != nil { + logging.Logger.Println("DUMPDBTOS3 could not scan table names in SELECT") + os.Exit(-1) + } + table_names[table] = schema + } + + return table_names +} +func bucket_local_tables(source_creds vcap.Credentials, up vcap.Credentials) { + logging.Logger.Printf("DUMPDBTOS3 backing up from %s to %s\n", + source_creds.Get("name").String(), + up.Get("name").String(), + ) + table_to_schema := get_table_and_schema_names(source_creds) + for table, schema := range table_to_schema { + mc_pipe := pipes.Mc( + pipes.PG_Dump_Table(source_creds, schema, table), + up, + fmt.Sprintf("%s/%s-%s.dump", s3path, schema, table), + ) + mc_pipe.Wait() + if err := mc_pipe.Error(); err != nil { + logging.Logger.Println("DUMPDBTOS3 `dump | mc` pipe failed") + os.Exit(-1) + } + } +} + +func bucket_cgov_tables(source_creds vcap.Credentials, up vcap.Credentials) { + table_to_schema := get_table_and_schema_names(source_creds) + for table, schema := range table_to_schema { + s3_pipe := pipes.S3( + pipes.PG_Dump_Table(source_creds, schema, table), + up, + fmt.Sprintf("%s/%s-%s.dump", s3path, schema, table), + ) + s3_pipe.Wait() + if err := s3_pipe.Error(); err != nil { + logging.Logger.Println("DUMPDBTOS3 `dump | s3` pipe failed") + os.Exit(-1) + } + } +} + +// dumpDbToS3Cmd represents the dumpDbToS3 command +var dumpDbToS3Cmd = &cobra.Command{ + Use: "dumpDbToS3", + Short: "Dumps a full database to a file in S3", + Long: `Dumps a full database to a file in S3`, + Run: func(cmd *cobra.Command, args []string) { + + // Check that we can get credentials. + db_creds, err := vcap.VCS.GetCredentials("aws-rds", db) + if err != nil { + logging.Logger.Printf("DUMPDBTOS3 could not get DB credentials for %s", db) + os.Exit(-1) + } + + switch os.Getenv("ENV") { + case "LOCAL": + fallthrough + case "TESTING": + up, err := vcap.VCS.GetCredentials("user-provided", "backups") + if err != nil { + logging.Logger.Printf("DUMPDBTOS3 could not get minio credentials") + os.Exit(-1) + } + bucket_local_tables(db_creds, up) + case "DEV": + fallthrough + case "STAGING": + fallthrough + case "PRODUCTION": + up, err := vcap.VCS.GetCredentials("aws-rds", s3path) + if err != nil { + logging.Logger.Printf("DUMPDBTOS3 could not get s3 credentials") + os.Exit(-1) + } + bucket_cgov_tables(db_creds, up) + + } + }, +} + +func init() { + rootCmd.AddCommand(dumpDbToS3Cmd) + dumpDbToS3Cmd.Flags().StringVarP(&db, "db", "", "", "source database label") + dumpDbToS3Cmd.Flags().StringVarP(&s3path, "s3path", "", "", "destination path") + + dumpDbToS3Cmd.MarkFlagRequired("db") + dumpDbToS3Cmd.MarkFlagRequired("s3path") + +} diff --git a/go.mod b/go.mod index 05ba577..e11aa7f 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,8 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 79271df..c4b09b5 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,10 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/pipes/mc.go b/internal/pipes/mc.go index c1259ed..3f93505 100644 --- a/internal/pipes/mc.go +++ b/internal/pipes/mc.go @@ -8,48 +8,44 @@ import ( "github.com/bitfield/script" "github.com/google/uuid" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" "gov.gsa.fac.cgov-util/internal/util" + "gov.gsa.fac.cgov-util/internal/vcap" ) // https://bitfieldconsulting.com/golang/scripting func Mc(in_pipe *script.Pipe, - upc structs.UserProvidedCredentials, - prefix string, - source_db string, - schema string, - table string) *script.Pipe { + creds vcap.Credentials, + path string) *script.Pipe { // // mc pipe myminio/gsa-fac-private-s3/backups/${PREFIX}-${FROM_DATABASE}.dump // Always set the alias first. - os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"]) - os.Setenv("AWS_PRIVATE_SECRET_ACCESS_KEY", upc["secret_access_key"]) + os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", creds.Get("access_key_id").String()) + os.Setenv("AWS_PRIVATE_SECRET_ACCESS_KEY", creds.Get("secret_access_key").String()) minio_alias := fmt.Sprintf("minio_alias_%s", uuid.New()) set_alias := []string{ "mc", "alias", "set", minio_alias, - upc["endpoint"], - upc["admin_username"], - upc["admin_password"], + creds.Get("endpoint").String(), + creds.Get("admin_username").String(), + creds.Get("admin_password").String(), } sa_combined := strings.Join(set_alias[:], " ") - logging.Logger.Printf("BACKUPS Running `%s`\n", sa_combined) + logging.Logger.Printf("MC Running `%s`\n", sa_combined) script.Exec(sa_combined).Stdout() cmd := []string{ "mc", "pipe", - fmt.Sprintf("%s/%s/backups/%s-%s_%s.dump", + fmt.Sprintf("%s/%s/%s", minio_alias, - upc["bucket"], - prefix, - schema, table), + creds.Get("bucket").String(), + path), } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") if util.IsDebugLevel("DEBUG") { fmt.Printf("command: %s\n", combined) } - logging.Logger.Printf("BACKUPS mc targeting %s", prefix) + logging.Logger.Printf("MC mc targeting %s", path) return in_pipe.Exec(combined) } diff --git a/internal/pipes/pg_dump.go b/internal/pipes/pg_dump.go index 4aeb1bb..9742578 100644 --- a/internal/pipes/pg_dump.go +++ b/internal/pipes/pg_dump.go @@ -6,11 +6,13 @@ import ( "github.com/bitfield/script" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" "gov.gsa.fac.cgov-util/internal/util" + "gov.gsa.fac.cgov-util/internal/vcap" ) -func PG_Dump_Table(creds *structs.CredentialsRDS, schema string, table string) *script.Pipe { +func PG_Dump_Table(creds vcap.Credentials, + schema string, + table string) *script.Pipe { // Compose the command as a slice cmd := []string{ "pg_dump", @@ -24,16 +26,16 @@ func PG_Dump_Table(creds *structs.CredentialsRDS, schema string, table string) * fmt.Sprintf("%s.%s", schema, table), "--dbname", fmt.Sprintf("postgres://%s:%s@%s:%s/%s", - creds.Username, - creds.Password, - creds.Host, - creds.Port, - creds.DB_Name, + creds.Get("username").String(), + creds.Get("password").String(), + creds.Get("host").String(), + creds.Get("port").String(), + creds.Get("db_name").String(), ), } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name) + logging.Logger.Printf("BACKUPS pg_dump targeting %s.%s\n", schema, table) if util.IsDebugLevel("DEBUG") { fmt.Printf("command: %s\n", combined) } @@ -41,7 +43,7 @@ func PG_Dump_Table(creds *structs.CredentialsRDS, schema string, table string) * } // https://bitfieldconsulting.com/golang/scripting -func PG_Dump(creds *structs.CredentialsRDS) *script.Pipe { +func PG_Dump(creds vcap.Credentials) *script.Pipe { // Compose the command as a slice cmd := []string{ "pg_dump", @@ -53,16 +55,16 @@ func PG_Dump(creds *structs.CredentialsRDS) *script.Pipe { "--format plain", "--dbname", fmt.Sprintf("postgres://%s:%s@%s:%s/%s", - creds.Username, - creds.Password, - creds.Host, - creds.Port, - creds.DB_Name, + creds.Get("username").String(), + creds.Get("password").String(), + creds.Get("host").String(), + creds.Get("port").String(), + creds.Get("db_name").String(), ), } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name) + logging.Logger.Printf("BACKUPS pg_dump running\n") if util.IsDebugLevel("DEBUG") { fmt.Printf("command: %s\n", combined) } diff --git a/internal/pipes/psql.go b/internal/pipes/psql.go index 41df966..161c2f6 100644 --- a/internal/pipes/psql.go +++ b/internal/pipes/psql.go @@ -6,27 +6,27 @@ import ( "github.com/bitfield/script" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" "gov.gsa.fac.cgov-util/internal/util" + "gov.gsa.fac.cgov-util/internal/vcap" ) -func Psql(in_pipe *script.Pipe, creds *structs.CredentialsRDS) *script.Pipe { +func Psql(in_pipe *script.Pipe, creds vcap.Credentials) *script.Pipe { cmd := []string{ "psql", "--no-password", "--dbname", fmt.Sprintf("postgres://%s:%s@%s:%s/%s", - creds.Username, - creds.Password, - creds.Host, - creds.Port, - creds.DB_Name, + creds.Get("username").String(), + creds.Get("password").String(), + creds.Get("host").String(), + creds.Get("port").String(), + creds.Get("db_name"), ), } combined := strings.Join(cmd[:], " ") if util.IsDebugLevel("DEBUG") { logging.Logger.Printf("command: %s\n", combined) } - logging.Logger.Printf("BACKUPS psql targeting %s\n", creds.DB_Name) + logging.Logger.Printf("BACKUPS psql running\n") return in_pipe.Exec(combined) } diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go index 6cf5e90..6bc6e3c 100644 --- a/internal/pipes/s3.go +++ b/internal/pipes/s3.go @@ -8,34 +8,33 @@ import ( "github.com/bitfield/script" "gov.gsa.fac.cgov-util/internal/logging" "gov.gsa.fac.cgov-util/internal/util" + "gov.gsa.fac.cgov-util/internal/vcap" ) // For reasons that are unclear, the access key id and secret key // are coming through from VCAP empty. But, the endpoint is not. // This makes no sense. func S3(in_pipe *script.Pipe, - up map[string]string, - prefix string, - source_db string, - schema string, table string) *script.Pipe { - os.Setenv("AWS_SECRET_ACCESS_KEY", up["secret_access_key"]) - os.Setenv("AWS_ACCESS_KEY_ID", up["access_key_id"]) - os.Setenv("AWS_DEFAULT_REGION", up["region"]) + up vcap.Credentials, + path string) *script.Pipe { + + os.Setenv("AWS_SECRET_ACCESS_KEY", up.Get("secret_access_key").String()) + os.Setenv("AWS_ACCESS_KEY_ID", up.Get("access_key_id").String()) + os.Setenv("AWS_DEFAULT_REGION", up.Get("region").String()) // https://serverfault.com/questions/886562/streaming-postgresql-pg-dump-to-s3 cmd := []string{ "aws", "s3", "cp", "-", - fmt.Sprintf("s3://%s/backups/%s-%s_%s.dump", - up["bucket"], - prefix, - schema, table), + fmt.Sprintf("s3://%s/%s", + up.Get("bucket").String(), + path), } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - logging.Logger.Printf("BACKUPS s3 targeting %s\n", prefix) + logging.Logger.Printf("S3 s3 targeting %s\n", path) if util.IsDebugLevel("DEBUG") { fmt.Printf("command: %s\n", combined) } diff --git a/internal/structs/vcap.go b/internal/structs/vcap.go deleted file mode 100644 index 346a2ec..0000000 --- a/internal/structs/vcap.go +++ /dev/null @@ -1,66 +0,0 @@ -package structs - -type UserProvidedCredentials = map[string]string - -type CredentialsRDS struct { - DB_Name string `json:"db_name"` - Host string `json:"host"` - Name string `json:"name"` - Password string `json:"password"` - Port string `json:"port"` - Username string `json:"username"` - Uri string `json:"uri"` -} - -type CredentialsS3 struct { - Uri string `json:"uri"` - InsecureSkipVerify bool `json:"insecure_skip_verify"` - AccessKeyId string `json:"access_key_id"` - SecretAccessKey string `json:"secret_access_key"` - Region string `json:"region"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - FipsEndpoint string `json:"fips_endpoint"` - AdditionalBuckets []string `json:"additional_buckets"` -} - -type InstanceS3 struct { - Label string `json:"label"` - Provider string `json:"provider"` - Plan string `json:"plan"` - Name string `json:"name"` - Tags []string `json:"tags"` - InstanceGuid string `json:"instance_guid"` - InstanceName string `json:"instance_name"` - BindingGuid string `json:"binding_guid"` - BindingName string `json:"binding_name"` - Credentials map[string]string `json:"credentials"` - SyslogDrainUrl string `json:"syslog_drain_url"` - VolumeMounts []string `json:"volume_mounts"` -} - -type InstanceRDS struct { - Label string `json:"label"` - Provider string `json:"provider"` - Plan string `json:"plan"` - Name string `json:"name"` - Tags []string `json:"tags"` - InstanceGuid string `json:"instance_guid"` - InstanceName string `json:"instance_name"` - BindingGuid string `json:"binding_guid"` - BindingName string `json:"binding_name"` - Credentials CredentialsRDS `json:"credentials"` - SyslogDrainUrl string `json:"syslog_drain_url"` - VolumeMounts string `json:"volume_mounts"` -} - -type UserProvided struct { - Label string `json:"label"` - Name string `json:"name"` - Tags []string `json:"tags"` - InstanceGuid string `json:"instance_guid"` - InstanceName string `json:"instance_name"` - BindingGuid string `json:"binding_guid"` - BindingName string `json:"binding_name"` - Credentials map[string]string `json:"credentials"` -} diff --git a/internal/util/db.go b/internal/util/db.go deleted file mode 100644 index 7657934..0000000 --- a/internal/util/db.go +++ /dev/null @@ -1,40 +0,0 @@ -package util - -import ( - "database/sql" - "os" - - "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" -) - -func Get_table_and_schema_names(source_creds *structs.CredentialsRDS) map[string]string { - // Do this table-by-table for RAM reasons. - db, err := sql.Open("postgres", source_creds.Uri) - if err != nil { - logging.Logger.Println("BACKUPS could not connect to DB for table-by-table dump") - logging.Logger.Printf("BACKUPS %s\n", err) - os.Exit(-1) - } - - tables, err := db.Query("SELECT schemaname, tablename FROM pg_tables WHERE schemaname = 'public'") - if err != nil { - logging.Logger.Println("BACKUPS could not get table names for table-by-table dump") - logging.Logger.Printf("BACKUPS %s\n", err) - os.Exit(-1) - } - - table_names := make(map[string]string, 0) - - for tables.Next() { - var table string - var schema string - if err := tables.Scan(&schema, &table); err != nil { - logging.Logger.Println("BACKUPS could not scan table names in SELECT") - os.Exit(-1) - } - table_names[table] = schema - } - - return table_names -} diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go index f88b270..323d75f 100644 --- a/internal/vcap/vcap.go +++ b/internal/vcap/vcap.go @@ -1,142 +1,82 @@ package vcap import ( - "bytes" "fmt" + "io/ioutil" "os" - - "golang.org/x/exp/slices" + "path/filepath" "github.com/pkg/errors" - "github.com/spf13/viper" + "github.com/tidwall/gjson" "gov.gsa.fac.cgov-util/internal/logging" - "gov.gsa.fac.cgov-util/internal/structs" - "gov.gsa.fac.cgov-util/internal/util" ) -func GetRDSCredentials(label string) (*structs.CredentialsRDS, error) { - var instanceSlice []structs.InstanceRDS - err := viper.UnmarshalKey("aws-rds", &instanceSlice) - if err != nil { - logging.Logger.Println("Could not unmarshal aws-rds from VCAP_SERVICES") - } - for _, instance := range instanceSlice { - if instance.Name == label { - return &instance.Credentials, nil - } - } - return nil, errors.Errorf("No credentials found for '%s'", label) -} +//var vcap *gjson.Result = nil -// These are hardcoded to match the FAC stack. -func GetLocalRDSCredentials(label string) (*structs.CredentialsRDS, error) { - var instanceSlice []structs.InstanceRDS - err := viper.UnmarshalKey("aws-rds", &instanceSlice) - if err != nil { - logging.Logger.Println("Could not unmarshal aws-rds from VCAP_SERVICES") - } - for _, instance := range instanceSlice { - if instance.Name == label { - return &instance.Credentials, nil - } - } - return nil, errors.Errorf("No credentials found for '%s'", label) +// Alias +// type A = B +// New type +// type A B +type Credentials = gjson.Result + +type VcapServices struct { + Source string + VCAP gjson.Result } -// Returns a map, not a pointer to a structure -func GetUserProvidedCredentials(label string) (structs.UserProvidedCredentials, error) { - var instanceSlice []structs.UserProvided - err := viper.UnmarshalKey("user-provided", &instanceSlice) - if err != nil { - logging.Logger.Println("Could not unmarshal aws-rds from VCAP_SERVICES") - } - for _, instance := range instanceSlice { - if instance.Label == label { - return instance.Credentials, nil - } +var VCS *VcapServices = nil + +func (vcs *VcapServices) GetCredentials(service string, name string) (Credentials, error) { + query_string := fmt.Sprintf("%s.#(name==%s).credentials", service, name) + + r := vcs.VCAP.Get(query_string) + if r.Exists() { + return Credentials(r), nil + } else { + return Credentials{}, errors.Errorf("No <%s> credentials found for '%s'", service, name) } - return nil, errors.Errorf("No credentials found for '%s'", label) } -func GetS3Credentials(name string) (map[string]string, error) { - var instanceSlice []structs.InstanceS3 - err := viper.UnmarshalKey("s3", &instanceSlice) - if err != nil { - logging.Logger.Println("Could not unmarshal s3 from VCAP_SERVICES") +func ReadVCAPConfig() *VcapServices { + // Remotely, read it in from the VCAP_SERVICES env var, which will + // provide a large JSON structure. + vcap_string := os.Getenv("VCAP_SERVICES") + if util.IsDebugLevel("DEBUG") { + logging.Logger.Printf("---- VCAP ----\n%s\n---- END VCAP ----\n", vcap_string) } - for _, instance := range instanceSlice { - if instance.Name == name { - fmt.Println("INST", instance) - fmt.Println("AKI", instance.Credentials["access_key_id"]) - fmt.Println("SAK", instance.Credentials["secret_access_key"]) - fmt.Println("REG", instance.Credentials["region"]) - - return instance.Credentials, nil - } + json := gjson.Parse(vcap_string) + vcs := VcapServices{ + Source: "env", + VCAP: json, } - - return nil, errors.Errorf("No credentials found for '%s'", name) + VCS = &vcs + return VCS } -func GetRDSCreds(source_db string, dest_db string) (*structs.CredentialsRDS, *structs.CredentialsRDS) { - var source *structs.CredentialsRDS - var dest *structs.CredentialsRDS - var err error - - if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) { - if source_db != "" { - source, err = GetLocalRDSCredentials(source_db) - if err != nil { - logging.Logger.Println("BACKUPS Cannot get local source credentials") - logging.Logger.Println(err) - os.Exit(-1) - } +func ReadVCAPConfigFile(filename string) *VcapServices { + fp := "" + _, err := os.Open(filename) + if errors.Is(err, os.ErrNotExist) { + _, err = os.Open(filepath.Join(os.Getenv("HOME"), ".fac", filename)) + if errors.Is(err, os.ErrNotExist) { + fmt.Println("CGOVUTIL Cannot find config. Exiting.") } else { - source = nil + fp = filepath.Join(os.Getenv("HOME"), ".fac", filename) } - if dest_db != "" { - dest, err = GetLocalRDSCredentials(dest_db) - if err != nil { - logging.Logger.Println("BACKUPS Cannot get local dest credentials") - os.Exit(-1) - } - } else { - dest = nil - } - } else { - if source_db != "" { - source, err = GetRDSCredentials(source_db) - if err != nil { - logging.Logger.Println("BACKUPS Cannot get RDS source credentials") - os.Exit(-1) - } - } else { - source = nil - } - if dest_db != "" { - dest, err = GetRDSCredentials(dest_db) - if err != nil { - logging.Logger.Println("BACKUPS Cannot get RDS dest credentials") - os.Exit(-1) - } - } else { - dest = nil - } + fp = filename } - return source, dest -} - -func ReadVCAPConfig() { - // Remotely, read it in from the VCAP_SERVICES env var, which will - // provide a large JSON structure. - viper.SetConfigType("json") - vcap := os.Getenv("VCAP_SERVICES") - if util.IsDebugLevel("DEBUG") { - logging.Logger.Printf("---- VCAP ----\n%s\n---- END VCAP ----\n", vcap) + bytes, err := ioutil.ReadFile(fp) + if err != nil { + logging.Logger.Printf("CGOVUTIL could not load config from file.") + os.Exit(-1) } - - viper.ReadConfig(bytes.NewBufferString(vcap)) + json := gjson.ParseBytes(bytes) + VCS = &VcapServices{ + Source: filename, + VCAP: json, + } + return VCS } diff --git a/internal/vcap/vcap_test.go b/internal/vcap/vcap_test.go index 391cf12..dbfb55a 100644 --- a/internal/vcap/vcap_test.go +++ b/internal/vcap/vcap_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -var db_vcap = `{ +var test_vcap = `{ "s3": [], "user-provided": [ { @@ -83,58 +83,63 @@ var db_vcap = `{ func TestReadVCAP(t *testing.T) { // Load a test string into the env. - os.Setenv("VCAP_SERVICES", db_vcap) + os.Setenv("VCAP_SERVICES", test_vcap) // Read the VCAP config. - ReadVCAPConfig() + vcap := ReadVCAPConfig() // Check to see if we can find the source DB - creds, err := GetRDSCredentials("fac-db") + creds, err := vcap.GetCredentials("aws-rds", "fac-db") if err != nil { t.Error("Could not read fac-db credentials from env.") } - if creds.DB_Name != "the-source-db-name" { - t.Error("Did not get fac-db db_name") + + if creds.Get("db_name").String() != "the-source-db-name" { + t.Error("Did not get fac-db name") } // How about the dest DB? - creds, err = GetRDSCredentials("fac-snapshot-db") + creds, err = vcap.GetCredentials("aws-rds", "fac-snapshot-db") if err != nil { t.Error("Could not read fac-db credentials from env.") } - if creds.DB_Name != "the-dest-db-name" { - t.Error("Did not get fac-db db_name") + if creds.Get("db_name").String() != "the-dest-db-name" { + t.Error("Did not get fac-db name") } } func TestReadUserProvided(t *testing.T) { // Load a test string into the env. - os.Setenv("VCAP_SERVICES", db_vcap) + os.Setenv("VCAP_SERVICES", test_vcap) // Read the VCAP config. - ReadVCAPConfig() - creds, err := GetUserProvidedCredentials("mc") + vcap := ReadVCAPConfig() + creds, err := vcap.GetCredentials("user-provided", "backups") if err != nil { t.Error("Could not read user-provided credentials from env.") } - _, ok := creds["admin_username"] - if !ok { + r := creds.Get("admin_username") + if !r.Exists() { t.Error("Could not find a username") } } -func TestReadS3(t *testing.T) { - buffer, err := ioutil.ReadFile("example.json") +func setVcapToFile(t *testing.T, file string) *VcapServices { + buffer, err := ioutil.ReadFile(file) if err != nil { t.Error("Could not read example.json") } os.Setenv("VCAP_SERVICES", string(buffer)) - ReadVCAPConfig() - - creds, err := GetS3Credentials("backups") + return ReadVCAPConfig() +} +func TestReadS3(t *testing.T) { + vcap := setVcapToFile(t, "example.json") + creds, err := vcap.GetCredentials("s3", "backups") if err != nil { t.Error("Could not read backups credentials from s3.") } - if creds["access_key_id"] != "ACCESSKEYIDALPHA" { + if creds.Get("access_key_id").String() != "ACCESSKEYIDALPHA" { + t.Logf("AccessKeyId: %v", creds.Get("secret_access_key").String()) t.Error("Did not get s3 access key ACCESSKEYIDALPHA") } - if creds["secret_access_key"] != "SECRETACCESSKEY+ALPHA" { + if creds.Get("secret_access_key").String() != "SECRETACCESSKEY+ALPHA" { + t.Logf("SecretAccessKey: %v", creds.Get("secret_access_key").String()) t.Error("Did not get s3 secret key SECRETACCESSKEY+ALPHA") } } diff --git a/main.go b/main.go index 26bc566..041b28d 100644 --- a/main.go +++ b/main.go @@ -4,18 +4,14 @@ Copyright © 2024 NAME HERE package main import ( - "fmt" "os" "golang.org/x/exp/slices" - "github.com/spf13/viper" "gov.gsa.fac.cgov-util/cmd" "gov.gsa.fac.cgov-util/internal/vcap" ) -var SHA1 string - // Useful documentation for people new to Go, and // related to modules in this command. // https://www.digitalocean.com/community/tutorials/how-to-use-the-cobra-package-in-go @@ -28,25 +24,17 @@ var SHA1 string // Looks for config.yaml in the same directory as the app. // Optionally, the config can be in `$HOME/.fac/config.yaml` func readConfig() { - // Pass down the SHA - cmd.SHA1 = SHA1 // Do the right thing in the right env. if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) { // Locally, load the file from one of two places. - viper.SetConfigName("config") - viper.SetConfigType("json") - viper.AddConfigPath("$HOME/.fac") - viper.AddConfigPath(".") - err := viper.ReadInConfig() - if err != nil { - panic(fmt.Errorf("fatal error config file: %w", err)) - } + vcap.ReadVCAPConfigFile("config.json") } else { vcap.ReadVCAPConfig() } } func main() { + readConfig() cmd.Execute() }