From 816e20cbbcd6aa042d87032c482ed8bc753bc94b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Sat, 2 Mar 2024 17:28:51 -0500 Subject: [PATCH] Better readme, changing package name from .backups to .cgov-util --- .github/workflows/build_go.yaml | 2 +- .github/workflows/release.yaml | 4 ++-- .gitignore | 2 +- README.md | 33 ++++++++++++++++++++++++++------- cmd/bucket.go | 6 +++--- cmd/check.go | 4 ++-- cmd/clone.go | 6 +++--- go.mod | 2 +- internal/pipes/mc.go | 6 ++---- internal/pipes/pg_dump.go | 6 ++---- internal/pipes/psql.go | 4 ++-- internal/pipes/s3.go | 6 ++---- internal/vcap/vcap.go | 2 +- main.go | 4 ++-- 14 files changed, 50 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_go.yaml b/.github/workflows/build_go.yaml index 57b8332..36afbcf 100644 --- a/.github/workflows/build_go.yaml +++ b/.github/workflows/build_go.yaml @@ -1,7 +1,7 @@ # This workflow will build a golang project # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go -name: Build gov.gsa.fac.backups +name: Build gov.gsa.fac.cgov-util env: GO_VERSION: '1.22' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4c03123..cb99b6f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,6 +1,6 @@ # .github/workflows/release.yaml -name: Release gov.gsa.fac.backups +name: Release gov.gsa.fac.cgov-util on: release: @@ -22,5 +22,5 @@ jobs: goos: linux goarch: amd64 goversion: "https://go.dev/dl/go1.22.0.linux-amd64.tar.gz" - binary_name: "gov.gsa.fac.backups" + binary_name: "gov.gsa.fac.cgov-util" extra_files: LICENSE README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1ec6a33..8255885 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -gov.gsa.fac.backups \ No newline at end of file +gov.gsa.fac.cgov-util \ No newline at end of file diff --git a/README.md b/README.md index 628a052..e4030ce 100644 --- a/README.md +++ b/README.md @@ -3,28 +3,47 @@ ## while developing/testing ``` -go run main.go clone-db-to-db +go run main.go ``` ## to build ``` -go build +./build.sh ``` -## to run +## Usage: clone ``` -./gov.gsa.fac.backups +gov.gsa.fac.cgov-util clone --source-db --destination-db ``` -to see the options, and +This command clones one DB to another by piping STDOUT from `pg_dump` into the STDIN of `psql`, with the correct connection/credential parameters for each command. + +When run localling (assuming `ENV` is set to `LOCAL`) it will read a `config.json` from the directory `$HOME/.fac/config.json` (or, from `config.json` in the same folder as the application). This file should look like a `VCAP_SERVICES` variable that would be encountered in the Cloud Foundry/cloud.gov environment. + +When run in the cloud.gov environment (where `ENV` is anything other than `LOCAL` or `TESTING`), it will look at `$VCAP_SERVICES`, look in the `aws-rds` key, and look up the DB credentials by the friendly name provided on the command line. By this, if your brokered DB is called `fac-db`, this will then populate the credentials (internally) with the brokered DB name, password, URI, etc. in order to correctly `pg_dump` from one and, using another set of credentials, stream the data into another. + +This does *not* guarantee a perfect backup. It *does* do a rapid snapshot at a moment in time, without requiring the application to write any files to the local filesystem within a container. (On cloud.gov, this is limited to ~6GB, which makes dumping and loading DBs difficult.) + +## Usage: bucket ``` -./gov.gsa.fac.backups clone-db-to-db +gsa.gov.fac.cgov-util bucket --source-db --destination-bucket ``` -to backup a source DB to a destination, based on `config.yaml` settings. +Similar to above, but this pipes a `pg_dump` to `s3 copy`. + +For now, this writes to the key `s3:///backups/-.dump` + +This wants to be improved. + +The purpose here is to (again) dump a database to a storage location without touching the local (containerized) filesystem. It uses friendly names, again, to look up the credentials for both the RDS database and brokered S3 in order to stream a DB dump to S3. (In theory, S3 does multipart uploads, so you should end up with a single file, up to 5TB in size, for your dump.) + +When running locally, this assumes `minio` is running as a stand-in for S3, and is specified as a `user-specified` service in the (local, bogus) VCAP_SERVICES config. + +(An example `config.json` is in this repository, and a more complete file in `internal/vcap/vcap_test.go`). + ## assumptions diff --git a/cmd/bucket.go b/cmd/bucket.go index 81cd994..55198dd 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -9,9 +9,9 @@ import ( "golang.org/x/exp/slices" "github.com/spf13/cobra" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/pipes" - vcap "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/pipes" + vcap "gov.gsa.fac.cgov-util/internal/vcap" ) func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) { diff --git a/cmd/check.go b/cmd/check.go index a026129..2f1a5cc 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -9,8 +9,8 @@ import ( "os" "github.com/spf13/cobra" - "gov.gsa.fac.backups/internal/logging" - vcap "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + vcap "gov.gsa.fac.cgov-util/internal/vcap" ) func get_row_count(creds *vcap.CredentialsRDS, table string) int { diff --git a/cmd/clone.go b/cmd/clone.go index 948dac8..5ef8c98 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -7,9 +7,9 @@ import ( "os" "github.com/spf13/cobra" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/pipes" - vcap "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/pipes" + vcap "gov.gsa.fac.cgov-util/internal/vcap" _ "github.com/lib/pq" ) diff --git a/go.mod b/go.mod index 2091d3f..05ba577 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gov.gsa.fac.backups +module gov.gsa.fac.cgov-util go 1.20 diff --git a/internal/pipes/mc.go b/internal/pipes/mc.go index 450d640..6a027c1 100644 --- a/internal/pipes/mc.go +++ b/internal/pipes/mc.go @@ -7,8 +7,8 @@ import ( "github.com/bitfield/script" "github.com/google/uuid" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/vcap" ) // https://bitfieldconsulting.com/golang/scripting @@ -41,8 +41,6 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - // This will log the password... - logging.Logger.Printf("BACKUPS Running `%s`\n", combined) logging.Logger.Printf("BACKUPS mc targeting %s", prefix) return in_pipe.Exec(combined) } diff --git a/internal/pipes/pg_dump.go b/internal/pipes/pg_dump.go index 5e5b465..6bfca02 100644 --- a/internal/pipes/pg_dump.go +++ b/internal/pipes/pg_dump.go @@ -5,8 +5,8 @@ import ( "strings" "github.com/bitfield/script" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/vcap" ) // https://bitfieldconsulting.com/golang/scripting @@ -31,8 +31,6 @@ func PG_Dump(creds *vcap.CredentialsRDS) *script.Pipe { } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - // This will log the password... - logging.Logger.Printf("BACKUPS Running `%s`\n", combined) logging.Logger.Printf("BACKUPS pg_dump targeting %s", creds.DB_Name) return script.Exec(combined) } diff --git a/internal/pipes/psql.go b/internal/pipes/psql.go index 4000ee4..c5c7e45 100644 --- a/internal/pipes/psql.go +++ b/internal/pipes/psql.go @@ -5,8 +5,8 @@ import ( "strings" "github.com/bitfield/script" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/vcap" ) func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS) *script.Pipe { diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go index cf83fe5..dcb9152 100644 --- a/internal/pipes/s3.go +++ b/internal/pipes/s3.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/bitfield/script" - "gov.gsa.fac.backups/internal/logging" - "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/vcap" ) // https://bitfieldconsulting.com/golang/scripting @@ -26,8 +26,6 @@ func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db s // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - // This will log the password... - logging.Logger.Printf("BACKUPS Running `%s`\n", combined) logging.Logger.Printf("BACKUPS s3 targeting %s", prefix) return in_pipe.Exec(combined) } diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go index 6c521f5..65027bb 100644 --- a/internal/vcap/vcap.go +++ b/internal/vcap/vcap.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/viper" - "gov.gsa.fac.backups/internal/logging" + "gov.gsa.fac.cgov-util/internal/logging" ) type CredentialsRDS struct { diff --git a/main.go b/main.go index 3377865..26bc566 100644 --- a/main.go +++ b/main.go @@ -10,8 +10,8 @@ import ( "golang.org/x/exp/slices" "github.com/spf13/viper" - "gov.gsa.fac.backups/cmd" - "gov.gsa.fac.backups/internal/vcap" + "gov.gsa.fac.cgov-util/cmd" + "gov.gsa.fac.cgov-util/internal/vcap" ) var SHA1 string