Attempting to work around RAM limits

I wonder if the probelem in cgov is RAM limits in a container?

cmd/bucket.go

@@ -11,34 +11,60 @@ import (
 	"github.com/spf13/cobra"
 	"gov.gsa.fac.cgov-util/internal/logging"
 	"gov.gsa.fac.cgov-util/internal/pipes"
+	"gov.gsa.fac.cgov-util/internal/util"
+
 	vcap "gov.gsa.fac.cgov-util/internal/vcap"
 )
 
-func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) {
-	mc_pipe := pipes.Mc(
-		pipes.PG_Dump(source_creds),
-		up,
-		"LOCAL",
-		"local_db",
-	)
-	mc_pipe.Wait()
-	if err := mc_pipe.Error(); err != nil {
-		logging.Logger.Println("BACKUPS `dump | mc` pipe failed")
-		os.Exit(-1)
+var backup_tag string
+
+// func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) {
+// 	mc_pipe := pipes.Mc(
+// 		pipes.PG_Dump(source_creds),
+// 		up,
+// 		"LOCAL",
+// 		"local_db",
+// 	)
+// 	mc_pipe.Wait()
+// 	if err := mc_pipe.Error(); err != nil {
+// 		logging.Logger.Println("BACKUPS `dump | mc` pipe failed")
+// 		os.Exit(-1)
+// 	}
+// }
+
+func bucket_local_tables(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) {
+	table_to_schema := util.Get_table_and_schema_names(source_creds)
+	for table, schema := range table_to_schema {
+		mc_pipe := pipes.Mc(
+			pipes.PG_Dump_Table(source_creds, schema, table, Debug),
+			up,
+			backup_tag,
+			source_creds.DB_Name,
+			schema, table, Debug,
+		)
+		mc_pipe.Wait()
+		if err := mc_pipe.Error(); err != nil {
+			logging.Logger.Println("BACKUPS `dump | mc` pipe failed")
+			os.Exit(-1)
+		}
 	}
 }
 
-func bucket_cgov(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) {
-	s3_pipe := pipes.S3(
-		pipes.PG_Dump(source_creds),
-		up,
-		"LOCAL",
-		"local_db",
-	)
-	s3_pipe.Wait()
-	if err := s3_pipe.Error(); err != nil {
-		logging.Logger.Println("BACKUPS `dump | s3` pipe failed")
-		os.Exit(-1)
+func bucket_cgov_tables(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) {
+	table_to_schema := util.Get_table_and_schema_names(source_creds)
+	for table, schema := range table_to_schema {
+		s3_pipe := pipes.S3(
+			pipes.PG_Dump_Table(source_creds, schema, table, Debug),
+			up,
+			backup_tag,
+			source_creds.DB_Name,
+			schema, table, Debug,
+		)
+		s3_pipe.Wait()
+		if err := s3_pipe.Error(); err != nil {
+			logging.Logger.Println("BACKUPS `dump | s3` pipe failed")
+			os.Exit(-1)
+		}
 	}
 }
 
@@ -56,10 +82,10 @@ to quickly create a Cobra application.`,
 		source_creds, _ := vcap.GetRDSCreds(SourceDB, "")
 		if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) {
 			up, _ := vcap.GetUserProvidedCredentials("mc")
-			bucket_local(source_creds, up)
+			bucket_local_tables(source_creds, up)
 		} else {
 			up, _ := vcap.GetS3Credentials(DestinationBucket)
-			bucket_cgov(source_creds, up)
+			bucket_cgov_tables(source_creds, up)
 		}
 
 	},
@@ -69,7 +95,10 @@ func init() {
 	rootCmd.AddCommand(bucketCmd)
 	bucketCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)")
 	bucketCmd.Flags().StringVarP(&DestinationBucket, "destination-bucket", "", "", "destination database (req)")
-	cloneCmd.MarkFlagRequired("source-db")
-	cloneCmd.MarkFlagRequired("destination-bucket")
+	bucketCmd.Flags().StringVarP(&backup_tag, "backup-tag", "", "", "SNAPSHOT, HOURLY-03, etc. (req)")
+	bucketCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements")
+	bucketCmd.MarkFlagRequired("source-db")
+	bucketCmd.MarkFlagRequired("destination-bucket")
+	bucketCmd.MarkFlagRequired("backup_tag")
 
 }

cmd/clone.go

@@ -9,20 +9,33 @@ import (
 	"github.com/spf13/cobra"
 	"gov.gsa.fac.cgov-util/internal/logging"
 	"gov.gsa.fac.cgov-util/internal/pipes"
+	"gov.gsa.fac.cgov-util/internal/util"
 	vcap "gov.gsa.fac.cgov-util/internal/vcap"
 
 	_ "github.com/lib/pq"
 )
 
 func clone(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) {
-	psql_pipe := pipes.Psql(pipes.PG_Dump(source), dest)
+	psql_pipe := pipes.Psql(pipes.PG_Dump(source, Debug), dest, Debug)
 	psql_pipe.Wait()
 	if err := psql_pipe.Error(); err != nil {
 		logging.Logger.Println("BACKUPS Pipe failed")
 		os.Exit(-1)
 	}
 }
 
+func clone_tables(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) {
+	table_to_schema := util.Get_table_and_schema_names(source)
+	for table, schema := range table_to_schema {
+		psql_pipe := pipes.Psql(pipes.PG_Dump_Table(source, schema, table, Debug), dest, Debug)
+		psql_pipe.Wait()
+		if err := psql_pipe.Error(); err != nil {
+			logging.Logger.Printf("BACKUPS Pipe failed for %s, %s\n", schema, table)
+			os.Exit(-1)
+		}
+	}
+}
+
 // snapshotDbToDbCmd represents the snapshotDbToDb command
 var cloneCmd = &cobra.Command{
 	Use:   "clone",
@@ -37,14 +50,15 @@ writes to a snapshot clone DB.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
 		source_creds, dest_creds := vcap.GetRDSCreds(SourceDB, DestinationDB)
-		clone(source_creds, dest_creds)
+		clone_tables(source_creds, dest_creds)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(cloneCmd)
 	cloneCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)")
 	cloneCmd.Flags().StringVarP(&DestinationDB, "destination-db", "", "", "destination database (req)")
+	cloneCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements")
 	cloneCmd.MarkFlagRequired("source-db")
 	cloneCmd.MarkFlagRequired("destination-db")
 

cmd/root.go

@@ -15,6 +15,7 @@ var (
 	DestinationDB     string
 	DestinationBucket string
 	SHA1              string
+	Debug             bool
 
 	// rootCmd represents the base command when called without any subcommands
 	rootCmd = &cobra.Command{

config.json

@@ -38,7 +38,32 @@
                 "name": "postgres",
                 "password": "",
                 "port": "5432",
-                "uri": "postgres://USERNAMEINDIA:[email protected]:5432/DBNAMEINDIA",
+                "uri": "postgres://[email protected]:5432/postgres?sslmode=disable",
+                "username": "postgres"
+            },
+            "syslog_drain_url": null,
+            "volume_mounts": []
+        },
+        {
+            "label": "fac-snapshot-db",
+            "provider": null,
+            "plan": null,
+            "name": "fac-snapshot-db",
+            "tags": [
+                "database",
+                "docker"
+            ],
+            "instance_guid": "UUIDJULIET1",
+            "instance_name": "db",
+            "binding_guid": "UUIDJULIET2",
+            "binding_name": null,
+            "credentials": {
+                "db_name": "postgres",
+                "host": "127.0.0.1",
+                "name": "postgres",
+                "password": "",
+                "port": "6543",
+                "uri": "postgres://[email protected]:6543/postgres?sslmode=disable",
                 "username": "postgres"
             },
             "syslog_drain_url": null,

internal/pipes/mc.go

@@ -12,7 +12,12 @@ import (
 )
 
 // https://bitfieldconsulting.com/golang/scripting
-func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, source_db string) *script.Pipe {
+func Mc(in_pipe *script.Pipe,
+	upc vcap.UserProvidedCredentials,
+	prefix string,
+	source_db string,
+	schema string,
+	table string, debug bool) *script.Pipe {
 	// // mc pipe myminio/gsa-fac-private-s3/backups/${PREFIX}-${FROM_DATABASE}.dump
 	// Always set the alias first.
 	os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"])
@@ -33,14 +38,18 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s
 	cmd := []string{
 		"mc",
 		"pipe",
-		fmt.Sprintf("%s/%s/backups/%s-%s.dump",
+		fmt.Sprintf("%s/%s/backups/%s-%s-%s_%s.dump",
 			minio_alias,
 			upc["bucket"],
 			prefix,
-			source_db),
+			source_db,
+			schema, table),
 	}
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
+	if debug {
+		fmt.Printf("command: %s\n", combined)
+	}
 	logging.Logger.Printf("BACKUPS mc targeting %s", prefix)
 	return in_pipe.Exec(combined)
 }

internal/pipes/pg_dump.go

@@ -9,8 +9,38 @@ import (
 	"gov.gsa.fac.cgov-util/internal/vcap"
 )
 
+func PG_Dump_Table(creds *vcap.CredentialsRDS, schema string, table string, debug bool) *script.Pipe {
+	// Compose the command as a slice
+	cmd := []string{
+		"pg_dump",
+		"--clean",
+		"--no-password",
+		"--if-exists",
+		"--no-privileges",
+		"--no-owner",
+		"--format plain",
+		"--table",
+		fmt.Sprintf("%s.%s", schema, table),
+		"--dbname",
+		fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
+			creds.Username,
+			creds.Password,
+			creds.Host,
+			creds.Port,
+			creds.DB_Name,
+		),
+	}
+	// Combine the slice for printing and execution.
+	combined := strings.Join(cmd[:], " ")
+	if debug {
+		fmt.Printf("command: %s\n", combined)
+	}
+	logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name)
+	return script.Exec(combined)
+}
+
 // https://bitfieldconsulting.com/golang/scripting
-func PG_Dump(creds *vcap.CredentialsRDS) *script.Pipe {
+func PG_Dump(creds *vcap.CredentialsRDS, debug bool) *script.Pipe {
 	// Compose the command as a slice
 	cmd := []string{
 		"pg_dump",
@@ -31,7 +61,9 @@ func PG_Dump(creds *vcap.CredentialsRDS) *script.Pipe {
 	}
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
-	fmt.Printf("command: %s\n", combined)
+	if debug {
+		fmt.Printf("command: %s\n", combined)
+	}
 	logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name)
 	return script.Exec(combined)
 }

internal/pipes/psql.go

@@ -9,7 +9,7 @@ import (
 	"gov.gsa.fac.cgov-util/internal/vcap"
 )
 
-func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS) *script.Pipe {
+func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS, debug bool) *script.Pipe {
 	cmd := []string{
 		"psql",
 		"--no-password",
@@ -23,6 +23,9 @@ func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS) *script.Pipe {
 		),
 	}
 	combined := strings.Join(cmd[:], " ")
+	if debug {
+		logging.Logger.Printf("command: %s\n", combined)
+	}
 	logging.Logger.Printf("BACKUPS psql targeting %s\n", creds.DB_Name)
 	return in_pipe.Exec(combined)
 }

internal/pipes/s3.go

@@ -11,7 +11,11 @@ import (
 )
 
 // https://bitfieldconsulting.com/golang/scripting
-func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db string) *script.Pipe {
+func S3(in_pipe *script.Pipe,
+	up *vcap.CredentialsS3,
+	prefix string,
+	source_db string,
+	schema string, table string, debug bool) *script.Pipe {
 	os.Setenv("AWS_ACCESS_KEY_ID", up.AccessKeyId)
 	os.Setenv("AWS_SECRET_ACCESS_KEY", up.SecretAccessKey)
 	os.Setenv("AWS_DEFAULT_REGION", up.Region)
@@ -30,7 +34,10 @@ func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db s
 
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
-	fmt.Printf("command: %s\n", combined)
+	if debug {
+		fmt.Printf("command: %s\n", combined)
+
+	}
 	logging.Logger.Printf("BACKUPS s3 targeting %s\n", prefix)
 	return in_pipe.Exec(combined)
 }

internal/util/db.go

@@ -0,0 +1,40 @@
+package util
+
+import (
+	"database/sql"
+	"os"
+
+	"gov.gsa.fac.cgov-util/internal/logging"
+	"gov.gsa.fac.cgov-util/internal/vcap"
+)
+
+func Get_table_and_schema_names(source_creds *vcap.CredentialsRDS) map[string]string {
+	// Do this table-by-table for RAM reasons.
+	db, err := sql.Open("postgres", source_creds.Uri)
+	if err != nil {
+		logging.Logger.Println("BACKUPS could not connect to DB for table-by-table dump")
+		logging.Logger.Printf("BACKUPS %s\n", err)
+		os.Exit(-1)
+	}
+
+	tables, err := db.Query("SELECT schemaname, tablename FROM pg_tables WHERE schemaname = 'public'")
+	if err != nil {
+		logging.Logger.Println("BACKUPS could not get table names for table-by-table dump")
+		logging.Logger.Printf("BACKUPS %s\n", err)
+		os.Exit(-1)
+	}
+
+	table_names := make(map[string]string, 0)
+
+	for tables.Next() {
+		var table string
+		var schema string
+		if err := tables.Scan(&schema, &table); err != nil {
+			logging.Logger.Println("BACKUPS could not scan table names in SELECT")
+			os.Exit(-1)
+		}
+		table_names[table] = schema
+	}
+
+	return table_names
+}

internal/vcap/vcap.go

@@ -141,6 +141,7 @@ func GetRDSCreds(source_db string, dest_db string) (*CredentialsRDS, *Credential
 			source, err = GetLocalRDSCredentials(source_db)
 			if err != nil {
 				logging.Logger.Println("BACKUPS Cannot get local source credentials")
+				logging.Logger.Println(err)
 				os.Exit(-1)
 			}
 		} else {