Merge pull request #1 from GSA-TTS/check-if-table-exists

Check tables in DB before running

cmd/assets/db_tables.txt

@@ -0,0 +1,134 @@
+census_duns21
+census_eins18
+census_cfda19
+census_duns18
+census_eins16
+census_cpas21
+census_cpas17
+census_agency16
+census_captext19
+census_captext_formatted21
+census_cfda20
+census_captext20
+census_agency17
+census_captext21
+dissemination_note
+census_cfda16
+census_duns19
+census_cpas18
+census_cpas22
+census_agency19
+census_eins17
+census_duns22
+census_cfda21
+census_captext_formatted19
+census_cpas19
+census_duns16
+census_agency20
+census_cfda22
+census_cfda17
+census_notes20
+census_duns17
+census_cfda18
+census_cpas20
+census_duns20
+census_captext_formatted20
+census_agency22
+census_cpas16
+census_findings18
+census_eins19
+census_findings16
+census_findings20
+census_findings22
+census_findings21
+census_eins21
+census_eins22
+census_eins20
+census_findings19
+census_findings17
+census_findingstext_formatted22
+census_findingstext21
+census_gen16
+census_gen18
+census_findingstext_formatted19
+census_gen17
+census_findingstext_formatted20
+census_findingstext_formatted21
+census_findingstext19
+census_findingstext20
+census_gen22
+census_notes21
+census_notes19
+census_passthrough16
+census_passthrough18
+census_gen19
+census_gen21
+census_gen20
+census_passthrough17
+census_notes22
+census_passthrough22
+census_revisions22
+census_ueis22
+census_revisions19
+census_revisions21
+census_revisions20
+census_passthrough20
+census_passthrough19
+census_passthrough21
+audit_deletedaccess
+census_findingstext22
+auth_group_permissions
+django_content_type
+auth_permission
+django_migrations
+auth_group
+auth_user
+auth_user_user_permissions
+django_admin_log
+auth_user_groups
+audit_submissionevent
+authtoken_token
+audit_singleauditreportfile
+census_historical_migration_elecauditfindings
+census_historical_migration_eleceins
+census_historical_migration_elecfindingstext
+census_historical_migration_eleccpas
+census_historical_migration_elecnotes
+census_historical_migration_elecauditheader
+census_historical_migration_elecaudits
+census_historical_migration_eleccaptext
+dissemination_additionalein
+census_historical_migration_reportmigrationstatus
+census_historical_migration_migrationerrordetail
+census_historical_migration_elecueis
+census_historical_migration_elecpassthrough
+dissemination_finding
+support_administrative_key_uuids
+dissemination_general
+dissemination_migrationinspectionrecord
+django_session
+dissemination_issuedescriptionrecord
+dissemination_invalidauditrecord
+users_permission
+support_adminapievent
+users_staffuserlog
+users_userprofile
+support_cognizantassignment
+users_staffuser
+support_cognizantbaseline
+dissemination_tribalapiaccesskeyids
+dissemination_findingtext
+dissemination_additionaluei
+dissemination_captext
+users_userpermission
+dissemination_onetimeaccess
+dissemination_federalaward
+dissemination_passthrough
+audit_singleauditchecklist
+census_captext22
+census_captext_formatted22
+census_agency21
+dissemination_secondaryauditor
+census_agency18
+audit_excelfile
+audit_access

cmd/check_db.go

@@ -0,0 +1,98 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"bufio"
+	"database/sql"
+	"embed"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"gov.gsa.fac.cgov-util/internal/logging"
+	"gov.gsa.fac.cgov-util/internal/vcap"
+)
+
+var (
+	source_database string
+	//go:embed assets/db_tables.txt
+	f embed.FS
+)
+
+func check_if_table_exists(source_creds vcap.Credentials) {
+	// SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'schema_name' AND tablename  = 'table_name');
+	db, err := sql.Open("postgres", source_creds.Get("uri").String())
+	if err != nil {
+		logging.Logger.Println("TABLECHECK could not connect to DB for checking table existance")
+		logging.Logger.Printf("DBTOS3 %s\n", err)
+		os.Exit(logging.DB_SCHEMA_SCAN_FAILURE)
+	}
+
+	// file, err := os.Open("db_tables.txt")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// defer file.Close()
+
+	file, err := f.ReadFile("assets/db_tables.txt")
+	//print(string(file))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(string(file)))
+	//scanner := bufio.NewScanner(file)
+	var not_existing []string
+	for scanner.Scan() {
+		//scanner.Text()
+		query := fmt.Sprintf("select * from %s LIMIT 1;", scanner.Text())
+		//query := fmt.Sprintf("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = '%s')", scanner.Text())
+		rows, table_check := db.Query(query)
+
+		if table_check == nil {
+			//fmt.Println(i + " exists")
+			rows.Next()
+		} else {
+			//logging.Logger.Println(scanner.Text() + " does not exist")
+			// store all scanner.Text() into a map
+			// if map != nil
+			// hard exit
+			not_existing = append(not_existing, scanner.Text())
+		}
+	}
+	if len(not_existing) > 0 {
+		logging.Error.Println("A list of tables that does not exist in the database, but does exist in a manifest has been returned.")
+		logging.Error.Println("System exiting...")
+		joined_tables := strings.Join(not_existing[:], " ")
+		logging.Error.Printf("DBMISSINGTABLES " + joined_tables)
+		os.Exit(logging.DB_MISSING_TABLES)
+	} else {
+		logging.Status.Printf("Manifest and Database tables appear to be in sync.")
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// checkDbCmd represents the checkDb command
+var checkDbCmd = &cobra.Command{
+	Use:   "check_db",
+	Short: "A brief description of your command",
+	Long:  `A`,
+	Run: func(cmd *cobra.Command, args []string) {
+		db_creds := getDBCredentials(source_database)
+		//stringInSlice(table, list_of_tables)
+		check_if_table_exists(db_creds)
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(checkDbCmd)
+	parseFlags("check_tables", checkDbCmd)
+}

cmd/root.go

@@ -57,6 +57,10 @@ func parseFlags(cmd_name string, cmd *cobra.Command) {
 		fmt.Println("RUNNING TRUNCATE FLAGS")
 		cmd.Flags().StringVarP(&truncate_db, "db", "", "", "target database name")
 		cmd.Flags().StringSliceVarP(&truncate_truncate, "truncate", "", []string{}, "tables to truncate before load")
+	case "check_tables":
+		fmt.Println("RUNNING CHECK_TABLES FLAGS")
+		cmd.Flags().StringVarP(&source_database, "db", "", "", "database name")
+		cmd.MarkFlagRequired("source_database")
 	case "drop":
 		fmt.Println("RUNNING DROP FLAGS")
 		cmd.Flags().StringVarP(&target_db, "db", "", "", "target database name")

cmd/s3_to_db.go

@@ -25,6 +25,7 @@ func bucket_to_local_tables(
 	var PROTECTED_DB = "fac-db"
 	table_to_schema := get_table_and_schema_names(db_creds)
 	//fmt.Sprintf("%s%s/%s-%s.dump", s3path.Bucket, s3path.Key, schema, table)
+	check_if_table_exists(db_creds)
 	for table, schema := range table_to_schema {
 		dump_file_name := fmt.Sprintf("%s-%s.dump", schema, table)
 

internal/logging/logging.go

@@ -6,4 +6,6 @@ import (
 )
 
 var Logger = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime)
+var Error = log.New(os.Stdout, "ERROR: ", log.Ldate|log.Ltime)
+var Status = log.New(os.Stdout, "STATUS: ", log.Ldate|log.Ltime)
 var Warning = log.New(os.Stdout, "WARN: ", log.Ldate|log.Ltime)