Sk ez pp/hist data load

backend/Apple_M1_Dockerfile

@@ -21,10 +21,13 @@ RUN \
 
 RUN \
     apt-get update -yq && \
-    apt install curl -y && \
-    apt-get install -y gcc && \
-    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
-    apt-get install -y nodejs && \
+    apt install build-essential curl -y && \
+    apt-get install -y gcc ca-certificates gnupg && \
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    NODE_MAJOR=18 && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
+    apt-get install nodejs -y && \
     apt-get install -y npm && \
     npm i -g npm@^8
 

backend/census_historical_migration/README.md

@@ -1,4 +1,58 @@
-# Census Historical Migration
+# Census to FAC data migration
+
+## Overview
+
+This is implemented as a Django app to leverage existing management commands and settings. It includes Python and shell scripts to:
+
+* Load raw census data as CSV files into an S3 bucket
+* Create Postgres tables from these CSV files
+* Perform any data clean up required to create a table from a CSV file
+* Run the historic data migrator
+* Run the historic workbook generator
+
+## Infrastructure changes
+
+* Create a new S3 bucket in Cloud.gov spaces as well as in the local environment
+* Create a new Postgres instance both in CG and locally
+
+## Utilities
+
+* fac_s3.py - Uploads folders or files to an S3 bucket.
+
+```bash
+python manage.py fac_s3 fac-census-to-gsafac-s3 --upload --src census_historical_migration/data
+```
+
+* csv_to_postgres.py - Inserts data into Postgres tables using the contents of the CSV files in the S3 bucket. The first row of each file is assumed to have the column names (we convert to lowercase). The name of the table is determined by examining the name of the file. The sample source files do not have delimters for empty fields at the end of a line - so we assume these are nulls.
+
+```bash
+python manage.py csv_to_postgres --folder data
+python manage.py csv_to_postgres --clean True
+```
+
+* models.py These correspond to the incoming CSV files
+* routers.py This tells django to use a different postgres instance.
+
+* data A folder that contains sample data that we can use for development.
+
+## Prerequisites
+
+* A Django app that reads the tables created here as unmanaged models and populates SF-SAC tables by creating workbooks, etc. to simulate a real submission
+
+## How to load test Census data into Postgres
+
+1.  Download test Census data from https://drive.google.com/drive/folders/1TY-7yWsMd8DsVEXvwrEe_oWW1iR2sGoy into census_historical_migration/data folder.  
+NOTE:  Never check in the census_historical_migration/data folder into GitHub.
+
+2.  In the FAC/backend folder, run the following to load CSV files from census_historical_migration/data folder into fac-census-to-gsafac-s3 bucket.
+```bash
+docker compose run web python manage.py fac_s3 fac-census-to-gsafac-s3 --upload --src census_historical_migration/data
+```
+
+3.  In the FAC/backend folder, run the following to read the CSV files from fac-census-to-gsafac-s3 bucket and load into Postgres.
+```bash
+docker compose run web python manage.py csv_to_postgres --folder data
+```
 
 ### How to run the historic data migrator:
 ```
@@ -17,4 +71,4 @@ docker compose run web python manage.py historic_workbook_generator
   --dbkey 100010
 ```
 - `year` is optional and defaults to `22`.
-- The `output` directory will be created if it doesn't already exist.
+- The `output` directory will be created if it doesn't already exist.

backend/census_historical_migration/management/commands/csv_to_postgres.py

@@ -0,0 +1,115 @@
+import logging
+import boto3
+import csv
+
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.apps import apps
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.WARNING)
+
+census_to_gsafac_models = list(
+    apps.get_app_config("census_historical_migration").get_models()
+)
+census_to_gsafac_model_names = [m._meta.model_name for m in census_to_gsafac_models]
+s3_client = boto3.client(
+    "s3",
+    aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+    aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+    endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+)
+census_to_gsafac_bucket_name = settings.AWS_CENSUS_TO_GSAFAC_BUCKET_NAME
+DELIMITER = ","
+
+
+class Command(BaseCommand):
+    help = """
+        Populate Postgres database from csv files
+        Usage:
+        manage.py csv_to_postgres --folder <folder_name> --clean <True|False>
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("--folder", help="S3 folder name")
+        parser.add_argument("--clean")
+        parser.add_argument("--sample")
+        parser.add_argument("--load")
+
+    def handle(self, *args, **options):
+        if options.get("clean") == "True":
+            self.delete_data()
+            return
+        if options.get("sample") == "True":
+            self.sample_data()
+            return
+
+        folder = options.get("folder")
+        if not folder:
+            print("Please specify a folder name")
+            return
+
+        items = s3_client.list_objects(
+            Bucket=census_to_gsafac_bucket_name,
+            Prefix=folder,
+        )["Contents"]
+        for item in items:
+            if item["Key"].endswith("/"):
+                continue
+            model_name = self.get_model_name(item["Key"])
+            if model_name:
+                model_obj = census_to_gsafac_models[
+                    census_to_gsafac_model_names.index(model_name)
+                ]
+                response = s3_client.get_object(
+                    Bucket=census_to_gsafac_bucket_name, Key=item["Key"]
+                )
+                print("Obtained response from S3")
+                lines = response["Body"].read().decode("utf-8").splitlines(True)
+                print("Loaded Body into 'lines'")
+                rows = [row for row in csv.DictReader(lines)]
+                print("Completed processing 'lines'")
+                self.load_table(model_obj, rows)
+
+        for mdl in census_to_gsafac_models:
+            row_count = mdl.objects.all().count()
+            print(f"{row_count} in ", mdl)
+
+    def delete_data(self):
+        for mdl in census_to_gsafac_models:
+            print("Deleting ", mdl)
+            mdl.objects.all().delete()
+
+    def sample_data(self):
+        for mdl in census_to_gsafac_models:
+            print("Sampling ", mdl)
+            rows = mdl.objects.all()[:1]
+            for row in rows:
+                for col in mdl._meta.fields:
+                    print(f"{col.name}: {getattr(row, col.name)}")
+
+    def get_model_name(self, name):
+        print("Processing ", name)
+        file_name = name.split("/")[-1].split(".")[0]
+        for model_name in census_to_gsafac_model_names:
+            if file_name.lower().startswith(model_name):
+                print("model_name = ", model_name)
+                return model_name
+        print("Could not find a matching model for ", name)
+        return None
+
+    def load_table(self, model_obj, rows):
+        print("Loading data for model_obj ", model_obj)
+        for i in range(0, len(rows)):
+            model_instance = model_obj()
+
+            for column_name, value in rows[i].items():
+                if column_name == "id":
+                    continue
+                setattr(model_instance, column_name, value)
+            model_instance.save()
+            if i % 1000 == 0:
+                print(f"Loaded {i} of {len(rows)} rows to ", model_obj)
+        print(f"Loaded {len(rows)} rows to ", model_obj)

backend/census_historical_migration/management/commands/fac_s3.py

@@ -0,0 +1,81 @@
+from os import path
+import os
+
+import boto3
+
+from django.core.management.base import BaseCommand
+
+from django.conf import settings
+
+
+class Command(BaseCommand):
+    help = """
+    Alternative to aws s3 as the cli is not available in production.
+    Usage:
+        manage.py fac_s3 <bucket_name> --upload --src SRC [--tgt TGT]
+        manage.py fac_s3 <bucket_name> --download --src SRC [--tgt TGT]
+        manage.py fac_s3 <bucket_name> --rm --tgt TGT]
+        manage.py fac_s3 <bucket_name> --ls [--tgt TGT]
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("bucket_name", type=str, help="The S3 bucket name.")
+        parser.add_argument("--src", help="local file name.")
+        parser.add_argument("--tgt", help="s3 file name.")
+        parser.add_argument("--ls", action="store_true", help="List all files.")
+        parser.add_argument(
+            "--upload", action="store_true", help="Copy local src to S3 tgt."
+        )
+        parser.add_argument(
+            "--download", action="store_true", help="Copy S3 tgt to local src."
+        )
+        parser.add_argument("--rm", action="store_true", help="Delete tgt.")
+
+    def handle(self, *args, **options):
+        bucket_name = options["bucket_name"]
+        src_path = options["src"]
+        tgt_path = options["tgt"]
+
+        s3_client = boto3.client(
+            "s3",
+            aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+            endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+        )
+
+        if options["ls"]:
+            items = s3_client.list_objects(
+                Bucket=bucket_name,
+                Prefix=tgt_path or "",
+            ).get("Contents")
+            if not items:
+                print("Target is empty")
+                return
+            for item in items:
+                print(item["Key"], item["Size"], item["LastModified"])
+            return
+
+        if options["upload"]:
+            file_path = path.join(settings.BASE_DIR, src_path)
+            tgt_name = tgt_path or os.path.basename(file_path)
+            tgt_name_offset = len(str(file_path))
+            for subdir, dir, files in os.walk(file_path):
+                object_name = tgt_name + str(subdir)[tgt_name_offset:] + "/"
+                print(subdir, dir, object_name, files)
+                for file in files:
+                    full_path = os.path.join(subdir, file)
+                    s3_client.upload_file(full_path, bucket_name, object_name + file)
+                    print(f"Copied {full_path} to {bucket_name} {object_name+file}.")
+            return
+
+        if options["download"]:
+            file_path = path.join(settings.BASE_DIR, src_path)
+            object_name = tgt_path
+            s3_client.download_file(bucket_name, object_name, file_path)
+            return
+
+        if options["rm"]:
+            s3_client.delete_object(
+                Bucket=bucket_name,
+                Key=tgt_path,
+            )