Approach for changing old-format report_ids to new format (#2376)

* Approach for changing old-format report_ids to new format.

* Change data_source.

* Configure Terraform's s3 backend endpoint with a URL

The old parameter has been deprecated, and there appears to be stronger requirements for a URL, not just a hostname, for the endpoint value.

* Second try at giving the Terraform S3 backend a URL endpoint

The newer argument referred to in the deprecation warning (`endpoints.s3`) is not being accepted. So we'll go back to just specifying `endpoint`, but give it a URL instead of just a hostname for the endpoint.

* Propagate S3 backend endpoint URL change to local config

This file populates the necessary Terraform config for local development (an alternative to the GHA workflow, which supplies the backend vars as secrets). This change accounts for the stricter checking on the `endpoint`, which should be a URL rather than a hostname; see the previous commit for more context.

* Provide URL for s3 backend endpoint during Terraform apply

Similar to previous commits: We have to provide a URL here, since Terraform got stricter about the value it wants for "endpoint". This commit is for the workflow used during the actual deployment.

---------

Co-authored-by: Bret Mogilefsky <[email protected]>

.github/workflows/terraform-apply-env.yml

@@ -47,7 +47,7 @@ jobs:
           backend_config: >
             access_key=${{ secrets.terraform_AWS_ACCESS_KEY_ID }},
             secret_key=${{ secrets.terraform_AWS_SECRET_ACCESS_KEY }}
-            endpoint=${{ secrets.terraform_ENDPOINT }},
+            endpoint=https://${{ secrets.terraform_ENDPOINT }},
             bucket=${{ secrets.terraform_BUCKET }},
             region=${{ secrets.terraform_REGION }},
             key=${{ env.KEY }},

.github/workflows/terraform-plan-env.yml

@@ -44,7 +44,7 @@ jobs:
           backend_config: >
             access_key=${{ secrets.terraform_AWS_ACCESS_KEY_ID }},
             secret_key=${{ secrets.terraform_AWS_SECRET_ACCESS_KEY }}
-            endpoint=${{ secrets.terraform_ENDPOINT }},
+            endpoint=https://${{ secrets.terraform_ENDPOINT }},
             bucket=${{ secrets.terraform_BUCKET }},
             region=${{ secrets.terraform_REGION }},
             key=${{ env.KEY }},

backend/.profile

@@ -58,6 +58,8 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
     echo 'Starting seed_cog_baseline' &&
     python manage.py seed_cog_baseline &&
     echo 'Finished seed_cog_baseline'
+    python manage.py update_oldformat_reportids &&
+    echo 'Finished rewriting old-format report_id values'
 fi
 
 # Make psql usable by scripts, for debugging, etc.

backend/audit/management/commands/update_oldformat_reportids.py

@@ -0,0 +1,73 @@
+"""
+update_oldformat_report_ids command
+
+One-off idempotent command to take SAC instances that have the original report_id format
+(<startyear><startmonth3charabbr>GSA<count + 1000000>) and update their report_id
+to the new format (<endyear><endmonth2digit>GSAFAC<count>).
+"""
+import calendar
+import logging
+import time
+from pathlib import Path
+
+from django.core.management.base import BaseCommand
+
+from audit.models import SingleAuditChecklist
+
+logger = logging.getLogger(__name__)
+
+
+def get_newformat_report_id(sac):
+    """
+    Given a SAC instance, return the new-format report_id.
+    """
+    old_report_id = sac.report_id
+    end_date = sac.general_information["auditee_fiscal_period_end"]
+    end_year, end_month, _day = end_date.split("-")
+    count = old_report_id[-10:]
+    new_count = int(count) - 1000000 if int(count) > 1000000 else int(count)
+    return "-".join((end_year, end_month, "GSAFAC", str(new_count).zfill(10)))
+
+
+def get_oldformat_report_id(sac):
+    """
+    Given a SAC instance, return the old-format report_id.
+
+    Works, but needs to be run using a record of what SAC instances were changed in
+    order to truly reverse the operation. Such a list could be constructed out of
+    the file saved in handle() below.
+    """
+    fiscal_start = sac.general_information["auditee_fiscal_period_start"]
+    year = fiscal_start[:4]
+    month = calendar.month_abbr[int(fiscal_start[5:7])].upper()
+    count = sac.report_id[-10:]
+    oldcount = int(count) + 1000000
+    return f"{year}{month}{str(oldcount).zfill(10)}"
+
+
+class Command(BaseCommand):
+    """Django management command to change report_id format."""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--reverse",
+            action="store_true",
+            help="Switch to old-format report_ids instead.",
+        )
+
+    def handle(self, *args, **options):
+        changesfile = Path(__file__).parent / f"{time.time()}-changed_reportids.txt"
+        lines = []
+        change_sacs = SingleAuditChecklist.objects.exclude(report_id__contains="GSAFAC")
+
+        for sac in change_sacs:
+            old_rid = str(sac.report_id)
+            new_rid = get_newformat_report_id(sac)
+            sac.report_id = new_rid
+            sac.data_source = "GSAFAC"
+            sac.save(undocumentedoverride="HACKZ")
+            outstring = f"id: {sac.id} old: {old_rid} new: {new_rid}"
+            lines.append(outstring)
+
+        contents = "\n".join(lines)
+        changesfile.write_text(contents, encoding="utf-8")

backend/audit/models.py

@@ -173,20 +173,21 @@ def save(self, *args, **kwargs):
         in progress isn't being altered; skip this if we know this submission is
         in progress.
         """
-        if self.submission_status != self.STATUS.IN_PROGRESS:
-            try:
-                self._reject_late_changes()
-            except LateChangeError as err:
-                raise LateChangeError from err
-
-        event_user = kwargs.get("event_user")
-        event_type = kwargs.get("event_type")
-        if event_user and event_type:
-            SubmissionEvent.objects.create(
-                sac=self,
-                user=event_user,
-                event=event_type,
-            )
+        if not kwargs.get("undocumentedoverride") == "HACKZ":
+            if self.submission_status != self.STATUS.IN_PROGRESS:
+                try:
+                    self._reject_late_changes()
+                except LateChangeError as err:
+                    raise LateChangeError from err
+
+            event_user = kwargs.get("event_user")
+            event_type = kwargs.get("event_type")
+            if event_user and event_type:
+                SubmissionEvent.objects.create(
+                    sac=self,
+                    user=event_user,
+                    event=event_type,
+                )
 
         return super().save()
 

terraform/meta/bootstrap/main.tf

@@ -35,7 +35,7 @@ resource "local_file" "backend-tfvars" {
   access_key  = "${local.credentials.access_key_id}"
   secret_key  = "${local.credentials.secret_access_key}"
   bucket      = "${local.credentials.bucket}"
-  endpoint    = "${local.credentials.fips_endpoint}"
+  endpoint    = "https://${local.credentials.fips_endpoint}"
   region      = "${local.credentials.region}"
   EOF
 }