From 5f6d26539b3e872146dbac95e6a3d94ffd353614 Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:01:52 -0700 Subject: [PATCH] Approach for changing old-format report_ids to new format (#2376) * Approach for changing old-format report_ids to new format. * Change data_source. * Configure Terraform's s3 backend endpoint with a URL The old parameter has been deprecated, and there appears to be stronger requirements for a URL, not just a hostname, for the endpoint value. * Second try at giving the Terraform S3 backend a URL endpoint The newer argument referred to in the deprecation warning (`endpoints.s3`) is not being accepted. So we'll go back to just specifying `endpoint`, but give it a URL instead of just a hostname for the endpoint. * Propagate S3 backend endpoint URL change to local config This file populates the necessary Terraform config for local development (an alternative to the GHA workflow, which supplies the backend vars as secrets). This change accounts for the stricter checking on the `endpoint`, which should be a URL rather than a hostname; see the previous commit for more context. * Provide URL for s3 backend endpoint during Terraform apply Similar to previous commits: We have to provide a URL here, since Terraform got stricter about the value it wants for "endpoint". This commit is for the workflow used during the actual deployment. --------- Co-authored-by: Bret Mogilefsky --- .github/workflows/terraform-apply-env.yml | 2 +- .github/workflows/terraform-plan-env.yml | 2 +- backend/.profile | 2 + .../commands/update_oldformat_reportids.py | 73 +++++++++++++++++++ backend/audit/models.py | 29 ++++---- terraform/meta/bootstrap/main.tf | 2 +- 6 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 backend/audit/management/commands/update_oldformat_reportids.py diff --git a/.github/workflows/terraform-apply-env.yml b/.github/workflows/terraform-apply-env.yml index fd833bb103..96452a3c63 100644 --- a/.github/workflows/terraform-apply-env.yml +++ b/.github/workflows/terraform-apply-env.yml @@ -47,7 +47,7 @@ jobs: backend_config: > access_key=${{ secrets.terraform_AWS_ACCESS_KEY_ID }}, secret_key=${{ secrets.terraform_AWS_SECRET_ACCESS_KEY }} - endpoint=${{ secrets.terraform_ENDPOINT }}, + endpoint=https://${{ secrets.terraform_ENDPOINT }}, bucket=${{ secrets.terraform_BUCKET }}, region=${{ secrets.terraform_REGION }}, key=${{ env.KEY }}, diff --git a/.github/workflows/terraform-plan-env.yml b/.github/workflows/terraform-plan-env.yml index 4c9e48c1e9..3697db7dae 100644 --- a/.github/workflows/terraform-plan-env.yml +++ b/.github/workflows/terraform-plan-env.yml @@ -44,7 +44,7 @@ jobs: backend_config: > access_key=${{ secrets.terraform_AWS_ACCESS_KEY_ID }}, secret_key=${{ secrets.terraform_AWS_SECRET_ACCESS_KEY }} - endpoint=${{ secrets.terraform_ENDPOINT }}, + endpoint=https://${{ secrets.terraform_ENDPOINT }}, bucket=${{ secrets.terraform_BUCKET }}, region=${{ secrets.terraform_REGION }}, key=${{ env.KEY }}, diff --git a/backend/.profile b/backend/.profile index a4341d62e2..69d44e0ca1 100644 --- a/backend/.profile +++ b/backend/.profile @@ -58,6 +58,8 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then echo 'Starting seed_cog_baseline' && python manage.py seed_cog_baseline && echo 'Finished seed_cog_baseline' + python manage.py update_oldformat_reportids && + echo 'Finished rewriting old-format report_id values' fi # Make psql usable by scripts, for debugging, etc. diff --git a/backend/audit/management/commands/update_oldformat_reportids.py b/backend/audit/management/commands/update_oldformat_reportids.py new file mode 100644 index 0000000000..e219a644cf --- /dev/null +++ b/backend/audit/management/commands/update_oldformat_reportids.py @@ -0,0 +1,73 @@ +""" +update_oldformat_report_ids command + +One-off idempotent command to take SAC instances that have the original report_id format +(GSA) and update their report_id +to the new format (GSAFAC). +""" +import calendar +import logging +import time +from pathlib import Path + +from django.core.management.base import BaseCommand + +from audit.models import SingleAuditChecklist + +logger = logging.getLogger(__name__) + + +def get_newformat_report_id(sac): + """ + Given a SAC instance, return the new-format report_id. + """ + old_report_id = sac.report_id + end_date = sac.general_information["auditee_fiscal_period_end"] + end_year, end_month, _day = end_date.split("-") + count = old_report_id[-10:] + new_count = int(count) - 1000000 if int(count) > 1000000 else int(count) + return "-".join((end_year, end_month, "GSAFAC", str(new_count).zfill(10))) + + +def get_oldformat_report_id(sac): + """ + Given a SAC instance, return the old-format report_id. + + Works, but needs to be run using a record of what SAC instances were changed in + order to truly reverse the operation. Such a list could be constructed out of + the file saved in handle() below. + """ + fiscal_start = sac.general_information["auditee_fiscal_period_start"] + year = fiscal_start[:4] + month = calendar.month_abbr[int(fiscal_start[5:7])].upper() + count = sac.report_id[-10:] + oldcount = int(count) + 1000000 + return f"{year}{month}{str(oldcount).zfill(10)}" + + +class Command(BaseCommand): + """Django management command to change report_id format.""" + + def add_arguments(self, parser): + parser.add_argument( + "--reverse", + action="store_true", + help="Switch to old-format report_ids instead.", + ) + + def handle(self, *args, **options): + changesfile = Path(__file__).parent / f"{time.time()}-changed_reportids.txt" + lines = [] + change_sacs = SingleAuditChecklist.objects.exclude(report_id__contains="GSAFAC") + + for sac in change_sacs: + old_rid = str(sac.report_id) + new_rid = get_newformat_report_id(sac) + sac.report_id = new_rid + sac.data_source = "GSAFAC" + sac.save(undocumentedoverride="HACKZ") + outstring = f"id: {sac.id} old: {old_rid} new: {new_rid}" + lines.append(outstring) + + contents = "\n".join(lines) + changesfile.write_text(contents, encoding="utf-8") diff --git a/backend/audit/models.py b/backend/audit/models.py index 88019b1bac..0a0f2e84e1 100644 --- a/backend/audit/models.py +++ b/backend/audit/models.py @@ -173,20 +173,21 @@ def save(self, *args, **kwargs): in progress isn't being altered; skip this if we know this submission is in progress. """ - if self.submission_status != self.STATUS.IN_PROGRESS: - try: - self._reject_late_changes() - except LateChangeError as err: - raise LateChangeError from err - - event_user = kwargs.get("event_user") - event_type = kwargs.get("event_type") - if event_user and event_type: - SubmissionEvent.objects.create( - sac=self, - user=event_user, - event=event_type, - ) + if not kwargs.get("undocumentedoverride") == "HACKZ": + if self.submission_status != self.STATUS.IN_PROGRESS: + try: + self._reject_late_changes() + except LateChangeError as err: + raise LateChangeError from err + + event_user = kwargs.get("event_user") + event_type = kwargs.get("event_type") + if event_user and event_type: + SubmissionEvent.objects.create( + sac=self, + user=event_user, + event=event_type, + ) return super().save() diff --git a/terraform/meta/bootstrap/main.tf b/terraform/meta/bootstrap/main.tf index 0235f55d0e..552d2fbc36 100644 --- a/terraform/meta/bootstrap/main.tf +++ b/terraform/meta/bootstrap/main.tf @@ -35,7 +35,7 @@ resource "local_file" "backend-tfvars" { access_key = "${local.credentials.access_key_id}" secret_key = "${local.credentials.secret_access_key}" bucket = "${local.credentials.bucket}" - endpoint = "${local.credentials.fips_endpoint}" + endpoint = "https://${local.credentials.fips_endpoint}" region = "${local.credentials.region}" EOF }