Merge pull request #3104 from GSA-TTS/main

.github/workflows/create-release.yml

@@ -37,4 +37,4 @@ jobs:
           body: |
               ${{ steps.changelog.outputs.changelog }}
           draft: false
-          prerelease: true
+          prerelease: false

backend/Dockerfile

@@ -1,72 +1,76 @@
-FROM python:3.10-slim AS builder
+###############################
+# FOUNDATION
+###############################
+
+FROM python:3.10-slim AS foundation
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-RUN apt-get -y update
-RUN apt-get -y install git
-
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive \
-    apt-get install --no-install-recommends --assume-yes \
-      postgresql-client
-
-# RUN --mount=type=cache,target=/root/.cache \
-
-WORKDIR ..
-
-RUN \
-    apt-get update && \
-    apt-get install -yqq apt-transport-https wget gnupg2
-
-# Updated nodesource install via: https://github.com/nodesource/distributions#ubuntu-versions
-RUN \
-    apt-get update -yq && \
-    apt install build-essential curl -y && \
-    apt-get install -y gcc ca-certificates gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-    NODE_MAJOR=18 && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install nodejs -y
+RUN apt-get -yq update && \
+    apt install -y \
+        apt-transport-https \ 
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        gcc \
+        gnupg \
+        gnupg2 \
+        postgresql-client \
+        wget 
+
+###############################
+# STORAGE
+###############################
+FROM foundation AS storage
+RUN apt-get install -yqq groff && \
+    apt-get install -yqq zip && \
+    curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" && \
+    unzip awscli-bundle.zip && \
+    ./awscli-bundle/install -b /usr/local/bin/aws
+RUN curl "https://dl.min.io/client/mc/release/linux-amd64/mc" -o "/usr/local/bin/mc" && \
+    chmod +x /usr/local/bin/mc
 
+###############################
+# PIP
+###############################
+FROM storage AS pip
 COPY requirements.txt /tmp/requirements.txt
 COPY dev-requirements.txt /tmp/dev-requirements.txt
-
-RUN \
-    set -ex && \
+RUN set -ex && \
     pip install --upgrade pip && \
     pip install -r /tmp/requirements.txt && \
     pip install -r /tmp/dev-requirements.txt && \
     rm -rf /root/.cache/
 
-RUN \
-    apt-get install -yqq groff && \
-    apt-get install -yqq zip && \
-    curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" && \
-    unzip awscli-bundle.zip && \
-    ./awscli-bundle/install -b /usr/local/bin/aws
-
-RUN \
-    curl "https://dl.min.io/client/mc/release/linux-amd64/mc" -o "/usr/local/bin/mc" && \
-    chmod +x /usr/local/bin/mc
-
-COPY package.* /src/
+###############################
+# NODE
+###############################
+FROM pip AS node
 WORKDIR /src/
-
+ENV NODE_MAJOR 18 
+
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+# We have to update after adding to the apt sources lists.
+RUN apt-get update
+RUN apt-get install -y nodejs npm 
+COPY package.* /src/
+RUN npm install -g [email protected]
+# To address run-p error.
+RUN npm install -g npm-run-all
 RUN npm install
 RUN chown -R 1001:123 "/root/.npm"
 
 COPY . /src/
 
-RUN npm run build && python manage.py collectstatic
-
-# ------------------------------------------------------------
-# Dev/testing layer
-# ------------------------------------------------------------
-
-FROM builder AS release
+RUN npm run build 
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+###############################
+# DEV
+###############################
+FROM node AS dev
+WORKDIR /src/
+RUN python manage.py collectstatic

backend/audit/cross_validation/errors.py

@@ -32,8 +32,8 @@ def err_awards_findings_but_no_findings_text():
 
 def err_missing_tribal_data_sharing_consent():
     return (
-        "As a tribal organization, you must complete the data "
-        "sharing consent statement before submitting your audit."
+        "As a tribal organization, the Auditee Certifying Official must complete "
+        "the data sharing consent statement before submitting your audit."
     )
 
 

backend/audit/intakelib/checks/check_loan_balance_entries.py

@@ -6,6 +6,8 @@
     get_range_by_name,
 )
 from audit.intakelib.common import get_message, build_cell_error_tuple
+from ..common.util import is_int
+from .check_cluster_total import NOT_APPLICABLE
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +23,8 @@ def loan_balance_entry_is_valid(ir):
     for index, balance in enumerate(loan_balance_at_period_end):
         # Check if balance is not a number, empty string, or "N/A"
         if not (
-            balance in ["N/A", "", settings.GSA_MIGRATION, None] or _is_int(balance)
+            balance in [NOT_APPLICABLE, "", settings.GSA_MIGRATION, None]
+            or is_int(balance)
         ):
             errors.append(
                 build_cell_error_tuple(
@@ -33,12 +36,3 @@ def loan_balance_entry_is_valid(ir):
             )
 
     return errors
-
-
-# check if the given string can be converted to an int
-def _is_int(s):
-    try:
-        int(s)
-        return True
-    except ValueError:
-        return False

backend/audit/intakelib/checks/check_loan_balance_present.py

@@ -1,9 +1,12 @@
 import logging
+from django.conf import settings
 from audit.intakelib.intermediate_representation import (
     get_range_values_by_name,
     get_range_by_name,
 )
 from audit.intakelib.common import get_message, build_cell_error_tuple
+from ..common.util import is_int
+from .check_cluster_total import NOT_APPLICABLE
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +35,9 @@ def loan_balance_present(ir):
                     get_message("check_loan_guarantee_empty_when_n"),
                 )
             )
-        elif (guarantee == "Y") and not balance:
+        elif (guarantee == "Y") and not (
+            balance in [NOT_APPLICABLE, settings.GSA_MIGRATION] or is_int(balance)
+        ):
             errors.append(
                 build_cell_error_tuple(
                     ir,

backend/audit/intakelib/checks/runners.py

@@ -6,6 +6,7 @@
 from audit.fixtures.excel import FORM_SECTIONS
 from .check_gsa_migration_keyword import check_for_gsa_migration_keyword
 
+
 ############
 # General checks
 from .check_uei_exists import uei_exists
@@ -43,7 +44,6 @@
 from .check_has_all_the_named_ranges import has_all_the_named_ranges
 from .check_missing_required_fields import has_all_required_fields
 from .check_y_or_n__fields import has_invalid_yorn_field
-from .check_show_ir import show_ir
 
 ############
 # Audit findings checks
@@ -68,6 +68,7 @@
     has_all_required_fields(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
     has_invalid_yorn_field(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
     federal_award_amount_passed_through_optional,
+    check_cluster_names,
     state_cluster_names,
     other_cluster_names,
     passthrough_name_when_no_direct,
@@ -137,18 +138,23 @@
 
 
 def run_all_checks(ir, list_of_checks, section_name=None, is_data_migration=False):
-    show_ir
     errors = []
     if section_name:
         res = is_right_workbook(section_name)(ir)
         if res:
             errors.append(res)
 
-    if not is_data_migration:
+    # If this is a data migration, then there are some checks we do not want to run.
+    #
+    if is_data_migration:
+        if list_of_checks == federal_awards_checks:
+            list_of_checks = list(
+                filter(lambda f: f != check_cluster_names, list_of_checks)
+            )
+    else:
+        # This is not a data migration.
+        # We want to make sure no one put in a GSA_MIGRATION keyword.
         check_for_gsa_migration_keyword(ir)
-        res = check_cluster_names(ir)
-        if res:
-            errors.append(res)
 
     for fun in list_of_checks:
         res = fun(ir)
@@ -165,7 +171,9 @@ def run_all_checks(ir, list_of_checks, section_name=None, is_data_migration=Fals
 
 
 def run_all_general_checks(ir, section_name, is_data_migration=False):
-    run_all_checks(ir, general_checks, section_name, is_data_migration)
+    run_all_checks(
+        ir, general_checks, section_name, is_data_migration=is_data_migration
+    )
 
 
 def run_all_federal_awards_checks(ir, is_data_migration=False):

backend/audit/intakelib/common/__init__.py

@@ -1,14 +1,15 @@
-# flake8: noqa: F401
-from .util import (
-    get_message,
-    build_cell_error_tuple,
-    appears_empty,
-    is_value_marked_na,
-    get_names_of_all_ranges,
-    get_range_start_row,
-    get_missing_value_errors,
-    invalid_y_or_n_entry,
-    safe_int_conversion,
-    list_contains_non_null_values,
-    build_range_error_tuple,
-)
+# flake8: noqa: F401
+from .util import (
+    get_message,
+    build_cell_error_tuple,
+    appears_empty,
+    is_value_marked_na,
+    get_names_of_all_ranges,
+    get_range_start_row,
+    get_missing_value_errors,
+    invalid_y_or_n_entry,
+    make_named_range_uppercase,
+    safe_int_conversion,
+    list_contains_non_null_values,
+    build_range_error_tuple,
+)

backend/audit/intakelib/common/util.py

@@ -122,6 +122,25 @@ def get_names_of_all_ranges(data):
     return names
 
 
+def make_named_range_uppercase(ir, range_name, message_key):
+    range_data = get_range_by_name(ir, range_name)
+    errors = []
+    new_values = []
+    if range_data:
+        for index, value in enumerate(range_data["values"]):
+            try:
+                new_values.append(value.upper())
+            except (ValueError, TypeError, AttributeError):
+                logger.info(f"Could not uppercase range {range_name}, {value}")
+                message = get_message(message_key).format(value)
+                error = build_cell_error_tuple(ir, range_data, index, message)
+                errors.append(error)
+        if len(errors) > 0:
+            logger.info("Raising a validation error.")
+            raise ValidationError(errors)
+    return replace_range_by_name(ir, range_name, new_values)
+
+
 def safe_int_conversion(ir, range_name, other_values_allowed=None):
     range_data = get_range_by_name(ir, range_name)
     errors = []
@@ -155,3 +174,12 @@ def safe_int_conversion(ir, range_name, other_values_allowed=None):
             raise ValidationError(errors)
     new_ir = replace_range_by_name(ir, range_name, new_values)
     return new_ir
+
+
+# check if the given string can be converted to an int
+def is_int(s):
+    try:
+        int(s)
+        return True
+    except (ValueError, TypeError):
+        return False

backend/audit/intakelib/transforms/runners.py

@@ -10,6 +10,9 @@
 from .xform_all_federal_program_total_need_to_be_integers import (
     convert_federal_program_total_to_integers,
 )
+from .xform_federal_awards_cluster_name_to_uppercase import (
+    convert_federal_awards_cluster_name_to_uppercase,
+)
 from .xform_total_amount_expended_need_to_be_integers import (
     convert_total_amount_expended_to_integers,
 )
@@ -105,6 +108,7 @@ def run_all_secondary_auditors_transforms(ir):
     convert_total_amount_expended_to_integers,
     convert_number_of_findings_to_integers,
     convert_loan_balance_to_integers_or_na,
+    convert_federal_awards_cluster_name_to_uppercase,
     regenerate_uniform_cluster_names,
     reformat_federal_agency_prefix,
     generate_cfda_keys,

backend/audit/intakelib/transforms/xform_federal_awards_cluster_name_to_uppercase.py

@@ -0,0 +1,10 @@
+import logging
+from audit.intakelib.common import make_named_range_uppercase
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert federal_awards cluster_name to uppercase.
+def convert_federal_awards_cluster_name_to_uppercase(ir):
+    return make_named_range_uppercase(ir, "cluster_name", "check_cluster_names")

backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py

@@ -1,11 +1,16 @@
 import logging
+
+from django.conf import settings
 from audit.intakelib.common import safe_int_conversion
+from ..checks.check_cluster_total import NOT_APPLICABLE
 
 logger = logging.getLogger(__name__)
 
 
 # DESCRIPTION
 # Convert end of period loan balance to integers when applicable
 def convert_loan_balance_to_integers_or_na(ir):
-    xform_ir = safe_int_conversion(ir, "loan_balance_at_audit_period_end", {"N/A"})
+    xform_ir = safe_int_conversion(
+        ir, "loan_balance_at_audit_period_end", {NOT_APPLICABLE, settings.GSA_MIGRATION}
+    )
     return xform_ir