diff --git a/backend/census_historical_migration/README.md b/backend/census_historical_migration/README.md new file mode 100644 index 0000000000..209400fa09 --- /dev/null +++ b/backend/census_historical_migration/README.md @@ -0,0 +1,20 @@ +# Census Historical Migration + +### How to run the historic data migrator: +``` +docker compose run web python manage.py historic_data_migrator --email any_email_in_the_system@woo.gov \ + --year 22 \ + --dbkey 100010 +``` +- The email address currently must be a User in the system. As this has only been run locally so far, it would often be a test account in my local sandbox env. +- `year` and `dbkey` are optional. The script will use default values for these if they aren't provided. + +### How to run the historic workbook generator: +``` +docker compose run web python manage.py historic_workbook_generator + --year 22 \ + --output \ + --dbkey 100010 +``` +- `year` is optional and defaults to `22`. +- The `output` directory will be created if it doesn't already exist. diff --git a/backend/census_historical_migration/__init__.py b/backend/census_historical_migration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/census_historical_migration/admin.py b/backend/census_historical_migration/admin.py new file mode 100644 index 0000000000..8f217f70e8 --- /dev/null +++ b/backend/census_historical_migration/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin # noqa: F401 + +# Register your models here. diff --git a/backend/census_historical_migration/apps.py b/backend/census_historical_migration/apps.py new file mode 100644 index 0000000000..b36afe4117 --- /dev/null +++ b/backend/census_historical_migration/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CensusHistoricalMigrationConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "census_historical_migration" diff --git a/backend/census_historical_migration/management/commands/historic_data_migrator.py b/backend/census_historical_migration/management/commands/historic_data_migrator.py new file mode 100644 index 0000000000..352d9b348c --- /dev/null +++ b/backend/census_historical_migration/management/commands/historic_data_migrator.py @@ -0,0 +1,61 @@ +import os +import logging +import sys + +from config.settings import ENVIRONMENT +from django.core.management.base import BaseCommand +from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end + +CYPRESS_TEST_EMAIL_ADDR = os.getenv("CYPRESS_LOGIN_TEST_EMAIL_AUDITEE") +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--email", type=str, required=False) + parser.add_argument("--dbkeys", type=str, required=False, default="") + parser.add_argument("--years", type=str, required=False, default="") + + def handle(self, *args, **options): + dbkeys_str = options["dbkeys"] + years_str = options["years"] + dbkeys = dbkeys_str.split(",") + years = years_str.split(",") + + if len(dbkeys) != len(years): + logger.error( + "Received {} dbkeys and {} years. Must be equal. Exiting.".format( + len(dbkeys), len(years) + ) + ) + sys.exit(-1) + + lengths = [len(s) == 2 for s in years] + if dbkeys_str and years_str and (not all(lengths)): + logger.error("Years must be two digits. Exiting.") + sys.exit(-2) + + email = options.get("email", CYPRESS_TEST_EMAIL_ADDR) + + defaults = [ + (182926, 22), + (181744, 22), + (191734, 22), + ] + + if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]: + if dbkeys_str and years_str: + logger.info( + f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" + ) + for dbkey, year in zip(dbkeys, years): + run_end_to_end(email, dbkey, year) + else: + for pair in defaults: + logger.info("Running {}-{} end-to-end".format(pair[0], pair[1])) + run_end_to_end(email, str(pair[0]), str(pair[1])) + else: + logger.error( + "Cannot run end-to-end workbook generation in production. Exiting." + ) + sys.exit(-3) diff --git a/backend/census_historical_migration/management/commands/historic_workbook_generator.py b/backend/census_historical_migration/management/commands/historic_workbook_generator.py new file mode 100644 index 0000000000..0debe8bd8d --- /dev/null +++ b/backend/census_historical_migration/management/commands/historic_workbook_generator.py @@ -0,0 +1,217 @@ +from collections import namedtuple as NT +from playhouse.shortcuts import model_to_dict +import os +import sys +import json + +from django.core.management.base import BaseCommand + +import argparse +import pprint + +from census_historical_migration.workbooklib.workbook_creation import ( + sections, + workbook_loader, + setup_sac, +) + +import datetime + +from census_historical_migration.workbooklib.census_models.census import ( + CensusGen22 as Gen, +) + +import logging + +pp = pprint.PrettyPrinter(indent=2) + +parser = argparse.ArgumentParser() + +logger = logging.getLogger(__name__) +logging.basicConfig() +logging.getLogger().setLevel(logging.INFO) + +# This provides a way to map the sheet in the workbook to the +# column in the DB. It also has a default value and +# the type of value, so that things can be set correctly +# before filling in the XLSX workbooks. +FieldMap = NT("FieldMap", "in_sheet in_db default type") + +templates = { + "AdditionalUEIs": "additional-ueis-workbook.xlsx", + "AdditionalEINs": "additional-eins-workbook.xlsx", + "AuditFindingsText": "audit-findings-text-workbook.xlsx", + "CAP": "corrective-action-plan-workbook.xlsx", + "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", + "FederalAwards": "federal-awards-workbook.xlsx", + "SEFA": "notes-to-sefa-workbook.xlsx", + "SecondaryAuditors": "secondary-auditors-workbook.xlsx", +} + + +def set_single_cell_range(wb, range_name, value): + the_range = wb.defined_names[range_name] + # The above returns a generator. Turn it to a list, and grab + # the first element of the list. Now, this *tuple* contains a + # sheet name and a cell reference... which you need to get rid + # of the '$' to use. + # https://itecnote.com/tecnote/python-using-excel-named-ranges-in-python-with-openpyxl/ + tup = list(the_range.destinations)[0] + sheet_title = tup[0] + cell_ref = tup[1].replace("$", "") + ws = wb[sheet_title] + ws[cell_ref] = value + + +# A tiny helper to index into workbooks. +# Assumes a capital letter. +def col_to_ndx(col): + return ord(col) - 65 + 1 + + +# Helper to set a range of values. +# Takes a named range, and then walks down the range, +# filling in values from the list past in (values). +def set_range(wb, range_name, values, default=None, type=str): + the_range = wb.defined_names[range_name] + dest = list(the_range.destinations)[0] + sheet_title = dest[0] + ws = wb[sheet_title] + + start_cell = dest[1].replace("$", "").split(":")[0] + col = col_to_ndx(start_cell[0]) + start_row = int(start_cell[1]) + + for ndx, v in enumerate(values): + row = ndx + start_row + if v: + # This is a very noisy statement, showing everything + # written into the workbook. + # print(f'{range_name} c[{row}][{col}] <- {v} len({len(v)}) {default}') + if v is not None: + ws.cell(row=row, column=col, value=type(v)) + if len(v) == 0 and default is not None: + # This is less noisy. Shows up for things like + # empty findings counts. 2023 submissions + # require that field to be 0, not empty, + # if there are no findings. + # print('Applying default') + ws.cell(row=row, column=col, value=type(default)) + if not v: + if default is not None: + ws.cell(row=row, column=col, value=type(default)) + else: + ws.cell(row=row, column=col, value="") + else: + # Leave it blank if we have no default passed in + pass + + +def set_uei(wb, dbkey): + g = Gen.select().where(Gen.dbkey == dbkey).get() + set_single_cell_range(wb, "auditee_uei", g.uei) + return g + + +def map_simple_columns(wb, mappings, values): + # Map all the simple ones + for m in mappings: + set_range( + wb, + m.in_sheet, + map(lambda v: model_to_dict(v)[m.in_db], values), + m.default, + m.type, + ) + + +# FIXME: Get the padding/shape right on the report_id +def dbkey_to_test_report_id(dbkey): + g = Gen.select(Gen.audityear, Gen.fyenddate).where(Gen.dbkey == dbkey).get() + # month = g.fyenddate.split('-')[1] + # 2022JUN0001000003 + # We start new audits at 1 million. + # So, we want 10 digits, and zero-pad for + # historic DBKEY report_ids + return f"{g.audityear}-TEST-{dbkey.zfill(7)}" + + +def generate_dissemination_test_table(api_endpoint, dbkey, mappings, objects): + table = {"rows": list(), "singletons": dict()} + table["endpoint"] = api_endpoint + table["report_id"] = dbkey_to_test_report_id(dbkey) + for o in objects: + as_dict = model_to_dict(o) + test_obj = {} + test_obj["fields"] = [] + test_obj["values"] = [] + for m in mappings: + # What if we only test non-null values? + if ((m.in_db in as_dict) and as_dict[m.in_db] is not None) and ( + as_dict[m.in_db] != "" + ): + test_obj["fields"].append(m.in_sheet) + test_obj["values"].append(as_dict[m.in_db]) + table["rows"].append(test_obj) + return table + + +def make_file(dir, dbkey, slug): + return open(os.path.join(dir, f"{slug}-{dbkey}.xlsx")) + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--output", type=str, required=True) + parser.add_argument("--dbkey", type=str, required=True) + parser.add_argument("--year", type=str, default="22") + + def handle(self, *args, **options): # noqa: C901 + out_basedir = None + if options["output"]: + out_basedir = options["output"] + else: + out_basedir = "output" + + if not os.path.exists(out_basedir): + try: + os.mkdir(out_basedir) + logger.info(f"Made directory {out_basedir}") + except Exception as e: + logger.info(e) + logger.info(f"Could not create directory {out_basedir}") + sys.exit() + + outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{options["year"]}') + + if not os.path.exists(outdir): + try: + os.mkdir(outdir) + logger.info(f"Made directory {outdir}") + except Exception as e: + logger.info(e) + logger.info("could not create output directory. exiting.") + sys.exit() + + entity_id = "DBKEY {dbkey} {date:%Y_%m_%d_%H_%M_%S}".format( + dbkey=options["dbkey"], date=datetime.datetime.now() + ) + + sac = setup_sac(None, entity_id, options["dbkey"]) + loader = workbook_loader( + None, sac, options["dbkey"], options["year"], entity_id + ) + json_test_tables = [] + for section, fun in sections.items(): + (wb, api_json, filename) = loader(fun, section) + if wb: + wb_path = os.path.join(outdir, filename) + wb.save(wb_path) + if api_json: + json_test_tables.append(api_json) + + json_path = os.path.join(outdir, f'test-array-{options["dbkey"]}.json') + logger.info(f"Writing JSON to {json_path}") + with open(json_path, "w") as test_file: + jstr = json.dumps(json_test_tables, indent=2, sort_keys=True) + test_file.write(jstr) diff --git a/backend/census_historical_migration/migrations/__init__.py b/backend/census_historical_migration/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/census_historical_migration/models.py b/backend/census_historical_migration/models.py new file mode 100644 index 0000000000..af3844168d --- /dev/null +++ b/backend/census_historical_migration/models.py @@ -0,0 +1,3 @@ +from django.db import models # noqa: F401 + +# Create your models here. diff --git a/backend/census_historical_migration/tests.py b/backend/census_historical_migration/tests.py new file mode 100644 index 0000000000..47d5f1a54d --- /dev/null +++ b/backend/census_historical_migration/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase # noqa: F401 + +# Create your tests here. diff --git a/backend/census_historical_migration/views.py b/backend/census_historical_migration/views.py new file mode 100644 index 0000000000..9713e9c601 --- /dev/null +++ b/backend/census_historical_migration/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render # noqa: F401 + +# Create your views here. diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py new file mode 100644 index 0000000000..1f47ce7ed8 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -0,0 +1,44 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + WorkbookFieldInDissem, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, +) + + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + +mappings = [ + FieldMap("additional_ein", "ein", WorkbookFieldInDissem, None, str), +] + + +def generate_additional_eins(dbkey, year, outfile): + logger.info(f"--- generate additional eins {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Eins = dynamic_import("Eins", year) + wb = pyxl.load_workbook(templates["AdditionalEINs"]) + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "additional-eins-workbook") + + addl_eins = Eins.select().where(Eins.dbkey == g.dbkey) + map_simple_columns(wb, mappings, addl_eins) + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "additional_eins", dbkey, mappings, addl_eins + ) + table["singletons"]["auditee_uei"] = g.uei + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py new file mode 100644 index 0000000000..b662d2c9ae --- /dev/null +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -0,0 +1,47 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + WorkbookFieldInDissem, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, +) + + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + +mappings = [ + # FIXME: We have no dissemination nodel for this. + FieldMap("additional_uei", "uei", WorkbookFieldInDissem, None, str), +] + + +def generate_additional_ueis(dbkey, year, outfile): + logger.info(f"--- generate additional ueis {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + wb = pyxl.load_workbook(templates["AdditionalUEIs"]) + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "additional-ueis-workbook") + if int(year) >= 22: + Ueis = dynamic_import("Ueis", year) + addl_ueis = Ueis.select().where(Ueis.dbkey == g.dbkey) + map_simple_columns(wb, mappings, addl_ueis) + + table = generate_dissemination_test_table( + Gen, "additional_ueis", dbkey, mappings, addl_ueis + ) + else: + table = {} + table["singletons"] = {} + wb.save(outfile) + table["singletons"]["auditee_uei"] = g.uei + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/census_models/census.py b/backend/census_historical_migration/workbooklib/census_models/census.py new file mode 100644 index 0000000000..aef030d8a4 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/census_models/census.py @@ -0,0 +1,1978 @@ +from peewee import ( + Model, + TextField, + BigIntegerField, +) +from playhouse.postgres_ext import PostgresqlDatabase + +# FIXME: pull this from the config +database = PostgresqlDatabase("postgres", **{"host": "db", "user": "postgres"}) + + +def model_module_path(model, year): + return f"census_historical_migration.workbooklib.census_models.census.Census{model}{year}" + + +def dynamic_import(mod, year): + name = model_module_path(mod, year) + components = name.split(".") + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + return mod + + +class UnknownField(object): + def __init__(self, *_, **__): + pass + + +class BaseModel(Model): + class Meta: + database = database + + +class CensusAgency16(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency16" + schema = "public" + primary_key = False + + +class CensusAgency17(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency17" + schema = "public" + primary_key = False + + +class CensusAgency18(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency18" + schema = "public" + primary_key = False + + +class CensusAgency19(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency19" + schema = "public" + primary_key = False + + +class CensusAgency20(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency20" + schema = "public" + primary_key = False + + +class CensusAgency21(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency21" + schema = "public" + primary_key = False + + +class CensusAgency22(BaseModel): + agency = TextField(column_name="AGENCY", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_agency22" + schema = "public" + primary_key = False + + +class CensusCaptext19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext19" + schema = "public" + primary_key = False + + +class CensusCaptext20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext20" + schema = "public" + primary_key = False + + +class CensusCaptext21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext21" + schema = "public" + primary_key = False + + +class CensusCaptext22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext22" + schema = "public" + primary_key = False + + +class CensusCaptextFormatted19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext_formatted19" + schema = "public" + primary_key = False + + +class CensusCaptextFormatted20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext_formatted20" + schema = "public" + primary_key = False + + +class CensusCaptextFormatted21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext_formatted21" + schema = "public" + primary_key = False + + +class CensusCaptextFormatted22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_captext_formatted22" + schema = "public" + primary_key = False + + +class CensusCfda16(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda16" + schema = "public" + primary_key = False + + +class CensusCfda17(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda17" + schema = "public" + primary_key = False + + +class CensusCfda18(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda18" + schema = "public" + primary_key = False + + +class CensusCfda19(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda19" + schema = "public" + primary_key = False + + +class CensusCfda20(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda20" + schema = "public" + primary_key = False + + +class CensusCfda21(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda21" + schema = "public" + primary_key = False + + +class CensusCfda22(BaseModel): + amount = TextField(column_name="AMOUNT", null=True) + arra = TextField(column_name="ARRA", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + awardidentification = TextField(column_name="AWARDIDENTIFICATION", null=True) + cfda = TextField(column_name="CFDA", null=True) + cfdaprogramname = TextField(column_name="CFDAPROGRAMNAME", null=True) + clustername = TextField(column_name="CLUSTERNAME", null=True) + clustertotal = TextField(column_name="CLUSTERTOTAL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + direct = TextField(column_name="DIRECT", null=True) + ein = TextField(column_name="EIN", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + federalprogramname = TextField(column_name="FEDERALPROGRAMNAME", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingscount = TextField(column_name="FINDINGSCOUNT", null=True) + loanbalance = TextField(column_name="LOANBALANCE", null=True) + loans = TextField(column_name="LOANS", null=True) + majorprogram = TextField(column_name="MAJORPROGRAM", null=True) + otherclustername = TextField(column_name="OTHERCLUSTERNAME", null=True) + passthroughamount = TextField(column_name="PASSTHROUGHAMOUNT", null=True) + passthroughaward = TextField(column_name="PASSTHROUGHAWARD", null=True) + programtotal = TextField(column_name="PROGRAMTOTAL", null=True) + qcosts2 = TextField(column_name="QCOSTS2", null=True) + rd = TextField(column_name="RD", null=True) + stateclustername = TextField(column_name="STATECLUSTERNAME", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cfda22" + schema = "public" + primary_key = False + + +class CensusCpas16(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas16" + schema = "public" + primary_key = False + + +class CensusCpas17(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas17" + schema = "public" + primary_key = False + + +class CensusCpas18(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas18" + schema = "public" + primary_key = False + + +class CensusCpas19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas19" + schema = "public" + primary_key = False + + +class CensusCpas20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas20" + schema = "public" + primary_key = False + + +class CensusCpas21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas21" + schema = "public" + primary_key = False + + +class CensusCpas22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpaein = TextField(column_name="CPAEIN", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_cpas22" + schema = "public" + primary_key = False + + +class CensusDuns16(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns16" + schema = "public" + primary_key = False + + +class CensusDuns17(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns17" + schema = "public" + primary_key = False + + +class CensusDuns18(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns18" + schema = "public" + primary_key = False + + +class CensusDuns19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns19" + schema = "public" + primary_key = False + + +class CensusDuns20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns20" + schema = "public" + primary_key = False + + +class CensusDuns21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns21" + schema = "public" + primary_key = False + + +class CensusDuns22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + duns = TextField(column_name="DUNS", null=True) + dunseqnum = TextField(column_name="DUNSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_duns22" + schema = "public" + primary_key = False + + +class CensusEins16(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins16" + schema = "public" + primary_key = False + + +class CensusEins17(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins17" + schema = "public" + primary_key = False + + +class CensusEins18(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins18" + schema = "public" + primary_key = False + + +class CensusEins19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins19" + schema = "public" + primary_key = False + + +class CensusEins20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins20" + schema = "public" + primary_key = False + + +class CensusEins21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins21" + schema = "public" + primary_key = False + + +class CensusEins22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + ein = TextField(column_name="EIN", null=True) + einseqnum = TextField(column_name="EINSEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_eins22" + schema = "public" + primary_key = False + + +class CensusFindings16(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings16" + schema = "public" + primary_key = False + + +class CensusFindings17(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings17" + schema = "public" + primary_key = False + + +class CensusFindings18(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings18" + schema = "public" + primary_key = False + + +class CensusFindings19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings19" + schema = "public" + primary_key = False + + +class CensusFindings20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings20" + schema = "public" + primary_key = False + + +class CensusFindings21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings21" + schema = "public" + primary_key = False + + +class CensusFindings22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditfindingsid = TextField(column_name="ELECAUDITFINDINGSID", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + findingsrefnums = TextField(column_name="FINDINGSREFNUMS", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + modifiedopinion = TextField(column_name="MODIFIEDOPINION", null=True) + otherfindings = TextField(column_name="OTHERFINDINGS", null=True) + othernoncompliance = TextField(column_name="OTHERNONCOMPLIANCE", null=True) + priorfindingrefnums = TextField(column_name="PRIORFINDINGREFNUMS", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + repeatfinding = TextField(column_name="REPEATFINDING", null=True) + significantdeficiency = TextField(column_name="SIGNIFICANTDEFICIENCY", null=True) + typerequirement = TextField(column_name="TYPEREQUIREMENT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findings22" + schema = "public" + primary_key = False + + +class CensusFindingstext19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext19" + schema = "public" + primary_key = False + + +class CensusFindingstext20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext20" + schema = "public" + primary_key = False + + +class CensusFindingstext21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext21" + schema = "public" + primary_key = False + + +class CensusFindingstext22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext22" + schema = "public" + primary_key = False + + +class CensusFindingstextFormatted19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext_formatted19" + schema = "public" + primary_key = False + + +class CensusFindingstextFormatted20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext_formatted20" + schema = "public" + primary_key = False + + +class CensusFindingstextFormatted21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext_formatted21" + schema = "public" + primary_key = False + + +class CensusFindingstextFormatted22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + chartstables = TextField(column_name="CHARTSTABLES", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + findingrefnums = TextField(column_name="FINDINGREFNUMS", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + text = TextField(column_name="TEXT", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_findingstext_formatted22" + schema = "public" + primary_key = False + + +class CensusGen16(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen16" + schema = "public" + primary_key = False + + +class CensusGen17(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen17" + schema = "public" + primary_key = False + + +class CensusGen18(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen18" + schema = "public" + primary_key = False + + +class CensusGen19(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen19" + schema = "public" + primary_key = False + + +class CensusGen20(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen20" + schema = "public" + primary_key = False + + +class CensusGen21(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen21" + schema = "public" + primary_key = False + + +class CensusGen22(BaseModel): + auditeecontact = TextField(column_name="AUDITEECONTACT", null=True) + auditeedatesigned = TextField(column_name="AUDITEEDATESIGNED", null=True) + auditeeemail = TextField(column_name="AUDITEEEMAIL", null=True) + auditeefax = TextField(column_name="AUDITEEFAX", null=True) + auditeename = TextField(column_name="AUDITEENAME", null=True) + auditeenametitle = TextField(column_name="AUDITEENAMETITLE", null=True) + auditeephone = TextField(column_name="AUDITEEPHONE", null=True) + auditeetitle = TextField(column_name="AUDITEETITLE", null=True) + auditor_ein = TextField(column_name="AUDITOR_EIN", null=True) + audittype = TextField(column_name="AUDITTYPE", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + city = TextField(column_name="CITY", null=True) + cogagency = TextField(column_name="COGAGENCY", null=True) + cog_over = TextField(column_name="COG_OVER", null=True) + cpacity = TextField(column_name="CPACITY", null=True) + cpacontact = TextField(column_name="CPACONTACT", null=True) + cpacountry = TextField(column_name="CPACOUNTRY", null=True) + cpadatesigned = TextField(column_name="CPADATESIGNED", null=True) + cpaemail = TextField(column_name="CPAEMAIL", null=True) + cpafax = TextField(column_name="CPAFAX", null=True) + cpafirmname = TextField(column_name="CPAFIRMNAME", null=True) + cpaforeign = TextField(column_name="CPAFOREIGN", null=True) + cpaphone = TextField(column_name="CPAPHONE", null=True) + cpastate = TextField(column_name="CPASTATE", null=True) + cpastreet1 = TextField(column_name="CPASTREET1", null=True) + cpastreet2 = TextField(column_name="CPASTREET2", null=True) + cpatitle = TextField(column_name="CPATITLE", null=True) + cpazipcode = TextField(column_name="CPAZIPCODE", null=True) + cyfindings = TextField(column_name="CYFINDINGS", null=True) + datefirewall = TextField(column_name="DATEFIREWALL", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + dollarthreshold = TextField(column_name="DOLLARTHRESHOLD", null=True) + duns = TextField(column_name="DUNS", null=True) + dup_reports = TextField(column_name="DUP_REPORTS", null=True) + ein = TextField(column_name="EIN", null=True) + einsubcode = TextField(column_name="EINSUBCODE", null=True) + entity_type = TextField(column_name="ENTITY_TYPE", null=True) + facaccepteddate = TextField(column_name="FACACCEPTEDDATE", null=True) + fyenddate = TextField(column_name="FYENDDATE", null=True) + goingconcern = TextField(column_name="GOINGCONCERN", null=True) + lowrisk = TextField(column_name="LOWRISK", null=True) + materialnoncompliance = TextField(column_name="MATERIALNONCOMPLIANCE", null=True) + materialweakness = TextField(column_name="MATERIALWEAKNESS", null=True) + materialweakness_mp = TextField(column_name="MATERIALWEAKNESS_MP", null=True) + multipleduns = TextField(column_name="MULTIPLEDUNS", null=True) + multipleeins = TextField(column_name="MULTIPLEEINS", null=True) + multipleueis = TextField(column_name="MULTIPLEUEIS", null=True) + multiple_cpas = TextField(column_name="MULTIPLE_CPAS", null=True) + numbermonths = TextField(column_name="NUMBERMONTHS", null=True) + oversightagency = TextField(column_name="OVERSIGHTAGENCY", null=True) + periodcovered = TextField(column_name="PERIODCOVERED", null=True) + previousdatefirewall = TextField(column_name="PREVIOUSDATEFIREWALL", null=True) + pyschedule = TextField(column_name="PYSCHEDULE", null=True) + qcosts = TextField(column_name="QCOSTS", null=True) + reportablecondition = TextField(column_name="REPORTABLECONDITION", null=True) + reportablecondition_mp = TextField(column_name="REPORTABLECONDITION_MP", null=True) + reportrequired = TextField(column_name="REPORTREQUIRED", null=True) + sp_framework = TextField(column_name="SP_FRAMEWORK", null=True) + sp_framework_required = TextField(column_name="SP_FRAMEWORK_REQUIRED", null=True) + state = TextField(column_name="STATE", null=True) + street1 = TextField(column_name="STREET1", null=True) + street2 = TextField(column_name="STREET2", null=True) + totfedexpend = TextField(column_name="TOTFEDEXPEND", null=True) + typeofentity = TextField(column_name="TYPEOFENTITY", null=True) + typereport_fs = TextField(column_name="TYPEREPORT_FS", null=True) + typereport_mp = TextField(column_name="TYPEREPORT_MP", null=True) + typereport_sp_framework = TextField( + column_name="TYPEREPORT_SP_FRAMEWORK", null=True + ) + uei = TextField(column_name="UEI", null=True) + zipcode = TextField(column_name="ZIPCODE", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_gen22" + schema = "public" + primary_key = False + + +class CensusNotes19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + content = TextField(column_name="CONTENT", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + id = TextField(column_name="ID", null=True) + note_index = TextField(column_name="NOTE_INDEX", null=True) + reportid = TextField(column_name="REPORTID", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + title = TextField(column_name="TITLE", null=True) + type_id = TextField(column_name="TYPE_ID", null=True) + version = TextField(column_name="VERSION", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_notes19" + schema = "public" + primary_key = False + + +class CensusNotes20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + content = TextField(column_name="CONTENT", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + id = TextField(column_name="ID", null=True) + note_index = TextField(column_name="NOTE_INDEX", null=True) + reportid = TextField(column_name="REPORTID", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + title = TextField(column_name="TITLE", null=True) + type_id = TextField(column_name="TYPE_ID", null=True) + version = TextField(column_name="VERSION", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_notes20" + schema = "public" + primary_key = False + + +class CensusNotes21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + content = TextField(column_name="CONTENT", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + id = TextField(column_name="ID", null=True) + note_index = TextField(column_name="NOTE_INDEX", null=True) + reportid = TextField(column_name="REPORTID", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + title = TextField(column_name="TITLE", null=True) + type_id = TextField(column_name="TYPE_ID", null=True) + version = TextField(column_name="VERSION", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_notes21" + schema = "public" + primary_key = False + + +class CensusNotes22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + content = TextField(column_name="CONTENT", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + id = TextField(column_name="ID", null=True) + note_index = TextField(column_name="NOTE_INDEX", null=True) + reportid = TextField(column_name="REPORTID", null=True) + seq_number = TextField(column_name="SEQ_NUMBER", null=True) + title = TextField(column_name="TITLE", null=True) + type_id = TextField(column_name="TYPE_ID", null=True) + version = TextField(column_name="VERSION", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_notes22" + schema = "public" + primary_key = False + + +class CensusPassthrough16(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough16" + schema = "public" + primary_key = False + + +class CensusPassthrough17(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough17" + schema = "public" + primary_key = False + + +class CensusPassthrough18(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough18" + schema = "public" + primary_key = False + + +class CensusPassthrough19(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough19" + schema = "public" + primary_key = False + + +class CensusPassthrough20(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough20" + schema = "public" + primary_key = False + + +class CensusPassthrough21(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough21" + schema = "public" + primary_key = False + + +class CensusPassthrough22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecauditsid = TextField(column_name="ELECAUDITSID", null=True) + passthroughid = TextField(column_name="PASSTHROUGHID", null=True) + passthroughname = TextField(column_name="PASSTHROUGHNAME", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_passthrough22" + schema = "public" + primary_key = False + + +class CensusRevisions19(BaseModel): + auditinfo = TextField(column_name="AUDITINFO", null=True) + auditinfo_explain = TextField(column_name="AUDITINFO_EXPLAIN", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + cap = TextField(column_name="CAP", null=True) + cap_explain = TextField(column_name="CAP_EXPLAIN", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecrptrevisionid = TextField(column_name="ELECRPTREVISIONID", null=True) + federalawards = TextField(column_name="FEDERALAWARDS", null=True) + federalawards_explain = TextField(column_name="FEDERALAWARDS_EXPLAIN", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingstext = TextField(column_name="FINDINGSTEXT", null=True) + findingstext_explain = TextField(column_name="FINDINGSTEXT_EXPLAIN", null=True) + findings_explain = TextField(column_name="FINDINGS_EXPLAIN", null=True) + geninfo = TextField(column_name="GENINFO", null=True) + geninfo_explain = TextField(column_name="GENINFO_EXPLAIN", null=True) + notestosefa = TextField(column_name="NOTESTOSEFA", null=True) + notestosefa_explain = TextField(column_name="NOTESTOSEFA_EXPLAIN", null=True) + other = TextField(column_name="OTHER", null=True) + other_explain = TextField(column_name="OTHER_EXPLAIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_revisions19" + schema = "public" + primary_key = False + + +class CensusRevisions20(BaseModel): + auditinfo = TextField(column_name="AUDITINFO", null=True) + auditinfo_explain = TextField(column_name="AUDITINFO_EXPLAIN", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + cap = TextField(column_name="CAP", null=True) + cap_explain = TextField(column_name="CAP_EXPLAIN", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecrptrevisionid = TextField(column_name="ELECRPTREVISIONID", null=True) + federalawards = TextField(column_name="FEDERALAWARDS", null=True) + federalawards_explain = TextField(column_name="FEDERALAWARDS_EXPLAIN", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingstext = TextField(column_name="FINDINGSTEXT", null=True) + findingstext_explain = TextField(column_name="FINDINGSTEXT_EXPLAIN", null=True) + findings_explain = TextField(column_name="FINDINGS_EXPLAIN", null=True) + geninfo = TextField(column_name="GENINFO", null=True) + geninfo_explain = TextField(column_name="GENINFO_EXPLAIN", null=True) + notestosefa = TextField(column_name="NOTESTOSEFA", null=True) + notestosefa_explain = TextField(column_name="NOTESTOSEFA_EXPLAIN", null=True) + other = TextField(column_name="OTHER", null=True) + other_explain = TextField(column_name="OTHER_EXPLAIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_revisions20" + schema = "public" + primary_key = False + + +class CensusRevisions21(BaseModel): + auditinfo = TextField(column_name="AUDITINFO", null=True) + auditinfo_explain = TextField(column_name="AUDITINFO_EXPLAIN", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + cap = TextField(column_name="CAP", null=True) + cap_explain = TextField(column_name="CAP_EXPLAIN", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecrptrevisionid = TextField(column_name="ELECRPTREVISIONID", null=True) + federalawards = TextField(column_name="FEDERALAWARDS", null=True) + federalawards_explain = TextField(column_name="FEDERALAWARDS_EXPLAIN", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingstext = TextField(column_name="FINDINGSTEXT", null=True) + findingstext_explain = TextField(column_name="FINDINGSTEXT_EXPLAIN", null=True) + findings_explain = TextField(column_name="FINDINGS_EXPLAIN", null=True) + geninfo = TextField(column_name="GENINFO", null=True) + geninfo_explain = TextField(column_name="GENINFO_EXPLAIN", null=True) + notestosefa = TextField(column_name="NOTESTOSEFA", null=True) + notestosefa_explain = TextField(column_name="NOTESTOSEFA_EXPLAIN", null=True) + other = TextField(column_name="OTHER", null=True) + other_explain = TextField(column_name="OTHER_EXPLAIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_revisions21" + schema = "public" + primary_key = False + + +class CensusRevisions22(BaseModel): + auditinfo = TextField(column_name="AUDITINFO", null=True) + auditinfo_explain = TextField(column_name="AUDITINFO_EXPLAIN", null=True) + audityear = TextField(column_name="AUDITYEAR", null=True) + cap = TextField(column_name="CAP", null=True) + cap_explain = TextField(column_name="CAP_EXPLAIN", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + elecrptrevisionid = TextField(column_name="ELECRPTREVISIONID", null=True) + federalawards = TextField(column_name="FEDERALAWARDS", null=True) + federalawards_explain = TextField(column_name="FEDERALAWARDS_EXPLAIN", null=True) + findings = TextField(column_name="FINDINGS", null=True) + findingstext = TextField(column_name="FINDINGSTEXT", null=True) + findingstext_explain = TextField(column_name="FINDINGSTEXT_EXPLAIN", null=True) + findings_explain = TextField(column_name="FINDINGS_EXPLAIN", null=True) + geninfo = TextField(column_name="GENINFO", null=True) + geninfo_explain = TextField(column_name="GENINFO_EXPLAIN", null=True) + notestosefa = TextField(column_name="NOTESTOSEFA", null=True) + notestosefa_explain = TextField(column_name="NOTESTOSEFA_EXPLAIN", null=True) + other = TextField(column_name="OTHER", null=True) + other_explain = TextField(column_name="OTHER_EXPLAIN", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_revisions22" + schema = "public" + primary_key = False + + +class CensusUeis22(BaseModel): + audityear = TextField(column_name="AUDITYEAR", null=True) + dbkey = TextField(column_name="DBKEY", null=True) + uei = TextField(column_name="UEI", null=True) + ueiseqnum = TextField(column_name="UEISEQNUM", null=True) + index = BigIntegerField(index=True, null=True) + + class Meta: + table_name = "census_ueis22" + schema = "public" + primary_key = False diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py new file mode 100644 index 0000000000..b7be56afde --- /dev/null +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -0,0 +1,50 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + WorkbookFieldInDissem, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, + test_pfix, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + + +def generate_corrective_action_plan(dbkey, year, outfile): + logger.info(f"--- generate corrective action plan {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Captext = dynamic_import("Captext", year) + wb = pyxl.load_workbook(templates["CAP"]) + mappings = [ + FieldMap("reference_number", "findingrefnums", "finding_ref_number", None, str), + FieldMap("planned_action", "text", WorkbookFieldInDissem, None, test_pfix(3)), + FieldMap( + "contains_chart_or_table", "chartstables", WorkbookFieldInDissem, None, str + ), + ] + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "corrective-action-plan-workbook") + + captexts = Captext.select().where(Captext.dbkey == g.dbkey) + + map_simple_columns(wb, mappings, captexts) + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "corrective_action_plans", dbkey, mappings, captexts + ) + table["singletons"]["auditee_uei"] = g.uei + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/end_to_end_core.py b/backend/census_historical_migration/workbooklib/end_to_end_core.py new file mode 100644 index 0000000000..d61d05e670 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/end_to_end_core.py @@ -0,0 +1,228 @@ +from users.models import User +import argparse +import logging +import sys +import math +from config import settings +import os +import jwt +import requests +from pprint import pprint +from datetime import datetime + +from census_historical_migration.workbooklib.workbook_creation import ( + sections, + workbook_loader, + setup_sac, +) +from census_historical_migration.workbooklib.sac_creation import _post_upload_pdf +from audit.intake_to_dissemination import IntakeToDissemination + +from dissemination.models import ( + AdditionalEin, + AdditionalUei, + CapText, + FederalAward, + Finding, + FindingText, + General, + Note, + Passthrough, + SecondaryAuditor, +) + +logger = logging.getLogger(__name__) +logging.basicConfig() +logging.getLogger().setLevel(logging.INFO) +parser = argparse.ArgumentParser() + +# Peewee runs a really noisy DEBUG log. +pw = logging.getLogger("peewee") +pw.addHandler(logging.StreamHandler()) +pw.setLevel(logging.INFO) + + +def step_through_certifications(sac): + sac.transition_to_ready_for_certification() + sac.transition_to_auditor_certified() + sac.transition_to_auditee_certified() + sac.transition_to_submitted() + sac.transition_to_disseminated() + sac.save() + + +def disseminate(sac, year): + logger.info("Invoking movement of data from Intake to Dissemination") + for model in [ + AdditionalEin, + AdditionalUei, + CapText, + FederalAward, + Finding, + FindingText, + General, + Note, + Passthrough, + SecondaryAuditor, + ]: + model.objects.filter(report_id=sac.report_id).delete() + + if sac.general_information: + etl = IntakeToDissemination(sac) + etl.load_all() + etl.save_dissemination_objects() + + +def create_payload(api_url, role="api_fac_gov"): + payload = { + # PostgREST only cares about the role. + "role": role, + "created": datetime.today().isoformat(), + } + return payload + + +def call_api(api_url, endpoint, rid, field): + # We must pass a properly signed JWT to access the API + encoded_jwt = jwt.encode( + create_payload(api_url), os.getenv("PGRST_JWT_SECRET"), algorithm="HS256" + ) + full_request = f"{api_url}/{endpoint}?report_id=eq.{rid}&select={field}" + response = requests.get( + full_request, + headers={ + "Authorization": f"Bearer {encoded_jwt}", + "X-Api-Key": os.getenv("CYPRESS_API_GOV_KEY"), + }, + timeout=10, + ) + return response + + +def just_numbers(s): + try: + float(s) + return True + except ValueError: + return False + + +def are_they_both_none_or_empty(a, b): + a_val = True if (a is None or a == "") else False + b_val = True if (b is None or b == "") else False + return a_val and b_val + + +def check_equality(in_wb, in_json): + # Type requirement is sometimes just 'N' + if in_wb in ["Y", "N"] and isinstance(in_json, bool): + return (True if in_wb == "Y" else False) == in_json + elif just_numbers(in_wb) and just_numbers(in_json): + return ( + True if math.isclose(float(in_wb), float(in_json), rel_tol=1e-1) else False + ) + elif isinstance(in_wb, str) and isinstance(in_json, str): + return in_wb.strip() == in_json.strip() + elif in_wb is None or in_json is None: + return are_they_both_none_or_empty(in_wb, in_json) + else: + return in_wb == in_json + + +def get_api_values(endpoint, rid, field): + api_url = settings.POSTGREST.get(settings.ENVIRONMENT) + res = call_api(api_url, endpoint, rid, field) + + if res.status_code == 200: + # print(f'{res.status_code} {res.url} {res.json()}') + return list(map(lambda d: d[field], res.json())) + else: + print(f"{res.status_code} {res.url}") + return [] + + +def count(d, key): + if key in d: + d[key] += 1 + else: + d[key] = 1 + + +def combine_counts(combined, d): + for k in combined.keys(): + if k in d: + combined[k] = combined[k] + d[k] + return combined + + +def api_check(json_test_tables): + combined_summary = {"endpoints": 0, "correct_rows": 0, "incorrect_rows": 0} + for endo in json_test_tables: + count(combined_summary, "endpoints") + endpoint = endo["endpoint"] + report_id = endo["report_id"] + print(f"-------------------- {endpoint} --------------------") + summary = {} + for row_ndx, row in enumerate(endo["rows"]): + count(summary, "total_rows") + equality_results = [] + for field_ndx, f in enumerate(row["fields"]): + # logger.info(f"Checking /{endpoint} {report_id} {f}") + # logger.info(f"{get_api_values(endpoint, report_id, f)}") + api_values = get_api_values(endpoint, report_id, f) + this_api_value = api_values[row_ndx] + this_field_value = row["values"][field_ndx] + eq = check_equality(this_field_value, this_api_value) + if not eq: + logger.info( + f"Does not match. [eq {eq}] [field {f}] [field val {this_field_value}] != [api val {this_api_value}]" + ) + equality_results.append(eq) + + if all(equality_results): + count(summary, "correct_fields") + else: + count(summary, "incorrect_fields") + sys.exit(-1) + logger.info(summary) + combined_summary = combine_counts(combined_summary, summary) + return combined_summary + + +def generate_workbooks(user, email, dbkey, year): + entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format( + dbkey=dbkey, year=year, date=datetime.now() + ) + sac = setup_sac(user, entity_id, dbkey) + if sac.general_information["audit_type"] == "alternative-compliance-engagement": + print(f"Skipping ACE audit: {dbkey}") + else: + loader = workbook_loader(user, sac, dbkey, year, entity_id) + json_test_tables = [] + for section, fun in sections.items(): + # FIXME: Can we conditionally upload the addl' and secondary workbooks? + (_, json, _) = loader(fun, section) + json_test_tables.append(json) + _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf") + step_through_certifications(sac) + + # shaped_sac = sac_validation_shape(sac) + # result = submission_progress_check(shaped_sac, sar=None, crossval=False) + # print(result) + + errors = sac.validate_cross() + pprint(errors.get("errors", "No errors found in cross validation")) + + disseminate(sac, year) + # pprint(json_test_tables) + combined_summary = api_check(json_test_tables) + logger.info(combined_summary) + + +def run_end_to_end(email, dbkey, year): + try: + user = User.objects.get(email=email) + except User.DoesNotExist: + logger.info("No user found for %s, have you logged in once?", email) + return + generate_workbooks(user, email, dbkey, year) diff --git a/backend/census_historical_migration/workbooklib/excel_creation.py b/backend/census_historical_migration/workbooklib/excel_creation.py new file mode 100644 index 0000000000..becac8b609 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/excel_creation.py @@ -0,0 +1,222 @@ +from collections import namedtuple as NT +from playhouse.shortcuts import model_to_dict +import os +import sys + +import logging +from datetime import date +from config import settings +import re +import json + +logger = logging.getLogger(__name__) + +# This provides a way to map the sheet in the workbook to the +# column in the DB. It also has a default value and +# the type of value, so that things can be set correctly +# before filling in the XLSX workbooks. +FieldMap = NT("FieldMap", "in_sheet in_db in_dissem default type") +WorkbookFieldInDissem = 1000 + +templates_root = "schemas/output/excel/xlsx/" +templates_raw = { + "AdditionalUEIs": "additional-ueis-workbook.xlsx", + "AdditionalEINs": "additional-eins-workbook.xlsx", + "AuditFindingsText": "audit-findings-text-workbook.xlsx", + "CAP": "corrective-action-plan-workbook.xlsx", + "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", + "FederalAwards": "federal-awards-workbook.xlsx", + "SEFA": "notes-to-sefa-workbook.xlsx", + "SecondaryAuditors": "secondary-auditors-workbook.xlsx", +} + +templates = {} +for k, v in templates_raw.items(): + templates[k] = os.path.join(templates_root, v) + + +def test_pfix(n): + def _test(o): + # return ' '.join(["TEST" for x in range(n)]) + " " + str(o) + return o + + return _test + + +def set_single_cell_range(wb, range_name, value): + the_range = wb.defined_names[range_name] + # The above returns a generator. Turn it to a list, and grab + # the first element of the list. Now, this *tuple* contains a + # sheet name and a cell reference... which you need to get rid + # of the '$' to use. + # https://itecnote.com/tecnote/python-using-excel-named-ranges-in-python-with-openpyxl/ + tup = list(the_range.destinations)[0] + sheet_title = tup[0] + cell_ref = tup[1].replace("$", "") + ws = wb[sheet_title] + ws[cell_ref] = value + + +# A tiny helper to index into workbooks. +# Assumes a capital letter. +def col_to_ndx(col): + return ord(col) - 65 + 1 + + +# Helper to set a range of values. +# Takes a named range, and then walks down the range, +# filling in values from the list past in (values). +def set_range(wb, range_name, values, default=None, conversion_fun=str): + the_range = wb.defined_names[range_name] + dest = list(the_range.destinations)[0] + sheet_title = dest[0] + ws = wb[sheet_title] + + start_cell = dest[1].replace("$", "").split(":")[0] + col = col_to_ndx(start_cell[0]) + start_row = int(start_cell[1]) + + for ndx, v in enumerate(values): + row = ndx + start_row + if v: + # This is a very noisy statement, showing everything + # written into the workbook. + # print(f'{range_name} c[{row}][{col}] <- {type(v)} len({len(v)}) {default}') + if v is not None: + ws.cell(row=row, column=col, value=conversion_fun(v)) + if len(str(v)) == 0 and default is not None: + # This is less noisy. Shows up for things like + # empty findings counts. 2023 submissions + # require that field to be 0, not empty, + # if there are no findings. + # print('Applying default') + ws.cell(row=row, column=col, value=conversion_fun(default)) + if not v: + if default is not None: + ws.cell(row=row, column=col, value=conversion_fun(default)) + else: + ws.cell(row=row, column=col, value="") + else: + # Leave it blank if we have no default passed in + pass + + +def set_uei(Gen, wb, dbkey): + g = Gen.select().where(Gen.dbkey == dbkey).get() + if g.uei: + set_single_cell_range(wb, "auditee_uei", g.uei) + else: + g.uei = "BADBADBADBAD" + set_single_cell_range(wb, "auditee_uei", g.uei) + return g + + +def map_simple_columns(wb, mappings, values): + len_passed_in = len(mappings) + unique_fields = set() + for mapping in mappings: + unique_fields.add(mapping.in_sheet) + if len_passed_in != len(unique_fields): + logger.info(f"unique: {len(unique_fields)} list: {len(mappings)}") + logger.error( + "You repeated a field in the mappings: {}".format( + list(map(lambda m: m.in_sheet, mappings)) + ) + ) + sys.exit(-1) + + # Map all the simple ones + for m in mappings: + set_range( + wb, + m.in_sheet, + map(lambda v: model_to_dict(v)[m.in_db], values), + m.default, + m.type, + ) + + +def _census_date_to_datetime(cd): + lookup = { + "JAN": 1, + "FEB": 2, + "MAR": 3, + "APR": 4, + "MAY": 5, + "JUN": 6, + "JUL": 7, + "AUG": 8, + "SEP": 9, + "OCT": 10, + "NOV": 11, + "DEC": 12, + } + year = int(cd.split("-")[2]) + month = lookup[cd.split("-")[1]] + day = int(cd.split("-")[0]) + return date(year + 2000, month, day) + + +# FIXME: Get the padding/shape right on the report_id +def dbkey_to_test_report_id(Gen, dbkey): + g = Gen.select(Gen.audityear, Gen.fyenddate).where(Gen.dbkey == dbkey).get() + # month = g.fyenddate.split('-')[1] + # 2022JUN0001000003 + # We start new audits at 1 million. + # So, we want 10 digits, and zero-pad for + # historic DBKEY report_ids + dt = _census_date_to_datetime(g.fyenddate) + return f"{g.audityear}-{dt.month:02}-TSTDAT-{dbkey.zfill(10)}" + + +def generate_dissemination_test_table(Gen, api_endpoint, dbkey, mappings, objects): + table = {"rows": list(), "singletons": dict()} + table["endpoint"] = api_endpoint + table["report_id"] = dbkey_to_test_report_id(Gen, dbkey) + for o in objects: + as_dict = model_to_dict(o) + test_obj = {} + test_obj["fields"] = [] + test_obj["values"] = [] + for m in mappings: + # What if we only test non-null values? + if ((m.in_db in as_dict) and as_dict[m.in_db] is not None) and ( + as_dict[m.in_db] != "" + ): + if m.in_dissem == WorkbookFieldInDissem: + # print(f'in_sheet {m.in_sheet} <- {as_dict[m.in_db]}') + test_obj["fields"].append(m.in_sheet) + # The typing must be applied here as well, as in the case of + # type_requirement, it alphabetizes the value... + test_obj["values"].append(m.type(as_dict[m.in_db])) + else: + # print(f'in_dissem {m.in_dissem} <- {as_dict[m.in_db]}') + test_obj["fields"].append(m.in_dissem) + test_obj["values"].append(m.type(as_dict[m.in_db])) + + table["rows"].append(test_obj) + return table + + +def extract_metadata(sheet_json, range): + excel_defn = open( + f"{settings.BASE_DIR}/schemas/output/excel/json/{sheet_json}.json" + ) + excel_defn_json = json.load(excel_defn) + result = None + for sheet in excel_defn_json["sheets"]: + if "name" in sheet and sheet["name"] == "Coversheet": + coversheet = sheet + for scell in coversheet["single_cells"]: + if ("range_name" in scell) and (scell["range_name"] == range): + result = scell["formula"] + return result + + +def insert_version_and_sheet_name(wb, sheet_json): + ver_cell = extract_metadata(sheet_json, "version") + ver_re = re.search('"(.*?)"', ver_cell)[1] + wb_name_cell = extract_metadata(sheet_json, "section_name") + wb_name_re = re.search('"(.*?)"', wb_name_cell)[1] + set_single_cell_range(wb, "version", ver_re) + set_single_cell_range(wb, "section_name", wb_name_re) diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py new file mode 100644 index 0000000000..00c421c60e --- /dev/null +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -0,0 +1,311 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + WorkbookFieldInDissem, + templates, + set_uei, + set_single_cell_range, + map_simple_columns, + generate_dissemination_test_table, + set_range, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + +from config import settings + +import openpyxl as pyxl +import json +import re + +import logging + +logger = logging.getLogger(__name__) + + +def if_zero_empty(v): + if int(v) == 0: + return "" + else: + return int(v) + + +mappings = [ + FieldMap("program_name", "federalprogramname", "federal_program_name", None, str), + FieldMap( + "state_cluster_name", "stateclustername", WorkbookFieldInDissem, None, str + ), + FieldMap("federal_program_total", "programtotal", WorkbookFieldInDissem, 0, int), + FieldMap("cluster_total", "clustertotal", WorkbookFieldInDissem, 0, int), + FieldMap("is_guaranteed", "loans", "is_loan", None, str), + FieldMap( + "loan_balance_at_audit_period_end", "loanbalance", "loan_balance", None, int + ), + FieldMap("is_direct", "direct", WorkbookFieldInDissem, None, str), + FieldMap("is_passed", "passthroughaward", "is_passthrough_award", None, str), + FieldMap( + "subrecipient_amount", + "passthroughamount", + "passthrough_amount", + None, + if_zero_empty, + ), + FieldMap("is_major", "majorprogram", WorkbookFieldInDissem, None, str), + FieldMap("audit_report_type", "typereport_mp", "audit_report_type", None, str), + FieldMap("number_of_audit_findings", "findingscount", "findings_count", 0, int), + FieldMap("amount_expended", "amount", WorkbookFieldInDissem, 0, int), +] + + +def get_list_index(all, index): + counter = 0 + for o in list(all): + if o.index == index: + return counter + else: + counter += 1 + return -1 + + +def int_or_na(o): + if o == "N/A": + return o + elif isinstance(o, int): + return int(o) + else: + return "N/A" + + +def _generate_cluster_names(Cfda, cfdas, valid_json): + cluster_names = [] + state_cluster_names = [] + other_cluster_names = [] + cfda: Cfda + for cfda in cfdas: + if cfda.clustername is None: + cluster_names.append("N/A") + other_cluster_names.append("") + state_cluster_names.append("") + elif cfda.clustername == "STATE CLUSTER": + cluster_names.append(cfda.clustername) + state_cluster_names.append(cfda.stateclustername) + other_cluster_names.append("") + elif cfda.clustername == "OTHER CLUSTER NOT LISTED ABOVE": + cluster_names.append(cfda.clustername) + other_cluster_names.append(cfda.otherclustername) + state_cluster_names.append("") + elif cfda.clustername in valid_json["cluster_names"]: + cluster_names.append(cfda.clustername) + other_cluster_names.append("") + state_cluster_names.append("") + else: + logger.debug(f"Cluster {cfda.clustername} not in the list. Replacing.") + cluster_names.append("OTHER CLUSTER NOT LISTED ABOVE") + other_cluster_names.append(f"{cfda.clustername}") + state_cluster_names.append("") + return (cluster_names, other_cluster_names, state_cluster_names) + + +def _fix_addl_award_identification(Cfda, cfdas, dbkey): + addls = ["" for x in list(range(0, len(cfdas)))] + for cfda in ( + Cfda.select() + .where( + (Cfda.dbkey == dbkey) + & ( + (Cfda.cfda % "%U%") + | (Cfda.cfda % "%u%") + | (Cfda.cfda % "%rd%") + | (Cfda.cfda % "%RD%") + ) + ) + .order_by(Cfda.index) + ): + if cfda.awardidentification is None or len(cfda.awardidentification) < 1: + addls[ + get_list_index(cfdas, cfda.index) + ] = f"ADDITIONAL AWARD INFO - DBKEY {dbkey}" + else: + addls[get_list_index(cfdas, cfda.index)] = cfda.awardidentification + return addls + + +def _fix_pfixes(cfdas): + # Map things with transformations + prefixes = map(lambda v: (v.cfda).split(".")[0], cfdas) + # prefixes = map(lambda v: f'0{v}' if int(v) < 10 else v, prefixes) + # Truncate any nastiness in the CFDA extensions to three characters. + extensions = map(lambda v: ((v.cfda).split(".")[1])[:3].upper(), cfdas) + extensions = map( + lambda v: v + if re.search("^(RD|RD[0-9]|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v) + else "000", + extensions, + ) + return (prefixes, extensions, map(lambda v: v.cfda, cfdas)) + + +def _fix_passthroughs(Cfda, Passthrough, cfdas, dbkey): + passthrough_names = ["" for x in list(range(0, len(cfdas)))] + passthrough_ids = ["" for x in list(range(0, len(cfdas)))] + ls = Cfda.select().where(Cfda.dbkey == dbkey).order_by(Cfda.index) + cfda: Cfda + for cfda in ls: + pnq = Passthrough() + if cfda.direct == "Y": + pnq.passthroughname = "" + pnq.passthroughid = "" + if cfda.direct == "N": + try: + pnq = ( + Passthrough.select().where( + (Passthrough.dbkey == cfda.dbkey) + & (Passthrough.elecauditsid == cfda.elecauditsid) + ) + ).get() + except Exception as e: + print(e) + pnq.passthroughname = "EXCEPTIONAL PASSTHROUGH NAME" + pnq.passthroughid = "EXCEPTIONAL PASSTHROUGH ID" + + name = pnq.passthroughname + if name is None: + name = "" + name = name.rstrip() + if name == "" and cfda.direct == "N": + passthrough_names[ + get_list_index(cfdas, cfda.index) + ] = "NO PASSTHROUGH NAME PROVIDED" + else: + passthrough_names[get_list_index(cfdas, cfda.index)] = name + + id = pnq.passthroughid + if id is None: + id = "" + id = id.rstrip() + if id == "" and cfda.direct == "N": + passthrough_ids[ + get_list_index(cfdas, cfda.index) + ] = "NO PASSTHROUGH ID PROVIDED" + else: + passthrough_ids[get_list_index(cfdas, cfda.index)] = pnq.passthroughid + + return (passthrough_names, passthrough_ids) + + +def generate_federal_awards(dbkey, year, outfile): + logger.info(f"--- generate federal awards {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Passthrough = dynamic_import("Passthrough", year) + Cfda = dynamic_import("Cfda", year) + wb = pyxl.load_workbook(templates["FederalAwards"]) + # In sheet : in DB + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "federal-awards-workbook") + + cfdas = Cfda.select().where(Cfda.dbkey == dbkey).order_by(Cfda.index) + map_simple_columns(wb, mappings, cfdas) + + # Patch the clusternames. They used to be allowed to enter anything + # they wanted. + valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/ClusterNames.json") + valid_json = json.load(valid_file) + + (cluster_names, other_cluster_names, state_cluster_names) = _generate_cluster_names( + Cfda, cfdas, valid_json + ) + set_range(wb, "cluster_name", cluster_names) + set_range(wb, "other_cluster_name", other_cluster_names) + + # Fix the additional award identification. If they had a "U", we want + # to see something in the addl. column. + addls = _fix_addl_award_identification(Cfda, cfdas, dbkey) + set_range(wb, "additional_award_identification", addls) + + (prefixes, extensions, full_cfdas) = _fix_pfixes(cfdas) + set_range(wb, "federal_agency_prefix", prefixes) + set_range(wb, "three_digit_extension", extensions) + + # We need a `cfda_key` as a magic column for the summation logic to work/be checked. + set_range(wb, "cfda_key", full_cfdas, conversion_fun=str) + + # We need `uniform_state_cluster_name` and `uniform_other_cluster_name` magic columns for cluster summation logic to work/be checked. + set_range( + wb, + "uniform_state_cluster_name", + [s.strip().upper() for s in state_cluster_names], + conversion_fun=str, + ) + set_range( + wb, + "uniform_other_cluster_name", + [s.strip().upper() for s in other_cluster_names], + conversion_fun=str, + ) + + (passthrough_names, passthrough_ids) = _fix_passthroughs( + Cfda, Passthrough, cfdas, dbkey + ) + set_range(wb, "passthrough_name", passthrough_names) + set_range(wb, "passthrough_identifying_number", passthrough_ids) + + # The award numbers! + set_range( + wb, + "award_reference", + [f"AWARD-{n+1:04}" for n in range(len(passthrough_names))], + ) + + # Total amount expended must be calculated and inserted + total = 0 + for cfda in cfdas: + total += int(cfda.amount) + set_single_cell_range(wb, "total_amount_expended", total) + + loansatend = list() + for ndx, cfda in enumerate( + Cfda.select().where((Cfda.dbkey == dbkey)).order_by(Cfda.index) + ): + if cfda.loans == "Y": + if cfda.loanbalance is None: + # loansatend.append("N/A") + loansatend.append(1) + else: + loansatend.append(cfda.loanbalance) + else: + loansatend.append("") + # set_range(wb, "loan_balance_at_audit_period_end", loansatend, type=int_or_na) + set_range(wb, "loan_balance_at_audit_period_end", loansatend, conversion_fun=int) + + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "federal_awards", dbkey, mappings, cfdas + ) + award_counter = 1 + # prefix + for obj, pfix, ext, addl, cn, ocn in zip( + table["rows"], prefixes, extensions, addls, cluster_names, other_cluster_names + ): + obj["fields"].append("federal_agency_prefix") + obj["values"].append(pfix) + obj["fields"].append("three_digit_extension") + obj["values"].append(ext) + # Sneak in the award number here + obj["fields"].append("award_reference") + obj["values"].append(f"AWARD-{award_counter:04}") + obj["fields"].append("additional_award_identification") + obj["values"].append(addl) + obj["fields"].append("cluster_name") + obj["values"].append(cn) + obj["fields"].append("other_cluster_name") + obj["fields"].append(ocn) + award_counter += 1 + + table["singletons"]["auditee_uei"] = g.uei + table["singletons"]["total_amount_expended"] = total + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py new file mode 100644 index 0000000000..29e138a1cf --- /dev/null +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -0,0 +1,134 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, + set_range, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + + +def sorted_string(s): + s_sorted = "".join(sorted(s)) + # print(f's before: {s} after {s_sorted}') + return s_sorted + + +mappings = [ + FieldMap( + "compliance_requirement", + "typerequirement", + "type_requirement", + "ABC", + sorted_string, + ), + FieldMap("reference_number", "findingsrefnums", "reference_number", None, str), + FieldMap("modified_opinion", "modifiedopinion", "is_modified_opinion", None, str), + FieldMap("other_matters", "othernoncompliance", "is_other_matters", None, str), + FieldMap( + "material_weakness", "materialweakness", "is_material_weakness", None, str + ), + FieldMap( + "significant_deficiency", + "significantdeficiency", + "is_significant_deficiency", + None, + str, + ), + FieldMap("other_findings", "otherfindings", "is_other_findings", None, str), + FieldMap("questioned_costs", "qcosts", "is_questioned_costs", None, str), + FieldMap("repeat_prior_reference", "repeatfinding", "is_repeat_finding", None, str), + FieldMap( + "prior_references", + "priorfindingrefnums", + "prior_finding_ref_numbers", + "N/A", + str, + ), +] + + +def _get_findings_grid(findings_list): + # The original copy of allowed_combos is in audit/intakelib/checks/check_findings_grid_validation.py + allowed_combos = { + "YNNNN", + "YNYNN", + "YNNYN", + "NYNNN", + "NYYNN", + "NYNYN", + "NNYNN", + "NNNYN", + "NNNNY", + } + + attributes = [ + "modifiedopinion", + "othernoncompliance", + "materialweakness", + "significantdeficiency", + "otherfindings", + ] + + return [ + "Y" + if "".join((getattr(finding, attr, "") or "").strip() for attr in attributes) + in allowed_combos + else "N" + for finding in findings_list + ] + + +def generate_findings(dbkey, year, outfile): + logger.info(f"--- generate findings {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Findings = dynamic_import("Findings", year) + Cfda = dynamic_import("Cfda", year) + wb = pyxl.load_workbook(templates["AuditFindings"]) + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "federal-awards-audit-findings-workbook") + + cfdas = Cfda.select().where(Cfda.dbkey == g.dbkey).order_by(Cfda.index) + # For each of them, I need to generate an elec -> award mapping. + e2a = {} + for ndx, cfda in enumerate(cfdas): + e2a[cfda.elecauditsid] = f"AWARD-{ndx+1:04d}" + + # CFDAs have elecauditid (FK). Findings have elecauditfindingsid, which is unique. + # The linkage here is that a given finding will have an elecauditid. + # Multiple findings will have a given elecauditid. That's how to link them. + findings = ( + Findings.select().where(Findings.dbkey == g.dbkey).order_by(Findings.index) + ) + award_references = [] + for find in findings: + award_references.append(e2a[find.elecauditsid]) + + map_simple_columns(wb, mappings, findings) + set_range(wb, "award_reference", award_references) + + grid = _get_findings_grid(findings) + # We need a magic "is_valid" column, which is computed in the workbook. + set_range(wb, "is_valid", grid, conversion_fun=str) + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "findings", dbkey, mappings, findings + ) + for obj, ar in zip(table["rows"], award_references): + obj["fields"].append("award_reference") + obj["values"].append(ar) + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py new file mode 100644 index 0000000000..22ee35eec2 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -0,0 +1,48 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + WorkbookFieldInDissem, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, + test_pfix, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + +mappings = [ + FieldMap("reference_number", "findingrefnums", "finding_ref_number", None, str), + FieldMap("text_of_finding", "text", "finding_text", None, test_pfix(3)), + FieldMap( + "contains_chart_or_table", "chartstables", WorkbookFieldInDissem, None, str + ), +] + + +def generate_findings_text(dbkey, year, outfile): + logger.info(f"--- generate findings text {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Findingstext = dynamic_import("Findingstext", year) + wb = pyxl.load_workbook(templates["AuditFindingsText"]) + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "audit-findings-text-workbook") + + ftexts = Findingstext.select().where(Findingstext.dbkey == g.dbkey) + map_simple_columns(wb, mappings, ftexts) + wb.save(outfile) + table = generate_dissemination_test_table( + Gen, "findings_text", dbkey, mappings, ftexts + ) + table["singletons"]["auditee_uei"] = g.uei + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py new file mode 100644 index 0000000000..c441d98062 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -0,0 +1,116 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + templates, + set_uei, + set_single_cell_range, + map_simple_columns, + generate_dissemination_test_table, + test_pfix, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, + set_range, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + + +import openpyxl as pyxl +import re +import logging + +# import unidecode + +logger = logging.getLogger(__name__) + +mappings = [ + FieldMap("note_title", "title", "title", None, test_pfix(3)), + FieldMap("note_content", "content", "content", None, test_pfix(3)), + # FieldMap("seq_number", "seq_number", "note_seq_number", 0, int), +] + + +def cleanup_string(s): + if s is None: + return "" + else: + s = s.rstrip() + # s = unidecode.unidecode(s) + s = str(s.encode("utf-8").decode("ascii", "ignore")) + return s + + +def generate_notes_to_sefa(dbkey, year, outfile): + logger.info(f"--- generate notes to sefa {dbkey} {year}---") + Gen = dynamic_import("Gen", year) + Notes = dynamic_import("Notes", year) + wb = pyxl.load_workbook(templates["SEFA"]) + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "notes-to-sefa-workbook") + + # The mapping is weird. + # https://facdissem.census.gov/Documents/DataDownloadKey.xlsx + # The TYPEID column determines which field in the form a given row corresponds to. + # TYPEID=1 is the description of significant accounting policies. + # TYPEID=2 is the De Minimis cost rate. + # TYPEID=3 is for notes, which have sequence numbers... that must align somewhere. + policies = ( + Notes.select().where((Notes.dbkey == g.dbkey) & (Notes.type_id == 1)).get() + ) + rate = Notes.select().where((Notes.dbkey == g.dbkey) & (Notes.type_id == 2)).get() + notes = ( + Notes.select() + .where((Notes.dbkey == g.dbkey) & (Notes.type_id == 3)) + .order_by(Notes.seq_number) + ) + + rate_content = cleanup_string(rate.content) + policies_content = cleanup_string(policies.content) + + if rate_content == "": + rate_content = "FILLED FOR TESTING" + if policies_content == "": + policies_content = "FILLED FOR TESTING" + + # WARNING + # This is being faked. We're askign a Y/N question in the collection. + # Census just let them type some stuff. So, this is a rough + # attempt to generate a Y/N value from the content. + # This means the data is *not* true to what was intended, but + # it *is* good enough for us to use for testing. + is_used = "Huh" + if ( + re.search("did not use", rate_content) + or re.search("not to use", rate_content) + or re.search("not use", rate_content) + or re.search("not elected", rate_content) + ): + is_used = "N" + elif re.search("used", rate_content): + is_used = "Y" + else: + is_used = "Both" + + set_single_cell_range(wb, "accounting_policies", policies_content) + set_single_cell_range(wb, "is_minimis_rate_used", is_used) + set_single_cell_range(wb, "rate_explained", rate_content) + + # Map the rest as notes. + map_simple_columns(wb, mappings, notes) + # Add a Y/N column + # def set_range(wb, range_name, values, default=None, conversion_fun=str): + + set_range(wb, "contains_chart_or_table", map(lambda v: "N", notes), "N", str) + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "notes_to_sefa", dbkey, mappings, notes + ) + + table["singletons"]["accounting_policies"] = policies_content + table["singletons"]["is_minimis_rate_used"] = is_used + table["singletons"]["rate_explained"] = rate_content + table["singletons"]["auditee_uei"] = g.uei + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/sac_creation.py b/backend/census_historical_migration/workbooklib/sac_creation.py new file mode 100644 index 0000000000..b9e8b165d5 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/sac_creation.py @@ -0,0 +1,386 @@ +"""Fixtures for SingleAuditChecklist. + +We want to create a variety of SACs in different states of +completion. +""" +from datetime import timedelta +import logging +from pathlib import Path + +from django.apps import apps + +from django.core.files.uploadedfile import SimpleUploadedFile + +from audit.intakelib import ( + extract_federal_awards, + extract_audit_findings as extract_findings_uniform_guidance, + extract_audit_findings_text as extract_findings_text, + extract_corrective_action_plan, + extract_secondary_auditors, + extract_notes_to_sefa, + extract_additional_ueis, + extract_additional_eins, +) +import audit.validators + +from audit.fixtures.excel import FORM_SECTIONS + +from census_historical_migration.workbooklib.excel_creation import ( + dbkey_to_test_report_id, + _census_date_to_datetime, +) + +from census_historical_migration.workbooklib.census_models.census import ( + CensusGen22 as Gen, + CensusCfda22 as Cfda, + CensusFindings22 as Finding, +) + +logger = logging.getLogger(__name__) + + +def get_field_by_section(sac, section): + if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: + return sac.federal_awards + elif section == FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: + return sac.findings_uniform_guidance + elif section == FORM_SECTIONS.FINDINGS_TEXT: + return sac.findings_text + elif section == FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: + return sac.corrective_action_plan + elif section == FORM_SECTIONS.SECONDARY_AUDITORS: + return sac.secondary_auditors + elif section == FORM_SECTIONS.NOTES_TO_SEFA: + return sac.notes_to_sefa + elif section == FORM_SECTIONS.ADDITIONAL_UEIS: + return sac.additional_ueis + + +extract_mapping = { + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: extract_federal_awards, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: extract_findings_uniform_guidance, + FORM_SECTIONS.FINDINGS_TEXT: extract_findings_text, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: extract_corrective_action_plan, + FORM_SECTIONS.SECONDARY_AUDITORS: extract_secondary_auditors, + FORM_SECTIONS.NOTES_TO_SEFA: extract_notes_to_sefa, + FORM_SECTIONS.ADDITIONAL_UEIS: extract_additional_ueis, + FORM_SECTIONS.ADDITIONAL_EINS: extract_additional_eins, +} + +validator_mapping = { + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: audit.validators.validate_federal_award_json, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: audit.validators.validate_findings_uniform_guidance_json, + FORM_SECTIONS.FINDINGS_TEXT: audit.validators.validate_findings_text_json, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: audit.validators.validate_corrective_action_plan_json, + FORM_SECTIONS.SECONDARY_AUDITORS: audit.validators.validate_secondary_auditors_json, + FORM_SECTIONS.NOTES_TO_SEFA: audit.validators.validate_notes_to_sefa_json, + FORM_SECTIONS.ADDITIONAL_UEIS: audit.validators.validate_additional_ueis_json, + FORM_SECTIONS.ADDITIONAL_EINS: audit.validators.validate_additional_eins_json, + "PDF": audit.validators.validate_single_audit_report_file, +} + + +def _period_covered(s): + return {"A": "annual", "B": "biennial", "O": "other"}[s] + + +def _census_audit_type(s): + return { + "S": "single-audit", + "P": "program-specific", + "A": "alternative-compliance-engagement", + }[s] + + +def add_hyphen_to_zip(zip): + strzip = str(zip) + if len(strzip) == 5: + return strzip + elif len(strzip) == 9: + return f"{strzip[0:5]}-{strzip[5:9]}" + else: + logger.info("ZIP IS MALFORMED IN WORKBOOKS E2E / SAC_CREATION") + return strzip + + +def _fake_general_information(dbkey, auditee_name="DEFAULT AUDITEE"): + """Create a fake general_information object.""" + # TODO: can we generate this object from the schema definition in + # schemas/output/GeneralInformation.schema.json? + gobj: Gen = Gen.select().where(Gen.dbkey == dbkey).first() + auditee_fiscal_period_end = _census_date_to_datetime(gobj.fyenddate).strftime( + "%Y-%m-%d" + ) + auditee_fiscal_period_start = ( + _census_date_to_datetime(gobj.fyenddate) - timedelta(days=365) + ).strftime("%Y-%m-%d") + if gobj.cpacountry == "US": + cpacountry = "USA" + elif gobj.cpacountry != "US": + cpacountry = "non-USA" + + general_information = { + "auditee_fiscal_period_start": auditee_fiscal_period_start, + "auditee_fiscal_period_end": auditee_fiscal_period_end, + "audit_period_covered": _period_covered(gobj.periodcovered), + "audit_type": _census_audit_type(gobj.audittype), + "auditee_address_line_1": gobj.street1, + "auditee_city": gobj.city, + "auditee_contact_name": gobj.auditeecontact, + "auditee_contact_title": gobj.auditeetitle, + "auditee_email": gobj.auditeeemail, + "auditee_name": gobj.auditeename, + "auditee_phone": gobj.auditeephone, + # TODO: when we include territories in our valid states, remove this restriction + "auditee_state": gobj.state, + # TODO: this is GSA's UEI. We could do better at making random choices that + # pass the schema's complex regex validation + "auditee_uei": gobj.uei, + "auditee_zip": gobj.zipcode, + "auditor_address_line_1": gobj.cpastreet1, + "auditor_city": gobj.cpacity, + "auditor_contact_name": gobj.cpacontact, + "auditor_contact_title": gobj.cpatitle, + "auditor_country": cpacountry, + "auditor_ein": gobj.auditor_ein, + "auditor_ein_not_an_ssn_attestation": True, + "auditor_email": gobj.cpaemail if gobj.cpaemail else "noemailfound@noemail.com", + "auditor_firm_name": gobj.cpafirmname, + "auditor_phone": gobj.cpaphone, + # TODO: when we include territories in our valid states, remove this restriction + "auditor_state": gobj.cpastate, + "auditor_zip": gobj.cpazipcode, + "ein": gobj.ein, + "ein_not_an_ssn_attestation": True, + "is_usa_based": True, + "met_spending_threshold": True, + "multiple_eins_covered": True if gobj.multipleeins == "Y" else False, + "multiple_ueis_covered": True if gobj.multipleueis == "Y" else False, + # TODO: could improve this by randomly choosing from the enum of possible values + "user_provided_organization_type": "unknown", + "secondary_auditors_exist": True if gobj.multiple_cpas == "Y" else False, + } + + # verify that our created object validates against the schema + audit.validators.validate_general_information_complete_json(general_information) + + return general_information + + +# TODO: Pull this from actual information. +def _fake_audit_information(dbkey, auditee_name=None): + gobj: Gen = Gen.select().where(Gen.dbkey == dbkey).first() + cfdas = Cfda.select().where(Cfda.dbkey == dbkey) + + agencies = {} + cfda: Cfda + for cfda in cfdas: + agencies[int((cfda.cfda).split(".")[0])] = 1 + + findings = Finding.select().where(Finding.dbkey == dbkey) + finding: Finding + gaap_results = {} + # THIS IS NOT A GOOD WAY TO DO THIS, BUT IT IS CLOSE. + # IT IS FOR TEST DATA... + for finding in findings: + if finding.modifiedopinion == "Y": + gaap_results["unmodified_opinion"] = 1 + if finding.materialweakness == "Y": + gaap_results["adverse_opinion"] = 1 + if finding.significantdeficiency == "Y": + gaap_results["disclaimer_of_opinion"] = 1 + + audit_information = { + "agencies": list( + map(lambda i: str(i) if len(str(i)) > 1 else f"0{str(i)}", agencies.keys()) + ), + "dollar_threshold": 750000, + "gaap_results": list(gaap_results.keys()), + "is_aicpa_audit_guide_included": True + if gobj.reportablecondition == "Y" + else False, + "is_going_concern_included": True if gobj.goingconcern == "Y" else False, + "is_internal_control_deficiency_disclosed": True + if gobj.materialweakness == "Y" + else False, + "is_internal_control_material_weakness_disclosed": True + if gobj.materialweakness_mp == "Y" + else False, + "is_low_risk_auditee": False, + "is_material_noncompliance_disclosed": True + if gobj.materialnoncompliance == "Y" + else False, + } + + audit.validators.validate_audit_information_json(audit_information) + + return audit_information + + +def _create_test_sac(user, auditee_name, dbkey): + """Create a single example SAC.""" + SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist") + + try: + exists = SingleAuditChecklist.objects.get( + report_id=dbkey_to_test_report_id(Gen, dbkey) + ) + except SingleAuditChecklist.DoesNotExist: + exists = None + if exists: + exists.delete() + + sac = SingleAuditChecklist.objects.create( + submitted_by=user, + general_information=_fake_general_information(dbkey, auditee_name), + audit_information=_fake_audit_information(dbkey, auditee_name), + ) + # Set a TEST report id for this data + sac.report_id = dbkey_to_test_report_id(Gen, dbkey) + + Access = apps.get_model("audit.Access") + Access.objects.create( + sac=sac, + user=user, + email=user.email, + role="editor", + ) + + # We need these to be different. + Access.objects.create( + sac=sac, + user=user, + email="bob_the_auditee_official@auditee.org", # user.email, + role="certifying_auditee_contact", + ) + Access.objects.create( + sac=sac, + user=user, + email="bob_the_auditor_official@auditor.org", # user.email, + role="certifying_auditor_contact", + ) + + sac.auditee_certification = {} + sac.auditee_certification["auditee_signature"] = {} + sac.auditee_certification["auditee_signature"][ + "auditee_name" + ] = "Bob the Auditee Name" + sac.auditee_certification["auditee_signature"][ + "auditee_title" + ] = "Bob the Auditee Signature" + + sac.auditor_certification = {} + sac.auditor_certification["auditor_signature"] = {} + sac.auditor_certification["auditor_signature"][ + "auditor_name" + ] = "Alice the Auditor Name" + sac.auditor_certification["auditor_signature"][ + "auditor_title" + ] = "Alice the Auditor Signature" + + sac.data_source = "TSTDAT" + sac.save() + + logger.info("Created single audit checklist %s", sac) + return sac + + +def _make_excel_file(filename, f_obj): + content = f_obj.read() + f_obj.seek(0) + file = SimpleUploadedFile(filename, content, "application/vnd.ms-excel") + return file + + +def _post_upload_pdf(this_sac, this_user, pdf_filename): + """Upload a workbook for this SAC. + + This should be idempotent if it is called on a SAC that already + has a federal awards file uploaded. + """ + PDFFile = apps.get_model("audit.SingleAuditReportFile") + + if PDFFile.objects.filter(sac_id=this_sac.id).exists(): + # there is already an uploaded file and data in the object so + # nothing to do here + return + + with open(pdf_filename, "rb") as f: + content = f.read() + file = SimpleUploadedFile(pdf_filename, content, "application/pdf") + print(file.__dict__) + pdf_file = PDFFile( + file=file, + component_page_numbers={ + "financial_statements": 1, + "financial_statements_opinion": 2, + "schedule_expenditures": 3, + "schedule_expenditures_opinion": 4, + "uniform_guidance_control": 5, + "uniform_guidance_compliance": 6, + "GAS_control": 6, + "GAS_compliance": 7, + "schedule_findings": 8, + }, + filename=Path(pdf_filename).stem, + user=this_user, + sac_id=this_sac.id, + ) + + validator_mapping["PDF"](pdf_file.file) + + pdf_file.full_clean() + pdf_file.save() + + this_sac.save() + + +def _post_upload_workbook(this_sac, this_user, section, xlsx_file): + """Upload a workbook for this SAC. + + This should be idempotent if it is called on a SAC that already + has a federal awards file uploaded. + """ + ExcelFile = apps.get_model("audit.ExcelFile") + + if ( + ExcelFile.objects.filter(sac_id=this_sac.id, form_section=section).exists() + and get_field_by_section(this_sac, section) is not None + ): + # there is already an uploaded file and data in the object so + # nothing to do here + return + + excel_file = ExcelFile( + file=xlsx_file, + filename=Path("xlsx.xlsx").stem, + user=this_user, + sac_id=this_sac.id, + form_section=section, + ) + excel_file.full_clean() + excel_file.save() + + audit_data = extract_mapping[section](excel_file.file) + validator_mapping[section](audit_data) + + if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: + this_sac.federal_awards = audit_data + elif section == FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: + this_sac.findings_uniform_guidance = audit_data + elif section == FORM_SECTIONS.FINDINGS_TEXT: + this_sac.findings_text = audit_data + elif section == FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: + this_sac.corrective_action_plan = audit_data + elif section == FORM_SECTIONS.SECONDARY_AUDITORS: + this_sac.secondary_auditors = audit_data + elif section == FORM_SECTIONS.NOTES_TO_SEFA: + this_sac.notes_to_sefa = audit_data + elif section == FORM_SECTIONS.ADDITIONAL_UEIS: + this_sac.additional_ueis = audit_data + elif section == FORM_SECTIONS.ADDITIONAL_EINS: + this_sac.additional_eins = audit_data + + this_sac.save() + + logger.info(f"Created {section} workbook upload for SAC {this_sac.id}") diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py new file mode 100644 index 0000000000..00e4ed7113 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -0,0 +1,77 @@ +from census_historical_migration.workbooklib.excel_creation import ( + FieldMap, + templates, + set_uei, + map_simple_columns, + generate_dissemination_test_table, + test_pfix, +) + +from census_historical_migration.workbooklib.excel_creation import ( + insert_version_and_sheet_name, +) +from census_historical_migration.workbooklib.census_models.census import dynamic_import + + +from census_historical_migration.workbooklib.sac_creation import add_hyphen_to_zip + +import openpyxl as pyxl + +import logging + +logger = logging.getLogger(__name__) + +mappings = [ + FieldMap("secondary_auditor_address_city", "cpacity", "address_city", None, str), + FieldMap("secondary_auditor_contact_name", "cpacontact", "contact_name", None, str), + FieldMap("secondary_auditor_ein", "cpaein", "auditor_ein", None, str), + FieldMap("secondary_auditor_contact_email", "cpaemail", "contact_email", None, str), + FieldMap("secondary_auditor_name", "cpafirmname", "auditor_name", None, str), + FieldMap("secondary_auditor_contact_phone", "cpaphone", "contact_phone", None, str), + FieldMap("secondary_auditor_address_state", "cpastate", "address_state", None, str), + FieldMap( + "secondary_auditor_address_street", + "cpastreet1", + "address_street", + None, + test_pfix(3), + ), + FieldMap( + "secondary_auditor_contact_title", + "cpatitle", + "contact_title", + None, + test_pfix(3), + ), + FieldMap( + "secondary_auditor_address_zipcode", + "cpazipcode", + "address_zipcode", + None, + add_hyphen_to_zip, + ), +] + + +def generate_secondary_auditors(dbkey, year, outfile): + logger.info(f"--- generate secondary auditors {dbkey} {year} ---") + Gen = dynamic_import("Gen", year) + Cpas = dynamic_import("Cpas", year) + wb = pyxl.load_workbook(templates["SecondaryAuditors"]) + + g = set_uei(Gen, wb, dbkey) + insert_version_and_sheet_name(wb, "secondary-auditors-workbook") + + sec_cpas = Cpas.select().where(Cpas.dbkey == g.dbkey) + + map_simple_columns(wb, mappings, sec_cpas) + + wb.save(outfile) + + table = generate_dissemination_test_table( + Gen, "secondary_auditor", dbkey, mappings, sec_cpas + ) + + table["singletons"]["auditee_uei"] = g.uei + + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/workbook_creation.py b/backend/census_historical_migration/workbooklib/workbook_creation.py new file mode 100644 index 0000000000..b678ddaf4f --- /dev/null +++ b/backend/census_historical_migration/workbooklib/workbook_creation.py @@ -0,0 +1,97 @@ +from fs.memoryfs import MemoryFS + +from census_historical_migration.workbooklib.sac_creation import ( + _post_upload_workbook, + _make_excel_file, + _create_test_sac, +) +from audit.fixtures.excel import FORM_SECTIONS +from django.apps import apps + + +from census_historical_migration.workbooklib.notes_to_sefa import generate_notes_to_sefa +from census_historical_migration.workbooklib.federal_awards import ( + generate_federal_awards, +) +from census_historical_migration.workbooklib.findings import generate_findings +from census_historical_migration.workbooklib.findings_text import generate_findings_text +from census_historical_migration.workbooklib.corrective_action_plan import ( + generate_corrective_action_plan, +) +from census_historical_migration.workbooklib.additional_ueis import ( + generate_additional_ueis, +) +from census_historical_migration.workbooklib.additional_eins import ( + generate_additional_eins, +) +from census_historical_migration.workbooklib.secondary_auditors import ( + generate_secondary_auditors, +) + +from model_bakery import baker +from django.contrib.auth import get_user_model + +import logging + +sections = { + FORM_SECTIONS.ADDITIONAL_EINS: generate_additional_eins, + FORM_SECTIONS.ADDITIONAL_UEIS: generate_additional_ueis, + FORM_SECTIONS.ADDITIONAL_UEIS: generate_additional_ueis, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: generate_corrective_action_plan, + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: generate_federal_awards, + FORM_SECTIONS.FINDINGS_TEXT: generate_findings_text, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: generate_findings, + FORM_SECTIONS.NOTES_TO_SEFA: generate_notes_to_sefa, + FORM_SECTIONS.SECONDARY_AUDITORS: generate_secondary_auditors, +} + +filenames = { + FORM_SECTIONS.ADDITIONAL_EINS: "additional-eins-{}.xlsx", + FORM_SECTIONS.ADDITIONAL_UEIS: "additional-ueis-{}.xlsx", + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: "corrective-action-plan-{}.xlsx", + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: "federal-awards-{}.xlsx", + FORM_SECTIONS.FINDINGS_TEXT: "audit-findings-text-{}.xlsx", + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: "audit-findings-{}.xlsx", + FORM_SECTIONS.NOTES_TO_SEFA: "notes-to-sefa-{}.xlsx", + FORM_SECTIONS.SECONDARY_AUDITORS: "secondary-auditors-{}.xlsx", +} + +logger = logging.getLogger(__name__) + + +def setup_sac(user, test_name, dbkey): + logger.info(f"Creating a SAC object for {user}, {test_name}") + SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist") + if user: + sac = SingleAuditChecklist.objects.filter( + submitted_by=user, general_information__auditee_name=test_name + ).first() + else: + sac = SingleAuditChecklist() + User = get_user_model() + user = baker.make(User) + sac.submitted_by = user + sac.general_information = {} + sac.general_information["auditee_name"] = test_name + + logger.info(sac) + if sac is None: + sac = _create_test_sac(user, test_name, dbkey) + return sac + + +def workbook_loader(user, sac, dbkey, year, entity_id): + def _loader(workbook_generator, section): + with MemoryFS() as mem_fs: + filename = filenames[section].format(dbkey) + outfile = mem_fs.openbin(filename, mode="w") + (wb, json) = workbook_generator(dbkey, year, outfile) + outfile.close() + outfile = mem_fs.openbin(filename, mode="r") + excel_file = _make_excel_file(filename, outfile) + if user: + _post_upload_workbook(sac, user, section, excel_file) + outfile.close() + return (wb, json, filename) + + return _loader diff --git a/backend/config/settings.py b/backend/config/settings.py index 16826a310d..9b5199a065 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -122,6 +122,7 @@ "report_submission", # "data_distro", "dissemination", + "census_historical_migration", "support", ] diff --git a/backend/dissemination/README.md b/backend/dissemination/README.md index 441b06e4bf..7e9186949b 100644 --- a/backend/dissemination/README.md +++ b/backend/dissemination/README.md @@ -95,3 +95,24 @@ When adding a new API version: - Change the values of `PGRST_DB_SCHEMAS` to your new API version. If previous versions of the API are needed, make the value a comma separated list and append your version to it. The first entry is the default, so only add to the front of the list if we are certain the schema should become the new default. See details on this [here](https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas) - This is likely true of TESTED patch version bumps (v1_0_0 to v1_0_1), and *maybe* minor version bumps (v1_0_0 to v1_1_0). MAJOR bumps require change management messaging. 5. If previous versions of the API are needed, `APIViewTests` will need to be updated. At the time of writing this, it only tests the default API. + +# End-to-end workbook testing + +### How to run the end-to-end test data generator: +``` +docker compose run web python manage.py end_to_end_test_data_generator --email any_email_in_the_system@woo.gov \ + --year 22 \ + --dbkey 100010 +``` +- The email address currently must be a User in the system. As this has only been run locally so far, it would often be a test account in my local sandbox env. +- `year` and `dbkey` are optional. The script will use default values for these if they aren't provided. + +### How to run the workbook generator: +``` +docker compose run web python manage.py generate_workbook_files + --year 22 \ + --output \ + --dbkey 100010 +``` +- `year` is optional and defaults to `22`. +- The `output` directory will be created if it doesn't already exist.