diff --git a/backend/functions/lib/adapters/gateways/abstract-mssql-client.exception.test.ts b/backend/functions/lib/adapters/gateways/abstract-mssql-client.exception.test.ts index af2010790..ae4aaeef5 100644 --- a/backend/functions/lib/adapters/gateways/abstract-mssql-client.exception.test.ts +++ b/backend/functions/lib/adapters/gateways/abstract-mssql-client.exception.test.ts @@ -47,9 +47,9 @@ jest.mock('mssql', () => { }); class MssqlClient extends AbstractMssqlClient { - constructor(context: ApplicationContext) { + constructor(_context: ApplicationContext) { const config = { server: 'foo' } as IDbConfig; - super(context, config, 'Exception Tests'); + super(config, 'Exception Tests'); } } diff --git a/backend/functions/lib/adapters/gateways/abstract-mssql-client.test.ts b/backend/functions/lib/adapters/gateways/abstract-mssql-client.test.ts index 536f429e6..03ff7ec44 100644 --- a/backend/functions/lib/adapters/gateways/abstract-mssql-client.test.ts +++ b/backend/functions/lib/adapters/gateways/abstract-mssql-client.test.ts @@ -72,7 +72,7 @@ describe('Abstract MS-SQL client', () => { // execute method under test class TestDbClient extends AbstractMssqlClient { constructor(context: ApplicationContext, config: IDbConfig, childModuleName: string) { - super(context, config, childModuleName); + super(config, childModuleName); } } const client = new TestDbClient(context, context.config.dxtrDbConfig, 'TEST_MODULE'); diff --git a/backend/functions/lib/adapters/gateways/abstract-mssql-client.ts b/backend/functions/lib/adapters/gateways/abstract-mssql-client.ts index bc94bb6d1..d2601f3be 100644 --- a/backend/functions/lib/adapters/gateways/abstract-mssql-client.ts +++ b/backend/functions/lib/adapters/gateways/abstract-mssql-client.ts @@ -8,7 +8,7 @@ export abstract class AbstractMssqlClient { private static connectionPool: ConnectionPool; private readonly moduleName: string; - protected constructor(context: ApplicationContext, dbConfig: IDbConfig, childModuleName: string) { + protected constructor(dbConfig: IDbConfig, childModuleName: string) { this.moduleName = `ABSTRACT-MSSQL-CLIENT (${childModuleName})`; if (!AbstractMssqlClient.connectionPool) { AbstractMssqlClient.connectionPool = new ConnectionPool(dbConfig as config); @@ -21,12 +21,10 @@ export abstract class AbstractMssqlClient { query: string, input?: DbTableFieldSpec[], ): Promise { - // we should do some sanitization here to eliminate sql injection issues try { if (!AbstractMssqlClient.connectionPool.connected) { await AbstractMssqlClient.connectionPool.connect(); } - // const connection = await AbstractMssqlClient.connectionPool.connect(); const request = AbstractMssqlClient.connectionPool.request(); if (typeof input != 'undefined') { @@ -64,8 +62,8 @@ export abstract class AbstractMssqlClient { } else if (isMssqlError(error)) { const newError = { error: { - name: error.name, // RequestError - description: error.message, // Timeout: Request failed to complete in 15000ms + name: error.name, + description: error.message, }, originalError: {}, query, @@ -83,13 +81,6 @@ export abstract class AbstractMssqlClient { context.logger.error(this.moduleName, error.message, { error, query, input }); } - // // TODO May want to refactor to throw CamsError and remove returning QueryResults - // const queryResult: QueryResults = { - // results: {}, - // message: (error as Error).message, - // success: false, - // }; - // return queryResult; throw getCamsError(error, this.moduleName, error.message); } } diff --git a/backend/functions/lib/adapters/gateways/acms/acms.gateway.ts b/backend/functions/lib/adapters/gateways/acms/acms.gateway.ts index a4bcfbc3b..5af864dba 100644 --- a/backend/functions/lib/adapters/gateways/acms/acms.gateway.ts +++ b/backend/functions/lib/adapters/gateways/acms/acms.gateway.ts @@ -19,7 +19,7 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway // The context carries different database connection configurations. // We pick off the configuration specific to this ACMS gateway. const config = context.config.acmsDbConfig; - super(context, config, MODULE_NAME); + super(config, MODULE_NAME); } async getPageCount(context: ApplicationContext, predicate: AcmsPredicate): Promise { @@ -32,7 +32,7 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway AND CONSOLIDATED_CASE_NUMBER != '0'`; // Valid ACMS chapters: 09, 11, 12, 13, 15, 7A, 7N, AC - // 'AC' is the predecesor to chapter 15. We are not importing these old cases into CAMS. + // 'AC' is the predecessor to chapter 15. We are not importing these old cases into CAMS. // '7A' and '7N' are treated inclusively as chapter 7 cases when importing into CAMS. // Leading zero padding is added for chapter 9. @@ -184,7 +184,6 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway } private formatCaseId(caseId: string): string { - console.log(`Case ID: ${caseId} is of type ${typeof caseId}.`); const padded = caseId.padStart(10, '0'); return `${padded.slice(0, 3)}-${padded.slice(3, 5)}-${padded.slice(5)}`; } diff --git a/backend/functions/lib/adapters/utils/feature-flag.ts b/backend/functions/lib/adapters/utils/feature-flag.ts index 0a5788554..05cd60e68 100644 --- a/backend/functions/lib/adapters/utils/feature-flag.ts +++ b/backend/functions/lib/adapters/utils/feature-flag.ts @@ -12,6 +12,7 @@ export async function getFeatureFlags(config: ApplicationConfiguration): Promise eventsUri: 'https://events.launchdarkly.us', }); await client.waitForInitialization(); + //TODO: revisit if this is necessary when we look at why we need to restart the backend for feature flags const state = await client.allFlagsState({ kind: 'user', key: 'feature-flag-migration', diff --git a/backend/functions/lib/controllers/acms-orders/acms-orders.controller.test.ts b/backend/functions/lib/controllers/acms-orders/acms-orders.controller.test.ts index 7cd95e67e..b5e1aa581 100644 --- a/backend/functions/lib/controllers/acms-orders/acms-orders.controller.test.ts +++ b/backend/functions/lib/controllers/acms-orders/acms-orders.controller.test.ts @@ -1,5 +1,5 @@ import AcmsOrders, { - AcmsConsolidationReport, + AcmsTransformationResult, AcmsPredicate, AcmsPredicateAndPage, } from '../../use-cases/acms-orders/acms-orders'; @@ -45,8 +45,9 @@ describe('AcmsOrdersController', () => { test('should return processing report from migrateConsolidation', async () => { const leadCaseId = '811100000'; - const report: AcmsConsolidationReport = { + const report: AcmsTransformationResult = { leadCaseId, + childCaseCount: 1, success: true, }; const spy = jest.spyOn(AcmsOrders.prototype, 'migrateConsolidation').mockResolvedValue(report); diff --git a/backend/functions/lib/controllers/acms-orders/acms-orders.controller.ts b/backend/functions/lib/controllers/acms-orders/acms-orders.controller.ts index c0abd6fae..d3f53fe44 100644 --- a/backend/functions/lib/controllers/acms-orders/acms-orders.controller.ts +++ b/backend/functions/lib/controllers/acms-orders/acms-orders.controller.ts @@ -1,7 +1,6 @@ import { ApplicationContext } from '../../adapters/types/basic'; -// import { finalizeDeferrable } from '../../deferrable/finalize-deferrable'; import AcmsOrders, { - AcmsConsolidationReport, + AcmsTransformationResult, AcmsPredicate, AcmsPredicateAndPage, } from '../../use-cases/acms-orders/acms-orders'; @@ -11,9 +10,8 @@ class AcmsOrdersController { public async migrateConsolidation( context: ApplicationContext, leadCaseId: string, - ): Promise { + ): Promise { const response = this.useCase.migrateConsolidation(context, leadCaseId); - // await finalizeDeferrable(context); return response; } @@ -22,7 +20,6 @@ class AcmsOrdersController { predicate: AcmsPredicate, ): Promise { const response = this.useCase.getPageCount(context, predicate); - // await finalizeDeferrable(context); return response; } @@ -31,7 +28,6 @@ class AcmsOrdersController { predicate: AcmsPredicateAndPage, ): Promise { const response = this.useCase.getLeadCaseIds(context, predicate); - // await finalizeDeferrable(context); return response; } } diff --git a/backend/functions/lib/deferrable/defer-close.ts b/backend/functions/lib/deferrable/defer-close.ts index 72316a4e5..ef491d60f 100644 --- a/backend/functions/lib/deferrable/defer-close.ts +++ b/backend/functions/lib/deferrable/defer-close.ts @@ -43,7 +43,6 @@ const globalAccumulator: DeferCloseAccumulator = { }; async function closeGlobal() { - console.warn(`***** CLOSING ${globalAccumulator.closables.length} CLOSABLES ****`); closeDeferred(globalAccumulator); } diff --git a/backend/functions/lib/use-cases/acms-orders/acms-orders.ts b/backend/functions/lib/use-cases/acms-orders/acms-orders.ts index 60a8c8830..316b04055 100644 --- a/backend/functions/lib/use-cases/acms-orders/acms-orders.ts +++ b/backend/functions/lib/use-cases/acms-orders/acms-orders.ts @@ -38,11 +38,32 @@ export type AcmsConsolidation = { childCases: AcmsConsolidationChildCase[]; }; -export type AcmsConsolidationReport = { +export type AcmsTransformationResult = { leadCaseId: string; + childCaseCount: number; success: boolean; }; +export type AcmsAggregate = { + successful: { + leadCaseCount: number; + childCaseCount: number; + }; + failed: { + leadCaseIds: string[]; + leadCaseCount: number; + childCaseCount: number; + }; +}; + +export type AcmsPageReport = AcmsAggregate & { + predicateAndPage: AcmsPredicateAndPage; +}; + +export type AcmsPartitionReport = AcmsAggregate & { + predicate: AcmsPredicate; +}; + export class AcmsOrders { public async getPageCount( context: ApplicationContext, @@ -63,9 +84,13 @@ export class AcmsOrders { public async migrateConsolidation( context: ApplicationContext, acmsLeadCaseId: string, - ): Promise { + ): Promise { // TODO: Add child case count to the report?? - const report: AcmsConsolidationReport = { leadCaseId: acmsLeadCaseId, success: true }; + const report: AcmsTransformationResult = { + leadCaseId: acmsLeadCaseId, + childCaseCount: 0, + success: true, + }; try { const casesRepo = Factory.getCasesRepository(context); const dxtr = Factory.getCasesGateway(context); @@ -92,6 +117,8 @@ export class AcmsOrders { return acc; }, new Set()); + report.childCaseCount = exportedChildCaseIds.size; + const unimportedChildCaseIds = new Set(); exportedChildCaseIds.forEach((caseId) => { if (!existingChildCaseIds.has(caseId)) { diff --git a/backend/functions/migration/activity/migrateConsolidation.ts b/backend/functions/migration/activity/migrateConsolidation.ts index a58c166f6..ff5077efb 100644 --- a/backend/functions/migration/activity/migrateConsolidation.ts +++ b/backend/functions/migration/activity/migrateConsolidation.ts @@ -2,14 +2,14 @@ import { InvocationContext } from '@azure/functions'; import ContextCreator from '../../azure/application-context-creator'; import AcmsOrdersController from '../../lib/controllers/acms-orders/acms-orders.controller'; import { getCamsError } from '../../lib/common-errors/error-utilities'; -import { AcmsConsolidationReport } from '../../lib/use-cases/acms-orders/acms-orders'; +import { AcmsTransformationResult } from '../../lib/use-cases/acms-orders/acms-orders'; const MODULE_NAME = 'IMPORT_ACTION_MIGRATE_CONSOLIDATION'; async function migrateConsolidation( leadCaseId: string, invocationContext: InvocationContext, -): Promise { +): Promise { const logger = ContextCreator.getLogger(invocationContext); const appContext = await ContextCreator.getApplicationContext({ invocationContext, logger }); const controller = new AcmsOrdersController(); diff --git a/backend/functions/migration/activity/migrateConsolidations.test.ts b/backend/functions/migration/activity/migrateConsolidations.test.ts index 80603afe3..e2263fba8 100644 --- a/backend/functions/migration/activity/migrateConsolidations.test.ts +++ b/backend/functions/migration/activity/migrateConsolidations.test.ts @@ -3,6 +3,7 @@ import AcmsOrdersController from '../../lib/controllers/acms-orders/acms-orders. import module from './migrateConsolidation'; import { createMockAzureFunctionContext } from '../../azure/testing-helpers'; import { CamsError } from '../../lib/common-errors/cams-error'; +import { AcmsTransformationResult } from '../../lib/use-cases/acms-orders/acms-orders'; describe('getConsolidations test', () => { afterEach(() => { @@ -11,8 +12,9 @@ describe('getConsolidations test', () => { test('should call getLeadCaseIds controller method', async () => { const caseId = '000-11-22222'; - const expected = { + const expected: AcmsTransformationResult = { leadCaseId: caseId, + childCaseCount: 2, success: true, }; const getLeadCaseIdsSpy = jest diff --git a/backend/functions/migration/client/acms-migration-trigger.function.ts b/backend/functions/migration/client/acms-migration-trigger.function.ts index d2ae53079..7a56ad0e7 100644 --- a/backend/functions/migration/client/acms-migration-trigger.function.ts +++ b/backend/functions/migration/client/acms-migration-trigger.function.ts @@ -14,21 +14,22 @@ export default async function httpStart( context: InvocationContext, ): Promise { const client = df.getClient(context); - let body: TriggerRequest; + let params: TriggerRequest; if (request.body) { - body = (await request.json()) as unknown as TriggerRequest; + params = (await request.json()) as unknown as TriggerRequest; } - if (!isTriggerRequest(body)) { + if (!isTriggerRequest(params)) { throw new BadRequestError(MODULE_NAME, { message: 'Missing or malformed request body.' }); } - if (body.apiKey !== process.env.ADMIN_KEY) { + if (params.apiKey !== process.env.ADMIN_KEY) { throw new UnauthorizedError(MODULE_NAME, { message: 'API key was missing or did not match.' }); } + delete params.apiKey; const instanceId: string = await client.startNew(MAIN_ORCHESTRATOR, { - input: body, + input: params, }); return client.createCheckStatusResponse(request, instanceId); diff --git a/backend/functions/migration/orchestration/orchestrator.ts b/backend/functions/migration/orchestration/orchestrator.ts index 53858da2d..65d417a16 100644 --- a/backend/functions/migration/orchestration/orchestrator.ts +++ b/backend/functions/migration/orchestration/orchestrator.ts @@ -1,4 +1,9 @@ -import { AcmsBounds, AcmsPredicate } from '../../lib/use-cases/acms-orders/acms-orders'; +import { + AcmsAggregate, + AcmsBounds, + AcmsPartitionReport, + AcmsPredicate, +} from '../../lib/use-cases/acms-orders/acms-orders'; import { FLATTEN_BOUNDING_ARRAYS, SUB_ORCHESTRATOR_PAGING } from '../loadConsolidations'; import { OrchestrationContext } from 'durable-functions'; @@ -18,4 +23,23 @@ export function* main(context: OrchestrationContext) { ); } yield context.df.Task.all(provisioningTasks); + + const summary = provisioningTasks.reduce( + (acc, task) => { + const report = task.result as AcmsPartitionReport; + context.log('Report', JSON.stringify(report)); + acc.successful.leadCaseCount += report.successful.leadCaseCount; + acc.successful.childCaseCount += report.successful.childCaseCount; + acc.failed.leadCaseIds.push(...report.failed.leadCaseIds); + acc.failed.leadCaseCount += report.failed.leadCaseCount; + acc.failed.childCaseCount += report.failed.childCaseCount; + return acc; + }, + { + successful: { leadCaseCount: 0, childCaseCount: 0 }, + failed: { leadCaseIds: [], leadCaseCount: 0, childCaseCount: 0 }, + } as AcmsAggregate, + ); + + context.log('Summary', JSON.stringify(bounds), JSON.stringify(summary)); } diff --git a/backend/functions/migration/orchestration/sub-orchestrator-etl.ts b/backend/functions/migration/orchestration/sub-orchestrator-etl.ts index 1e2879a03..1d1b93669 100644 --- a/backend/functions/migration/orchestration/sub-orchestrator-etl.ts +++ b/backend/functions/migration/orchestration/sub-orchestrator-etl.ts @@ -1,6 +1,7 @@ import { - AcmsConsolidationReport, + AcmsTransformationResult, AcmsPredicateAndPage, + AcmsPageReport, } from '../../lib/use-cases/acms-orders/acms-orders'; import { GET_CONSOLIDATIONS, MIGRATE_CONSOLIDATION } from '../loadConsolidations'; import { OrchestrationContext } from 'durable-functions'; @@ -18,19 +19,23 @@ export function* subOrchestratorETL(context: OrchestrationContext) { yield context.df.Task.all(etlTasks); - const finalResults = etlTasks.reduce( + return etlTasks.reduce( (acc, task) => { - const taskResponse = task as AcmsConsolidationReport; - if (taskResponse.success) { - acc.successful += 1; + const transformationResult = task.result as AcmsTransformationResult; + if (transformationResult.success) { + acc.successful.leadCaseCount += 1; + acc.successful.childCaseCount += transformationResult.childCaseCount; } else { - acc.failed += 1; + acc.failed.leadCaseIds.push(transformationResult.leadCaseId); + acc.failed.leadCaseCount += 1; + acc.failed.childCaseCount += transformationResult.childCaseCount; } return acc; }, - { successful: 0, failed: 0 }, - ); - context.log( - `ACMS Consolidation Migration ETL: successful: ${finalResults.successful}, failures: ${finalResults.failed}`, + { + predicateAndPage, + successful: { leadCaseCount: 0, childCaseCount: 0 }, + failed: { leadCaseIds: [], leadCaseCount: 0, childCaseCount: 0 }, + } as AcmsPageReport, ); } diff --git a/backend/functions/migration/orchestration/sub-orchestrator-paging.ts b/backend/functions/migration/orchestration/sub-orchestrator-paging.ts index c0d177f4d..7c780b698 100644 --- a/backend/functions/migration/orchestration/sub-orchestrator-paging.ts +++ b/backend/functions/migration/orchestration/sub-orchestrator-paging.ts @@ -1,22 +1,44 @@ import { OrchestrationContext } from 'durable-functions'; import { GET_PAGE_COUNT, SUB_ORCHESTRATOR_ETL } from '../loadConsolidations'; -import { AcmsPredicate, AcmsPredicateAndPage } from '../../lib/use-cases/acms-orders/acms-orders'; +import { + AcmsPageReport, + AcmsPredicate, + AcmsPredicateAndPage, + AcmsPartitionReport, +} from '../../lib/use-cases/acms-orders/acms-orders'; export function* subOrchestratorPaging(context: OrchestrationContext) { const predicate: AcmsPredicate = context.df.getInput(); const pageCount: number = yield context.df.callActivity(GET_PAGE_COUNT, predicate); - const provisioningTasks = []; + const pagingTasks = []; for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { const predicateAndPage: AcmsPredicateAndPage = { ...predicate, pageNumber, }; const child_id = context.df.instanceId + `:${pageNumber}`; - provisioningTasks.push( + pagingTasks.push( context.df.callSubOrchestrator(SUB_ORCHESTRATOR_ETL, predicateAndPage, child_id), ); } - yield context.df.Task.all(provisioningTasks); + yield context.df.Task.all(pagingTasks); + + return pagingTasks.reduce( + (acc, task) => { + const report = task.result as AcmsPageReport; + acc.successful.leadCaseCount += report.successful.leadCaseCount; + acc.successful.childCaseCount += report.successful.childCaseCount; + acc.failed.leadCaseIds.push(...report.failed.leadCaseIds); + acc.failed.leadCaseCount += report.failed.leadCaseCount; + acc.failed.childCaseCount += report.failed.childCaseCount; + return acc; + }, + { + predicate, + successful: { leadCaseCount: 0, childCaseCount: 0 }, + failed: { leadCaseIds: [], leadCaseCount: 0, childCaseCount: 0 }, + } as AcmsPartitionReport, + ); }