Add logging of aggregate results

Jira ticket: CAMS-461

Co-authored-by: Fritz Madden <[email protected]>
Co-authored-by: Brian Posey <[email protected]>,

backend/functions/lib/adapters/gateways/abstract-mssql-client.exception.test.ts

@@ -47,9 +47,9 @@ jest.mock('mssql', () => {
 });
 
 class MssqlClient extends AbstractMssqlClient {
-  constructor(context: ApplicationContext) {
+  constructor(_context: ApplicationContext) {
     const config = { server: 'foo' } as IDbConfig;
-    super(context, config, 'Exception Tests');
+    super(config, 'Exception Tests');
   }
 }
 

backend/functions/lib/adapters/gateways/abstract-mssql-client.test.ts

@@ -72,7 +72,7 @@ describe('Abstract MS-SQL client', () => {
     // execute method under test
     class TestDbClient extends AbstractMssqlClient {
       constructor(context: ApplicationContext, config: IDbConfig, childModuleName: string) {
-        super(context, config, childModuleName);
+        super(config, childModuleName);
       }
     }
     const client = new TestDbClient(context, context.config.dxtrDbConfig, 'TEST_MODULE');

backend/functions/lib/adapters/gateways/abstract-mssql-client.ts

@@ -8,7 +8,7 @@ export abstract class AbstractMssqlClient {
   private static connectionPool: ConnectionPool;
   private readonly moduleName: string;
 
-  protected constructor(context: ApplicationContext, dbConfig: IDbConfig, childModuleName: string) {
+  protected constructor(dbConfig: IDbConfig, childModuleName: string) {
     this.moduleName = `ABSTRACT-MSSQL-CLIENT (${childModuleName})`;
     if (!AbstractMssqlClient.connectionPool) {
       AbstractMssqlClient.connectionPool = new ConnectionPool(dbConfig as config);
@@ -21,12 +21,10 @@ export abstract class AbstractMssqlClient {
     query: string,
     input?: DbTableFieldSpec[],
   ): Promise<QueryResults> {
-    // we should do some sanitization here to eliminate sql injection issues
     try {
       if (!AbstractMssqlClient.connectionPool.connected) {
         await AbstractMssqlClient.connectionPool.connect();
       }
-      // const connection = await AbstractMssqlClient.connectionPool.connect();
       const request = AbstractMssqlClient.connectionPool.request();
 
       if (typeof input != 'undefined') {
@@ -64,8 +62,8 @@ export abstract class AbstractMssqlClient {
       } else if (isMssqlError(error)) {
         const newError = {
           error: {
-            name: error.name, // RequestError
-            description: error.message, // Timeout: Request failed to complete in 15000ms
+            name: error.name,
+            description: error.message,
           },
           originalError: {},
           query,
@@ -83,13 +81,6 @@ export abstract class AbstractMssqlClient {
         context.logger.error(this.moduleName, error.message, { error, query, input });
       }
 
-      // // TODO May want to refactor to throw CamsError and remove returning QueryResults
-      // const queryResult: QueryResults = {
-      //   results: {},
-      //   message: (error as Error).message,
-      //   success: false,
-      // };
-      // return queryResult;
       throw getCamsError(error, this.moduleName, error.message);
     }
   }

backend/functions/lib/adapters/gateways/acms/acms.gateway.ts

@@ -19,7 +19,7 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway
     // The context carries different database connection configurations.
     // We pick off the configuration specific to this ACMS gateway.
     const config = context.config.acmsDbConfig;
-    super(context, config, MODULE_NAME);
+    super(config, MODULE_NAME);
   }
 
   async getPageCount(context: ApplicationContext, predicate: AcmsPredicate): Promise<number> {
@@ -32,7 +32,7 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway
       AND CONSOLIDATED_CASE_NUMBER != '0'`;
 
     // Valid ACMS chapters: 09, 11, 12, 13, 15, 7A, 7N, AC
-    // 'AC' is the predecesor to chapter 15. We are not importing these old cases into CAMS.
+    // 'AC' is the predecessor to chapter 15. We are not importing these old cases into CAMS.
     // '7A' and '7N' are treated inclusively as chapter 7 cases when importing into CAMS.
     // Leading zero padding is added for chapter 9.
 
@@ -184,7 +184,6 @@ export class AcmsGatewayImpl extends AbstractMssqlClient implements AcmsGateway
   }
 
   private formatCaseId(caseId: string): string {
-    console.log(`Case ID: ${caseId} is of type ${typeof caseId}.`);
     const padded = caseId.padStart(10, '0');
     return `${padded.slice(0, 3)}-${padded.slice(3, 5)}-${padded.slice(5)}`;
   }

backend/functions/lib/adapters/utils/feature-flag.ts

@@ -12,6 +12,7 @@ export async function getFeatureFlags(config: ApplicationConfiguration): Promise
     eventsUri: 'https://events.launchdarkly.us',
   });
   await client.waitForInitialization();
+  //TODO: revisit if this is necessary when we look at why we need to restart the backend for feature flags
   const state = await client.allFlagsState({
     kind: 'user',
     key: 'feature-flag-migration',

backend/functions/lib/controllers/acms-orders/acms-orders.controller.test.ts

@@ -1,5 +1,5 @@
 import AcmsOrders, {
-  AcmsConsolidationReport,
+  AcmsTransformationResult,
   AcmsPredicate,
   AcmsPredicateAndPage,
 } from '../../use-cases/acms-orders/acms-orders';
@@ -45,8 +45,9 @@ describe('AcmsOrdersController', () => {
 
   test('should return processing report from migrateConsolidation', async () => {
     const leadCaseId = '811100000';
-    const report: AcmsConsolidationReport = {
+    const report: AcmsTransformationResult = {
       leadCaseId,
+      childCaseCount: 1,
       success: true,
     };
     const spy = jest.spyOn(AcmsOrders.prototype, 'migrateConsolidation').mockResolvedValue(report);

backend/functions/lib/controllers/acms-orders/acms-orders.controller.ts

@@ -1,7 +1,6 @@
 import { ApplicationContext } from '../../adapters/types/basic';
-// import { finalizeDeferrable } from '../../deferrable/finalize-deferrable';
 import AcmsOrders, {
-  AcmsConsolidationReport,
+  AcmsTransformationResult,
   AcmsPredicate,
   AcmsPredicateAndPage,
 } from '../../use-cases/acms-orders/acms-orders';
@@ -11,9 +10,8 @@ class AcmsOrdersController {
   public async migrateConsolidation(
     context: ApplicationContext,
     leadCaseId: string,
-  ): Promise<AcmsConsolidationReport> {
+  ): Promise<AcmsTransformationResult> {
     const response = this.useCase.migrateConsolidation(context, leadCaseId);
-    // await finalizeDeferrable(context);
     return response;
   }
 
@@ -22,7 +20,6 @@ class AcmsOrdersController {
     predicate: AcmsPredicate,
   ): Promise<number> {
     const response = this.useCase.getPageCount(context, predicate);
-    // await finalizeDeferrable(context);
     return response;
   }
 
@@ -31,7 +28,6 @@ class AcmsOrdersController {
     predicate: AcmsPredicateAndPage,
   ): Promise<string[]> {
     const response = this.useCase.getLeadCaseIds(context, predicate);
-    // await finalizeDeferrable(context);
     return response;
   }
 }

backend/functions/lib/deferrable/defer-close.ts

@@ -43,7 +43,6 @@ const globalAccumulator: DeferCloseAccumulator = {
 };
 
 async function closeGlobal() {
-  console.warn(`***** CLOSING ${globalAccumulator.closables.length} CLOSABLES ****`);
   closeDeferred(globalAccumulator);
 }
 

backend/functions/lib/use-cases/acms-orders/acms-orders.ts

@@ -38,11 +38,32 @@ export type AcmsConsolidation = {
   childCases: AcmsConsolidationChildCase[];
 };
 
-export type AcmsConsolidationReport = {
+export type AcmsTransformationResult = {
   leadCaseId: string;
+  childCaseCount: number;
   success: boolean;
 };
 
+export type AcmsAggregate = {
+  successful: {
+    leadCaseCount: number;
+    childCaseCount: number;
+  };
+  failed: {
+    leadCaseIds: string[];
+    leadCaseCount: number;
+    childCaseCount: number;
+  };
+};
+
+export type AcmsPageReport = AcmsAggregate & {
+  predicateAndPage: AcmsPredicateAndPage;
+};
+
+export type AcmsPartitionReport = AcmsAggregate & {
+  predicate: AcmsPredicate;
+};
+
 export class AcmsOrders {
   public async getPageCount(
     context: ApplicationContext,
@@ -63,9 +84,13 @@ export class AcmsOrders {
   public async migrateConsolidation(
     context: ApplicationContext,
     acmsLeadCaseId: string,
-  ): Promise<AcmsConsolidationReport> {
+  ): Promise<AcmsTransformationResult> {
     // TODO: Add child case count to the report??
-    const report: AcmsConsolidationReport = { leadCaseId: acmsLeadCaseId, success: true };
+    const report: AcmsTransformationResult = {
+      leadCaseId: acmsLeadCaseId,
+      childCaseCount: 0,
+      success: true,
+    };
     try {
       const casesRepo = Factory.getCasesRepository(context);
       const dxtr = Factory.getCasesGateway(context);
@@ -92,6 +117,8 @@ export class AcmsOrders {
           return acc;
         }, new Set<string>());
 
+      report.childCaseCount = exportedChildCaseIds.size;
+
       const unimportedChildCaseIds = new Set<string>();
       exportedChildCaseIds.forEach((caseId) => {
         if (!existingChildCaseIds.has(caseId)) {

backend/functions/migration/activity/migrateConsolidation.ts

@@ -2,14 +2,14 @@ import { InvocationContext } from '@azure/functions';
 import ContextCreator from '../../azure/application-context-creator';
 import AcmsOrdersController from '../../lib/controllers/acms-orders/acms-orders.controller';
 import { getCamsError } from '../../lib/common-errors/error-utilities';
-import { AcmsConsolidationReport } from '../../lib/use-cases/acms-orders/acms-orders';
+import { AcmsTransformationResult } from '../../lib/use-cases/acms-orders/acms-orders';
 
 const MODULE_NAME = 'IMPORT_ACTION_MIGRATE_CONSOLIDATION';
 
 async function migrateConsolidation(
   leadCaseId: string,
   invocationContext: InvocationContext,
-): Promise<AcmsConsolidationReport> {
+): Promise<AcmsTransformationResult> {
   const logger = ContextCreator.getLogger(invocationContext);
   const appContext = await ContextCreator.getApplicationContext({ invocationContext, logger });
   const controller = new AcmsOrdersController();

backend/functions/migration/activity/migrateConsolidations.test.ts

@@ -3,6 +3,7 @@ import AcmsOrdersController from '../../lib/controllers/acms-orders/acms-orders.
 import module from './migrateConsolidation';
 import { createMockAzureFunctionContext } from '../../azure/testing-helpers';
 import { CamsError } from '../../lib/common-errors/cams-error';
+import { AcmsTransformationResult } from '../../lib/use-cases/acms-orders/acms-orders';
 
 describe('getConsolidations test', () => {
   afterEach(() => {
@@ -11,8 +12,9 @@ describe('getConsolidations test', () => {
 
   test('should call getLeadCaseIds controller method', async () => {
     const caseId = '000-11-22222';
-    const expected = {
+    const expected: AcmsTransformationResult = {
       leadCaseId: caseId,
+      childCaseCount: 2,
       success: true,
     };
     const getLeadCaseIdsSpy = jest

backend/functions/migration/client/acms-migration-trigger.function.ts

@@ -14,21 +14,22 @@ export default async function httpStart(
   context: InvocationContext,
 ): Promise<HttpResponse> {
   const client = df.getClient(context);
-  let body: TriggerRequest;
+  let params: TriggerRequest;
   if (request.body) {
-    body = (await request.json()) as unknown as TriggerRequest;
+    params = (await request.json()) as unknown as TriggerRequest;
   }
 
-  if (!isTriggerRequest(body)) {
+  if (!isTriggerRequest(params)) {
     throw new BadRequestError(MODULE_NAME, { message: 'Missing or malformed request body.' });
   }
 
-  if (body.apiKey !== process.env.ADMIN_KEY) {
+  if (params.apiKey !== process.env.ADMIN_KEY) {
     throw new UnauthorizedError(MODULE_NAME, { message: 'API key was missing or did not match.' });
   }
 
+  delete params.apiKey;
   const instanceId: string = await client.startNew(MAIN_ORCHESTRATOR, {
-    input: body,
+    input: params,
   });
 
   return client.createCheckStatusResponse(request, instanceId);

backend/functions/migration/orchestration/orchestrator.ts

@@ -1,4 +1,9 @@
-import { AcmsBounds, AcmsPredicate } from '../../lib/use-cases/acms-orders/acms-orders';
+import {
+  AcmsAggregate,
+  AcmsBounds,
+  AcmsPartitionReport,
+  AcmsPredicate,
+} from '../../lib/use-cases/acms-orders/acms-orders';
 import { FLATTEN_BOUNDING_ARRAYS, SUB_ORCHESTRATOR_PAGING } from '../loadConsolidations';
 import { OrchestrationContext } from 'durable-functions';
 
@@ -18,4 +23,23 @@ export function* main(context: OrchestrationContext) {
     );
   }
   yield context.df.Task.all(provisioningTasks);
+
+  const summary = provisioningTasks.reduce(
+    (acc, task) => {
+      const report = task.result as AcmsPartitionReport;
+      context.log('Report', JSON.stringify(report));
+      acc.successful.leadCaseCount += report.successful.leadCaseCount;
+      acc.successful.childCaseCount += report.successful.childCaseCount;
+      acc.failed.leadCaseIds.push(...report.failed.leadCaseIds);
+      acc.failed.leadCaseCount += report.failed.leadCaseCount;
+      acc.failed.childCaseCount += report.failed.childCaseCount;
+      return acc;
+    },
+    {
+      successful: { leadCaseCount: 0, childCaseCount: 0 },
+      failed: { leadCaseIds: [], leadCaseCount: 0, childCaseCount: 0 },
+    } as AcmsAggregate,
+  );
+
+  context.log('Summary', JSON.stringify(bounds), JSON.stringify(summary));
 }

backend/functions/migration/orchestration/sub-orchestrator-etl.ts

@@ -1,6 +1,7 @@
 import {
-  AcmsConsolidationReport,
+  AcmsTransformationResult,
   AcmsPredicateAndPage,
+  AcmsPageReport,
 } from '../../lib/use-cases/acms-orders/acms-orders';
 import { GET_CONSOLIDATIONS, MIGRATE_CONSOLIDATION } from '../loadConsolidations';
 import { OrchestrationContext } from 'durable-functions';
@@ -18,19 +19,23 @@ export function* subOrchestratorETL(context: OrchestrationContext) {
 
   yield context.df.Task.all(etlTasks);
 
-  const finalResults = etlTasks.reduce(
+  return etlTasks.reduce(
     (acc, task) => {
-      const taskResponse = task as AcmsConsolidationReport;
-      if (taskResponse.success) {
-        acc.successful += 1;
+      const transformationResult = task.result as AcmsTransformationResult;
+      if (transformationResult.success) {
+        acc.successful.leadCaseCount += 1;
+        acc.successful.childCaseCount += transformationResult.childCaseCount;
       } else {
-        acc.failed += 1;
+        acc.failed.leadCaseIds.push(transformationResult.leadCaseId);
+        acc.failed.leadCaseCount += 1;
+        acc.failed.childCaseCount += transformationResult.childCaseCount;
       }
       return acc;
     },
-    { successful: 0, failed: 0 },
-  );
-  context.log(
-    `ACMS Consolidation Migration ETL: successful: ${finalResults.successful}, failures: ${finalResults.failed}`,
+    {
+      predicateAndPage,
+      successful: { leadCaseCount: 0, childCaseCount: 0 },
+      failed: { leadCaseIds: [], leadCaseCount: 0, childCaseCount: 0 },
+    } as AcmsPageReport,
   );
 }