Stacky Glue / BCLConvert InteropQC / cttsoV2 Fixes / Deprecate ICAv2 Copy Batch Utility

config/config.ts

@@ -9,7 +9,6 @@ import { getMetadataManagerStackProps } from './stacks/metadataManager';
 import { getSequenceRunManagerStackProps } from './stacks/sequenceRunManager';
 import { getFileManagerStackProps } from './stacks/fileManager';
 import { getBsRunsUploadManagerStackProps } from './stacks/bsRunsUploadManager';
-import { getICAv2CopyBatchUtilityStackProps } from './stacks/icav2CopyBatchUtility';
 import { getBsshIcav2FastqCopyManagerStackProps } from './stacks/bsshIcav2FastqCopyManager';
 import {
   getCttsov2Icav2PipelineManagerStackProps,
@@ -22,6 +21,11 @@ import {
   getBclConvertManagerStackProps,
 } from './stacks/bclConvertManager';
 import { getWorkflowManagerStackProps } from './stacks/workflowRunManager';
+import {
+  getBclconvertInteropQcIcav2PipelineManagerStackProps,
+  getBclconvertInteropQcIcav2PipelineTableStackProps,
+} from './stacks/bclconvertInteropQcIcav2PipelineManager';
+import { getGlueStackProps, getStatefulGlueStackProps } from './stacks/stackyMcStackFace';
 
 interface EnvironmentConfig {
   name: string;
@@ -51,22 +55,27 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
             sharedStackProps: getSharedStackProps(stage),
             tokenServiceStackProps: getTokenServiceStackProps(),
             icaEventPipeStackProps: getIcaEventPipeStackProps(),
+            bclconvertInteropQcIcav2PipelineTableStackProps:
+              getBclconvertInteropQcIcav2PipelineTableStackProps(),
             cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(),
             BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
+            stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
           },
           statelessConfig: {
             postgresManagerStackProps: getPostgresManagerStackProps(),
             metadataManagerStackProps: getMetadataManagerStackProps(),
             sequenceRunManagerStackProps: getSequenceRunManagerStackProps(),
             fileManagerStackProps: getFileManagerStackProps(stage),
             bsRunsUploadManagerStackProps: getBsRunsUploadManagerStackProps(stage),
-            icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(stage),
             bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(stage),
+            bclconvertInteropQcIcav2PipelineManagerStackProps:
+              getBclconvertInteropQcIcav2PipelineManagerStackProps(stage),
             cttsov2Icav2PipelineManagerStackProps: getCttsov2Icav2PipelineManagerStackProps(stage),
             eventSchemaStackProps: getEventSchemaStackProps(),
             dataSchemaStackProps: getDataSchemaStackProps(),
             bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
             workflowManagerStackProps: getWorkflowManagerStackProps(),
+            stackyMcStackFaceProps: getGlueStackProps(),
           },
         },
       };
@@ -81,22 +90,27 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
             sharedStackProps: getSharedStackProps(stage),
             tokenServiceStackProps: getTokenServiceStackProps(),
             icaEventPipeStackProps: getIcaEventPipeStackProps(),
+            bclconvertInteropQcIcav2PipelineTableStackProps:
+              getBclconvertInteropQcIcav2PipelineTableStackProps(),
             cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(),
             BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
+            stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
           },
           statelessConfig: {
             postgresManagerStackProps: getPostgresManagerStackProps(),
             metadataManagerStackProps: getMetadataManagerStackProps(),
             sequenceRunManagerStackProps: getSequenceRunManagerStackProps(),
             fileManagerStackProps: getFileManagerStackProps(stage),
             bsRunsUploadManagerStackProps: getBsRunsUploadManagerStackProps(stage),
-            icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(stage),
             bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(stage),
+            bclconvertInteropQcIcav2PipelineManagerStackProps:
+              getBclconvertInteropQcIcav2PipelineManagerStackProps(stage),
             cttsov2Icav2PipelineManagerStackProps: getCttsov2Icav2PipelineManagerStackProps(stage),
             eventSchemaStackProps: getEventSchemaStackProps(),
             dataSchemaStackProps: getDataSchemaStackProps(),
             bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
             workflowManagerStackProps: getWorkflowManagerStackProps(),
+            stackyMcStackFaceProps: getGlueStackProps(),
           },
         },
       };
@@ -111,22 +125,27 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
             sharedStackProps: getSharedStackProps(stage),
             tokenServiceStackProps: getTokenServiceStackProps(),
             icaEventPipeStackProps: getIcaEventPipeStackProps(),
+            bclconvertInteropQcIcav2PipelineTableStackProps:
+              getBclconvertInteropQcIcav2PipelineTableStackProps(),
             cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(),
             BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
+            stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
           },
           statelessConfig: {
             postgresManagerStackProps: getPostgresManagerStackProps(),
             metadataManagerStackProps: getMetadataManagerStackProps(),
             sequenceRunManagerStackProps: getSequenceRunManagerStackProps(),
             fileManagerStackProps: getFileManagerStackProps(stage),
             bsRunsUploadManagerStackProps: getBsRunsUploadManagerStackProps(stage),
-            icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(stage),
             bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(stage),
+            bclconvertInteropQcIcav2PipelineManagerStackProps:
+              getBclconvertInteropQcIcav2PipelineManagerStackProps(stage),
             cttsov2Icav2PipelineManagerStackProps: getCttsov2Icav2PipelineManagerStackProps(stage),
             eventSchemaStackProps: getEventSchemaStackProps(),
             dataSchemaStackProps: getDataSchemaStackProps(),
             bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
             workflowManagerStackProps: getWorkflowManagerStackProps(),
+            stackyMcStackFaceProps: getGlueStackProps(),
           },
         },
       };

config/constants.ts

@@ -21,6 +21,9 @@ export const accountIdAlias: Record<AppStage, string> = {
 // external ICA constants
 export const icaAwsAccountNumber = '079623148045';
 
+// Name of the Ica Event Pipe stack
+export const icaEventPipeStackName = 'IcaEventPipeStack';
+
 // upstream infra: vpc
 const vpcName = 'main-vpc';
 const vpcStackName = 'networking';
@@ -110,36 +113,58 @@ export const icav2AccessTokenSecretName: Record<AppStage, string> = {
 };
 
 /*
-Resources generated by the ICAv2 Copy Batch utility stack
- */
-export const icav2CopyBatchSSMRoot = '/orcabus/icav2_copy_batch_utility';
-export const icav2CopyBatchSSMBatchArn = path.join(icav2CopyBatchSSMRoot, 'batch_sfn_arn');
-export const icav2CopyBatchSSMBatchName = path.join(icav2CopyBatchSSMRoot, 'batch_sfn_name');
-export const icav2CopyBatchSSMSingleArn = path.join(icav2CopyBatchSSMRoot, 'single_sfn_arn');
-export const icav2CopyBatchSSMSingleName = path.join(icav2CopyBatchSSMRoot, 'single_sfn_name');
-export const icav2CopyBatchUtilityName = 'icav2_copy_batch_utility_sfn';
-export const icav2CopySingleUtilityName = 'icav2_copy_single_utility_sfn';
+Resources generated by the BSSH Fastq Copy Manager
+*/
+
+export const bsshFastqCopyManagerWorkflowName = 'bsshFastqCopy';
+export const bsshFastqCopyManagerWorkflowTypeVersion = '1.0.0';
+export const bsshFastqCopyManagerServiceVersion = '2024.05.15';
+
+export const bsshFastqCopyManagerReadyEventSource = 'orcabus.workflowmanager';
+export const bsshFastqCopyManagerEventSource = 'orcabus.bsshfastqcopymanager';
+export const bsshFastqCopyManagerEventDetailType = 'WorkflowRunStateChange';
 
 /*
-Resources generated by the BSSH Fastq Copy Manager
- */
+Resources generated by the BCLConvert InterOp QC pipeline
+*/
+
+export const bclConvertInteropQcSSMRoot = '/orcabus/bclconvert_interop_qc';
+
+export const bclconvertInteropQcIcav2PipelineManagerDynamodbTableName =
+  'bclconvertInteropQcICAv2AnalysesDynamoDBTable';
 
-export const bsshFastqCopyManagerSSMRoot = '/orcabus/bssh_icav2_fastq_copy_manager';
-export const bsshFastqCopyManagerSSMName = path.join(
-  bsshFastqCopyManagerSSMRoot,
-  'state_machine_name'
+// Stateful
+export const bclconvertInteropQcDynamoDbTableSSMName = path.join(
+  bclConvertInteropQcSSMRoot,
+  'dynamodb_table_name'
 );
-export const bsshFastqCopyManagerSSMArn = path.join(
-  bsshFastqCopyManagerSSMRoot,
-  'state_machine_arn'
+export const bclconvertInteropQcDynamoDbTableSSMArn = path.join(
+  bclConvertInteropQcSSMRoot,
+  'dynamodb_table_arn'
 );
-export const bsshFastqCopyManagerSfnName = 'bssh_fastq_copy_manager_sfn';
+
+// Stateless
+
+export const bclconvertInteropQcIcav2PipelineWorkflowName = 'bclconvert_interop_qc';
+export const bclconvertInteropQcIcav2PipelineWorkflowTypeVersion = '1.3.1--1.21';
+export const bclconvertInteropQcIcav2ServiceVersion = '2024.05.07';
+
+export const bclconvertInteropQcIcav2ReadyEventSource = 'orcabus.workflowmanager';
+export const bclconvertInteropQcIcav2EventSource = 'orcabus.bclconvertinteropqc';
+export const bclconvertInteropQcIcav2EventDetailType = 'WorkflowRunStateChange';
+export const bclconvertInteropQcStateMachinePrefix = 'bclconvertInteropQcSfn';
+
+/*
+Resources used by the bclConvert InteropQc Pipeline
+*/
+export const bclconvertInteropQcIcav2PipelineIdSSMParameterPath =
+  '/icav2/umccr-prod/bclconvert_interop_qc_pipeline_id';
 
 /*
 External resources required by the ctTSO v2 Stack
 */
 export const cttsov2Icav2PipelineIdSSMParameterPath =
-  '/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id';
+  '/icav2/umccr-prod/tso500_ctdna_2.5_pipeline_id';
 
 /*
 Resources generated by the ctTSO v2 Stack
@@ -157,16 +182,25 @@ export const cttsov2DynamoDbTableSSMArn = path.join(
   'dynamodb_table_arn'
 );
 
-// Stateless
-export const cttsov2Icav2PipelineSfnSSMName = path.join(
-  cttsov2Icav2PipelineManagerSSMRoot,
-  'state_machine_name'
-);
-export const cttsov2Icav2PipelineSfnSSMArn = path.join(
-  cttsov2Icav2PipelineManagerSSMRoot,
-  'state_machine_arn'
-);
+// Event Handling
 
 export const cttsov2Icav2PipelineWorkflowType = 'cttsov2';
-export const cttsov2Icav2PipelineWorkflowTypeVersion = '2.1.1';
+export const cttsov2Icav2PipelineWorkflowTypeVersion = '2.5.0';
 export const cttsov2Icav2ServiceVersion = '2024.05.07';
+
+export const cttsov2Icav2ReadyEventSource = 'orcabus.workflowmanager';
+export const cttsov2Icav2EventSource = 'orcabus.cttsov2';
+export const cttsov2Icav2EventDetailType = 'WorkflowRunStateChange';
+export const cttsov2StateMachinePrefix = 'cttsov2Sfn';
+
+// Mock Stack
+export const mockEventBusName = eventBusName;
+export const mockInstrumentRunTableName = 'stacky-instrument-run-table';
+export const mockInputMakerTableName = 'stacky-input-maker-table';
+export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table';
+export const mockIcav2ProjectIdSsmParameterName = '/orcabus/stacky/icav2_project_id_and_name_json';
+export const mockPrimaryOutputUriSsmParameterName = '/orcabus/stacky/primary_output_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/primary_data/__instrument_run_id__/__portal_run_id__/
+export const mockAnalysisOutputUriSsmParameterName = '/orcabus/stacky/analysis_output_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_data/__workflow_name__/__workflow_version__/__portal_run_id__/
+export const mockAnalysisLogsUriSsmParameterName = '/orcabus/stacky/analysis_logs_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_logs/__workflow_name__/__workflow_version__/__portal_run_id__/
+
+export const mockAnalysisCacheUriSsmParameterName = '/orcabus/stacky/analysis_cache_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_cache/__workflow_name__/__workflow_version__/__portal_run_id__/

config/stacks/bclconvertInteropQcIcav2PipelineManager.ts

@@ -0,0 +1,57 @@
+import {
+  AppStage,
+  eventBusName,
+  bclconvertInteropQcIcav2PipelineIdSSMParameterPath,
+  bclconvertInteropQcIcav2PipelineManagerDynamodbTableName,
+  icav2AccessTokenSecretName,
+  bclconvertInteropQcDynamoDbTableSSMArn,
+  bclconvertInteropQcDynamoDbTableSSMName,
+  bclconvertInteropQcIcav2PipelineWorkflowName,
+  bclconvertInteropQcIcav2ServiceVersion,
+  bclconvertInteropQcIcav2PipelineWorkflowTypeVersion,
+  bclconvertInteropQcIcav2EventSource,
+  bclconvertInteropQcIcav2EventDetailType,
+  bclconvertInteropQcStateMachinePrefix,
+  bclconvertInteropQcIcav2ReadyEventSource,
+  icaEventPipeStackName,
+} from '../constants';
+import { BclconvertInteropQcIcav2PipelineManagerConfig } from '../../lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/deploy/stack';
+import { BclconvertInteropQcIcav2PipelineTableConfig } from '../../lib/workload/stateful/stacks/bclconvert-interop-qc-pipeline-dynamo-db/deploy/stack';
+
+// Stateful
+export const getBclconvertInteropQcIcav2PipelineTableStackProps =
+  (): BclconvertInteropQcIcav2PipelineTableConfig => {
+    return {
+      bclconvertInteropQcIcav2DynamodbTableArnSsmParameterPath:
+        bclconvertInteropQcDynamoDbTableSSMArn,
+      bclconvertInteropQcIcav2DynamodbTableNameSsmParameterPath:
+        bclconvertInteropQcDynamoDbTableSSMName,
+      dynamodbTableName: bclconvertInteropQcIcav2PipelineManagerDynamodbTableName,
+    };
+  };
+
+// Stateless
+export const getBclconvertInteropQcIcav2PipelineManagerStackProps = (
+  stage: AppStage
+): BclconvertInteropQcIcav2PipelineManagerConfig => {
+  return {
+    /* ICAv2 Configurations */
+    icav2TokenSecretId: icav2AccessTokenSecretName[stage],
+    pipelineIdSsmPath: bclconvertInteropQcIcav2PipelineIdSSMParameterPath,
+    /* Stateful names */
+    dynamodbTableName: bclconvertInteropQcIcav2PipelineManagerDynamodbTableName,
+    eventBusName: eventBusName,
+    icaEventPipeName: `${icaEventPipeStackName}Pipe`,
+    /* Event configurations */
+    workflowName: bclconvertInteropQcIcav2PipelineWorkflowName,
+    workflowVersion: bclconvertInteropQcIcav2PipelineWorkflowTypeVersion,
+    serviceVersion: bclconvertInteropQcIcav2ServiceVersion,
+    triggerLaunchSource: bclconvertInteropQcIcav2ReadyEventSource,
+    internalEventSource: bclconvertInteropQcIcav2EventSource,
+    detailType: bclconvertInteropQcIcav2EventDetailType,
+    /*
+    Names for statemachines
+    */
+    stateMachinePrefix: bclconvertInteropQcStateMachinePrefix,
+  };
+};

config/stacks/bsshIcav2FastqCopyManager.ts

@@ -1,22 +1,27 @@
 import {
   AppStage,
   icav2AccessTokenSecretName,
-  bsshFastqCopyManagerSfnName,
-  bsshFastqCopyManagerSSMName,
-  bsshFastqCopyManagerSSMArn,
   eventBusName,
+  bsshFastqCopyManagerWorkflowName,
+  bsshFastqCopyManagerWorkflowTypeVersion,
+  bsshFastqCopyManagerServiceVersion,
+  bsshFastqCopyManagerReadyEventSource,
+  bsshFastqCopyManagerEventSource,
+  bsshFastqCopyManagerEventDetailType,
 } from '../constants';
 import { BsshIcav2FastqCopyManagerConfig } from '../../lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/deploy/stack';
 
 export const getBsshIcav2FastqCopyManagerStackProps = (
   stage: AppStage
 ): BsshIcav2FastqCopyManagerConfig => {
   return {
-    icav2CopyBatchUtilityStateMachineName: bsshFastqCopyManagerSfnName,
-    bsshIcav2FastqCopyManagerStateMachineName: bsshFastqCopyManagerSfnName,
-    bsshIcav2FastqCopyManagerStateMachineNameSsmParameterPath: bsshFastqCopyManagerSSMName,
-    bsshIcav2FastqCopyManagerStateMachineArnSsmParameterPath: bsshFastqCopyManagerSSMArn,
+    detailType: bsshFastqCopyManagerEventDetailType,
+    icav2TokenSecretId: icav2AccessTokenSecretName[stage],
+    internalEventSource: bsshFastqCopyManagerEventSource,
+    serviceVersion: bsshFastqCopyManagerServiceVersion,
+    triggerLaunchSource: bsshFastqCopyManagerReadyEventSource,
+    workflowName: bsshFastqCopyManagerWorkflowName,
+    workflowVersion: bsshFastqCopyManagerWorkflowTypeVersion,
     eventBusName: eventBusName,
-    icav2JwtSecretsManagerPath: icav2AccessTokenSecretName[stage],
   };
 };

config/stacks/cttsov2Icav2PipelineManager.ts

@@ -2,16 +2,18 @@ import {
   AppStage,
   eventBusName,
   cttsov2Icav2PipelineIdSSMParameterPath,
-  icav2CopyBatchUtilityName,
-  cttsov2Icav2PipelineSfnSSMName,
-  cttsov2Icav2PipelineSfnSSMArn,
   cttsov2Icav2PipelineManagerDynamodbTableName,
   icav2AccessTokenSecretName,
   cttsov2DynamoDbTableSSMArn,
   cttsov2DynamoDbTableSSMName,
   cttsov2Icav2PipelineWorkflowType,
   cttsov2Icav2ServiceVersion,
   cttsov2Icav2PipelineWorkflowTypeVersion,
+  icaEventPipeStackName,
+  cttsov2Icav2ReadyEventSource,
+  cttsov2Icav2EventSource,
+  cttsov2Icav2EventDetailType,
+  cttsov2StateMachinePrefix,
 } from '../constants';
 import { Cttsov2Icav2PipelineManagerConfig } from '../../lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack';
 import { Cttsov2Icav2PipelineTableConfig } from '../../lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack';
@@ -30,15 +32,22 @@ export const getCttsov2Icav2PipelineManagerStackProps = (
   stage: AppStage
 ): Cttsov2Icav2PipelineManagerConfig => {
   return {
-    pipelineIdSsmPath: cttsov2Icav2PipelineIdSSMParameterPath,
-    icav2CopyBatchUtilityStateMachineName: icav2CopyBatchUtilityName,
-    cttsov2LaunchStateMachineArnSsmParameterPath: cttsov2Icav2PipelineSfnSSMArn,
-    cttsov2LaunchStateMachineNameSsmParameterPath: cttsov2Icav2PipelineSfnSSMName,
+    /* ICAv2 Pipeline analysis essentials */
+    icav2TokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn"
+    pipelineIdSsmPath: cttsov2Icav2PipelineIdSSMParameterPath, // List of parameters the workflow session state machine will need access to
+    /* Table to store analyis metadata */
     dynamodbTableName: cttsov2Icav2PipelineManagerDynamodbTableName,
-    eventbusName: eventBusName,
-    icav2TokenSecretId: icav2AccessTokenSecretName[stage],
+    /* Internal and external buses */
+    eventBusName: eventBusName,
+    icaEventPipeName: `${icaEventPipeStackName}Pipe`,
+    /* Event handling */
     workflowType: cttsov2Icav2PipelineWorkflowType,
     workflowVersion: cttsov2Icav2PipelineWorkflowTypeVersion,
     serviceVersion: cttsov2Icav2ServiceVersion,
+    triggerLaunchSource: cttsov2Icav2ReadyEventSource,
+    internalEventSource: cttsov2Icav2EventSource,
+    detailType: cttsov2Icav2EventDetailType,
+    /* Names for statemachines */
+    stateMachinePrefix: cttsov2StateMachinePrefix,
   };
 };