diff --git a/.secrets.baseline b/.secrets.baseline index eabe339a9..d8f9c2403 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -75,6 +75,10 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -109,10 +113,10 @@ { "path": "detect_secrets.filters.regex.should_exclude_file", "pattern": [ - "^(yarn.lock|.yarn/|.local/|openapi/)" + "^(yarn.lock|.yarn/|.local/|openapi/)|.sqlx/" ] } ], "results": {}, - "generated_at": "2023-05-24T11:39:46Z" + "generated_at": "2024-02-18T03:27:13Z" } diff --git a/config/constants.ts b/config/constants.ts index d5137f110..c38d7a3eb 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -36,6 +36,7 @@ const orcaBusStatefulConfig = { cloudwatchLogsExports: ['orcabus-postgresql'], }, databaseSecurityGroupName: 'database-security-group', + inboundSecurityGroupName: 'inbound-database-security-group', }, securityGroupProps: { securityGroupName: lambdaSecurityGroupName, @@ -82,7 +83,7 @@ const filemanagerDependencies: FilemanagerDependencies = { eventSourceBuckets: ['umccr-temp-dev'], eventSourceQueueName: eventSourceConfig.queueName, databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName, - databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.databaseSecurityGroupName, + databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.inboundSecurityGroupName, }; interface EnvironmentConfig { diff --git a/lib/workload/orcabus-stateful-stack.ts b/lib/workload/orcabus-stateful-stack.ts index 990f41753..532b6bbb4 100644 --- a/lib/workload/orcabus-stateful-stack.ts +++ b/lib/workload/orcabus-stateful-stack.ts @@ -2,7 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { EventBusConstruct, EventBusProps } from './stateful/eventbridge/component'; -import { Database, DatabasePropsNoVPC } from './stateful/database/component'; +import { Database, ConfigurableDatabaseProps } from './stateful/database/component'; import { SecurityGroupConstruct, SecurityGroupProps } from './stateful/securitygroup/component'; import { SchemaRegistryConstruct, SchemaRegistryProps } from './stateful/schemaregistry/component'; import { EventSource, EventSourceProps } from './stateful/event_source/component'; @@ -10,7 +10,7 @@ import { EventSource, EventSourceProps } from './stateful/event_source/component export interface OrcaBusStatefulConfig { schemaRegistryProps: SchemaRegistryProps; eventBusProps: EventBusProps; - databaseProps: DatabasePropsNoVPC; + databaseProps: ConfigurableDatabaseProps; securityGroupProps: SecurityGroupProps; eventSourceProps?: EventSourceProps; } diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index 59f920e36..61df62e02 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -78,7 +78,7 @@ export class OrcaBusStatelessStack extends cdk.Stack { // Opting to reconstruct the dependencies here, and pass them into the service as constructs. const queue = Queue.fromQueueArn( this, - 'Filemanager Queue', + 'FilemanagerQueue', Arn.format( { resource: dependencies.eventSourceQueueName, @@ -87,23 +87,23 @@ export class OrcaBusStatelessStack extends cdk.Stack { this ) ); - const securityGroup = SecurityGroup.fromLookupByName( + const databaseSecurityGroup = SecurityGroup.fromLookupByName( this, - 'Filemanager Database Security Group', + 'FilemanagerDatabaseSecurityGroup', dependencies.databaseSecurityGroupName, vpc ); const databaseSecret = Secret.fromSecretNameV2( this, - 'Filemanager Database Secret', + 'FilemanagerDatabaseSecret', dependencies.databaseSecretName ); new Filemanager(this, 'Filemanager', { buckets: dependencies.eventSourceBuckets, buildEnvironment: {}, - databaseSecret: databaseSecret, - databaseSecurityGroup: securityGroup, + databaseSecret, + databaseSecurityGroup, eventSources: [queue], migrateDatabase: true, vpc: vpc, diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index abc3f6290..624bfee0b 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -2,6 +2,8 @@ import { Construct } from 'constructs'; import { RemovalPolicy, Duration } from 'aws-cdk-lib'; import * as rds from 'aws-cdk-lib/aws-rds'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { SecurityGroup } from 'aws-cdk-lib/aws-ec2'; +import { DatabaseCluster } from 'aws-cdk-lib/aws-rds'; /** * Props for enabling enhanced monitoring. @@ -28,7 +30,7 @@ type MonitoringProps = { /** * Database props without a VPC. */ -export type DatabasePropsNoVPC = MonitoringProps & { +export type ConfigurableDatabaseProps = MonitoringProps & { /** * The cluster identifier. */ @@ -78,15 +80,16 @@ export type DatabasePropsNoVPC = MonitoringProps & { */ removalPolicy: RemovalPolicy; /** - * Inbound security groups that are allowed to connect to the database. + * Create an inbound security group that can connect to the database. Stateless resources can recreate + * this security group to access the database. */ - allowedInboundSG?: ec2.SecurityGroup; + inboundSecurityGroupName: string; }; /** * Database props with a vpc. */ -export type DatabaseProps = DatabasePropsNoVPC & { +export type DatabaseProps = ConfigurableDatabaseProps & { /** * The database VPC. */ @@ -94,8 +97,9 @@ export type DatabaseProps = DatabasePropsNoVPC & { }; export class Database extends Construct { - readonly securityGroup: ec2.SecurityGroup; - readonly cluster: rds.DatabaseCluster; + readonly securityGroup: SecurityGroup; + readonly inboundSecurityGroup: SecurityGroup; + readonly cluster: DatabaseCluster; constructor(scope: Construct, id: string, props: DatabaseProps) { super(scope, id); @@ -113,14 +117,19 @@ export class Database extends Construct { description: 'security group for OrcaBus RDS', }); - // give compute sg to access the rds - if (props.allowedInboundSG) { - this.securityGroup.addIngressRule( - props.allowedInboundSG, - ec2.Port.tcp(props.dbPort), - 'allow the OrcaBus compute sg to access db' - ); - } + this.inboundSecurityGroup = new ec2.SecurityGroup(this, 'DbInboundSecurityGroup', { + vpc: props.vpc, + allowAllOutbound: false, + allowAllIpv6Outbound: false, + securityGroupName: props.inboundSecurityGroupName, + description: 'an inbound security group to connect to the OrcaBus RDS', + }); + + this.securityGroup.addIngressRule( + this.inboundSecurityGroup, + ec2.Port.tcp(props.dbPort), + 'allow the OrcaBus security group to access db' + ); this.cluster = new rds.DatabaseCluster(this, id + 'Cluster', { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: props.version }), diff --git a/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json b/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json new file mode 100644 index 000000000..ba27d4bf6 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json @@ -0,0 +1,117 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Update the matching s3_objects which should be re-ordered based on the deleted event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::text[],\n $6::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n deleted_date,\n version_id,\n deleted_sequencer\n )\n),\n-- Then, select the objects that match the bucket, key and version_id\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.deleted_sequencer as input_deleted_sequencer,\n input.deleted_date as input_deleted_date\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id = s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a deleted\n -- sequencer that is closer to the created sequencer.\n current_objects.created_sequencer < current_objects.input_deleted_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.deleted_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer would belong to another object.\n current_objects.deleted_sequencer > current_objects.input_deleted_sequencer\n )\n -- And there should not be any objects with a deleted sequencer that is the same as the input deleted\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_deleted_sequencer not in (\n select deleted_sequencer from current_objects where deleted_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set deleted_sequencer = objects_to_update.input_deleted_sequencer,\n deleted_date = objects_to_update.input_deleted_date,\n number_reordered = s3_object.number_reordered +\n case when objects_to_update.deleted_sequencer is null then 0 else 1 end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n deleted_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n deleted_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order deleted event, so return a deleted event back.\n 'Deleted' as \"event_type!: EventType\"\nfrom objects_to_update;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "s3_object_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "bucket", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "event_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_modified_date", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "e_tag", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "storage_class?: StorageClass", + "type_info": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "version_id", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "sequencer", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "number_reordered", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "number_duplicate_events", + "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "size", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "event_type!: EventType", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null, + false, + false, + true, + true, + true, + true, + true, + true, + false, + false, + true, + null + ] + }, + "hash": "6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json b/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json new file mode 100644 index 000000000..81955501e --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[])\n) on conflict on constraint created_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "object_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "number_duplicate_events", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json b/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json new file mode 100644 index 000000000..ee9c54553 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n deleted_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n deleted_sequencer,\n number_reordered\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[]),\n unnest($13::integer[])\n) on conflict on constraint deleted_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "object_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "number_duplicate_events", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray", + "Int4Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json b/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json new file mode 100644 index 000000000..6a44fe521 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of objects\ninsert into object (object_id)\nvalues (\n unnest($1::uuid[])\n);\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json b/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json new file mode 100644 index 000000000..34ab88e3f --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json @@ -0,0 +1,147 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Update the matching s3_objects which should be re-ordered based on the created event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::integer[],\n $6::text[],\n $7::timestamptz[],\n $8::text[],\n $9::storage_class[],\n $10::text[],\n $11::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n )\n),\n-- Then, select the objects that need to be updated.\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.created_sequencer as input_created_sequencer,\n input.created_date as input_created_date,\n input.size as input_size,\n input.checksum as input_checksum,\n input.last_modified_date as input_last_modified_date,\n input.e_tag as input_e_tag,\n input.storage_class as input_storage_class\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id = s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a created\n -- sequencer that is closer to the deleted sequencer.\n current_objects.deleted_sequencer > current_objects.input_created_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.created_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer could belong to another object.\n current_objects.created_sequencer < current_objects.input_created_sequencer\n )\n -- And there should not be any objects with a created sequencer that is the same as the input created\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_created_sequencer not in (\n select created_sequencer from current_objects where created_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set created_sequencer = objects_to_update.input_created_sequencer,\n created_date = objects_to_update.input_created_date,\n size = coalesce(objects_to_update.input_size, objects_to_update.size),\n checksum = coalesce(objects_to_update.input_checksum, objects_to_update.checksum),\n last_modified_date = coalesce(objects_to_update.input_last_modified_date, objects_to_update.last_modified_date),\n e_tag = coalesce(objects_to_update.e_tag, objects_to_update.e_tag),\n storage_class = objects_to_update.storage_class,\n number_reordered = s3_object.number_reordered +\n -- Note the asymmetry between this and the reorder for deleted query.\n case when objects_to_update.deleted_sequencer is not null or objects_to_update.created_sequencer is not null then\n 1\n else\n 0\n end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n created_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n created_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order created event, so return a created event back.\n 'Created' as \"event_type!: EventType\"\nfrom objects_to_update;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "s3_object_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "bucket", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "event_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_modified_date", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "e_tag", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "storage_class?: StorageClass", + "type_info": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "version_id", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "sequencer", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "number_reordered", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "number_duplicate_events", + "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "size", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "event_type!: EventType", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null, + false, + false, + true, + true, + true, + true, + true, + true, + false, + false, + true, + null + ] + }, + "hash": "c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6" +} diff --git a/lib/workload/stateless/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile index 483123c4c..ef946520a 100644 --- a/lib/workload/stateless/filemanager/Makefile +++ b/lib/workload/stateless/filemanager/Makefile @@ -36,14 +36,17 @@ clippy: build check: lint clippy check-fix: lint-fix clippy - ## Clean clean: docker-clean @cargo clean -## Connect to the database +## Database related targets psql: @docker exec -it filemanager_db psql filemanager -U filemanager +prepare: + cargo sqlx prepare --workspace +prepare-check: + cargo sqlx prepare --workspace --check ## Help text help: diff --git a/lib/workload/stateless/filemanager/README.md b/lib/workload/stateless/filemanager/README.md index 0a795176d..c4872b8d2 100644 --- a/lib/workload/stateless/filemanager/README.md +++ b/lib/workload/stateless/filemanager/README.md @@ -42,10 +42,16 @@ or cargo install sccache && export RUSTC_WRAPPER=`which sccache` ``` -Then, cargo-watch can be used to recompile files as they change: +The sqlx-cli is also required for compiling code without a database running: ```sh -cargo install cargo-watch sqlx-cli +cargo install sqlx-cli +``` + +cargo-watch can be used to recompile files as they change: + +```sh +cargo install cargo-watch make watch ``` @@ -82,6 +88,38 @@ Which runs `cargo test -- --ignored`. ## Database +### Offline compilation + +In order to compile code without a local database running, queries should be prepared using `sqlx-cli`: + +```sh +make prepare +``` + +which runs: + +```sh +cargo sqlx prepare --workspace +``` + +This creates a `.sqlx` file in the workspace directory which contains the pre-compiled query information. +This is useful for deploying the orcabus code without a dependency on the filemanager's Docker compose, and it also +ensures that cdk tests which synthesize the filemanager stack without additional modifications. + +In order to check if this file is out of date, run: + +```sh +make prepare-check +``` + +which runs: + +```sh +cargo sqlx prepare --workspace --check +``` + +### Database connection + To connect to the local postgres database, run: ```bash diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 80c77bda0..67c9202fc 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -7,10 +7,24 @@ import { Architecture, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +/** + * Properties for the database. + */ +export type DatabaseProps = { + /** + * The database secret. + */ + readonly databaseSecret: ISecret; + /** + * The database security group. + */ + readonly databaseSecurityGroup: ISecurityGroup; +} + /** * Props for a Rust function without the package. */ -export type FunctionPropsNoPackage = { +export type FunctionPropsNoPackage = DatabaseProps & { /** * Additional build environment variables when building the Lambda function. */ @@ -23,14 +37,6 @@ export type FunctionPropsNoPackage = { * Vpc for the function. */ readonly vpc: IVpc; - /** - * The database secret. - */ - readonly databaseSecret: ISecret; - /** - * The database security group. - */ - readonly databaseSecurityGroup: ISecurityGroup; }; /** diff --git a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts index c4a943586..fd051fc09 100644 --- a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts +++ b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts @@ -3,26 +3,18 @@ import { IngestFunction, IngestFunctionProps } from '../constructs/functions/ing import { CdkResourceInvoke } from '../../../functions/cdk_resource_invoke'; import { MigrateFunction } from '../constructs/functions/migrate'; import * as fn from '../constructs/functions/function'; -import { ISecurityGroup, IVpc } from 'aws-cdk-lib/aws-ec2'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; -import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { DatabaseProps } from '../constructs/functions/function'; /** * Props for the filemanager stack. */ -type FilemanagerProps = IngestFunctionProps & { +type FilemanagerProps = IngestFunctionProps & DatabaseProps & { /** * VPC to use for filemanager. */ readonly vpc: IVpc, - /** - * The database secret. - */ - readonly databaseSecret: ISecret; - /** - * The database security group. - */ - readonly databaseSecurityGroup: ISecurityGroup; /** * Whether to initialize a database migration. */ @@ -42,21 +34,21 @@ type FilemanagerProps = IngestFunctionProps & { * Construct used to configure the filemanager. */ export class Filemanager extends Construct { - constructor(scope: Construct, id: string, settings: FilemanagerProps) { + constructor(scope: Construct, id: string, props: FilemanagerProps) { super(scope, id); - if (settings?.migrateDatabase) { + if (props?.migrateDatabase) { new CdkResourceInvoke(this, 'MigrateDatabase', { - vpc: settings.vpc, + vpc: props.vpc, createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { return new MigrateFunction(scope, id, props); }, functionProps: { - vpc: settings.vpc, - databaseSecret: settings.databaseSecret, - databaseSecurityGroup: settings.databaseSecurityGroup, - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, + vpc: props.vpc, + databaseSecret: props.databaseSecret, + databaseSecurityGroup: props.databaseSecurityGroup, + buildEnvironment: props?.buildEnvironment, + rustLog: props?.rustLog, }, id: 'MigrateFunction', // Assuming no dependencies because the database will already exist. @@ -64,13 +56,13 @@ export class Filemanager extends Construct { } new IngestFunction(this, 'IngestLambda', { - vpc: settings.vpc, - databaseSecret: settings.databaseSecret, - databaseSecurityGroup: settings.databaseSecurityGroup, - eventSources: settings.eventSources, - buckets: settings.buckets, - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, + vpc: props.vpc, + databaseSecret: props.databaseSecret, + databaseSecurityGroup: props.databaseSecurityGroup, + eventSources: props.eventSources, + buckets: props.buckets, + buildEnvironment: props?.buildEnvironment, + rustLog: props?.rustLog, }); } } diff --git a/lib/workload/stateful/filemanager/docs/ARCHITECTURE.md b/lib/workload/stateless/filemanager/docs/ARCHITECTURE.md similarity index 100% rename from lib/workload/stateful/filemanager/docs/ARCHITECTURE.md rename to lib/workload/stateless/filemanager/docs/ARCHITECTURE.md diff --git a/lib/workload/stateless/functions/cdk_resource_invoke.ts b/lib/workload/stateless/functions/cdk_resource_invoke.ts index a274530e2..a440d9556 100644 --- a/lib/workload/stateless/functions/cdk_resource_invoke.ts +++ b/lib/workload/stateless/functions/cdk_resource_invoke.ts @@ -9,6 +9,7 @@ import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Version } from 'aws-cdk-lib/aws-lambda'; +import { createHash } from 'node:crypto'; /** * The interface by which the generic type of `CdkResourceInvoke` is constrained by. @@ -66,6 +67,10 @@ export type CdkResourceInvokeProps = { * Dependencies for this resource. */ dependencies?: IDependable[]; + /** + * Any payload to pass to the Lambda function. + */ + payload?: string; }; /** @@ -82,9 +87,12 @@ export class CdkResourceInvoke extends Construct { super(scope, id); const stack = Stack.of(this); + + // It's necessary to hash this because stack names can exceed the 64 character limit of function names. + const stackHash = this.hashValue(stack.stackName); this._function = props.createFunction(this, props.id, { ...props.functionProps, - functionName: `${stack.stackName}-ResourceInvokeFunction-${props.id}`, + functionName: `${stackHash}-ResourceInvokeFunction-${props.id}`, }); // Call another lambda function with no arguments. @@ -93,9 +101,10 @@ export class CdkResourceInvoke extends Construct { action: 'invoke', parameters: { FunctionName: this.function.functionName, + ...(props.payload && { Payload: props.payload }) }, physicalResourceId: PhysicalResourceId.of( - `${id}-AwsSdkCall-${this.function.currentVersion}` + `${id}-AwsSdkCall-${this.function.currentVersion + this.hashValue(props.payload)}` ), }; @@ -107,7 +116,7 @@ export class CdkResourceInvoke extends Construct { resources: [ // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a // singleton Lambda function. - `arn:aws:lambda:${stack.region}:${stack.account}:function:${stack.stackName}-ResourceInvokeFunction-*`, + `arn:aws:lambda:${stack.region}:${stack.account}:function:${stackHash}-ResourceInvokeFunction-*`, ], actions: ['lambda:InvokeFunction'], }) @@ -138,6 +147,14 @@ export class CdkResourceInvoke extends Construct { }); } + private hashValue(value?: string): string { + if (!value) { + return ''; + } + + return createHash('md5').update(value).digest('hex').substring(0, 24); + } + /** * Add a dependency to this resource. */ diff --git a/test/stateful/databaseConstruct.test.ts b/test/stateful/databaseConstruct.test.ts index 31b1780d2..51a0c0eb5 100644 --- a/test/stateful/databaseConstruct.test.ts +++ b/test/stateful/databaseConstruct.test.ts @@ -39,18 +39,19 @@ test('Test DBCluster created props', () => { }); test('Test other SG Allow Ingress to DB SG', () => { - const allowedSG = new ec2.SecurityGroup(stack, 'AllowedSG', { - securityGroupName: 'Allowed DB Ingress', + const props = constructConfig.stackProps.orcaBusStatefulConfig.databaseProps; + props.inboundSecurityGroupName = 'test-security-group'; + const database = new Database(stack, 'TestDatabaseConstruct', { vpc, + ...props, }); - const sgLogicalId = stack.getLogicalId(allowedSG.node.defaultChild as ec2.CfnSecurityGroup); - new Database(stack, 'TestDatabaseConstruct', { - vpc, - ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, - allowedInboundSG: allowedSG, - }); + const sgLogicalId = stack.getLogicalId( + database.inboundSecurityGroup.node.defaultChild as ec2.CfnSecurityGroup + ); + const template = Template.fromStack(stack); + console.log(JSON.stringify(template, undefined, 2)); template.hasResourceProperties('AWS::EC2::SecurityGroupIngress', { ToPort: dbProps.dbPort, @@ -59,4 +60,8 @@ test('Test other SG Allow Ingress to DB SG', () => { 'Fn::GetAtt': [sgLogicalId, 'GroupId'], }, }); + + template.hasResourceProperties('AWS::EC2::SecurityGroup', { + GroupName: props.inboundSecurityGroupName, + }); });