[ETL-634] Implement Dispatch Lambda (#114)

* Refactor stacks related to input SNS

In preparation for dispatch related stacks

* Add Dispatch Lambda code and tests

* Add stacks related to dispatch Lambda

* Add `filter_object_info` function to dispatch Lambda

config/develop/namespaced/lambda-dispatch-role.yaml

@@ -0,0 +1,13 @@
+template:
+  path: lambda-dispatch-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch-role"
+dependencies:
+  - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/namespaced/sns-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+parameters:
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-dispatch.yaml

@@ -0,0 +1,15 @@
+template:
+  type: sam
+  path: src/lambda_function/dispatch/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - develop/namespaced/lambda-dispatch-role.yaml
+  - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch-role::RoleArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  DispatchSnsArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-s3-event-config.yaml

@@ -6,12 +6,12 @@ template:
 dependencies:
   - develop/namespaced/lambda-s3-event-config-role.yaml
   - develop/s3-cloudformation-bucket.yaml
-  - develop/namespaced/sns-topic.yaml
+  - develop/namespaced/sns-input.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
   Namespace: {{ stack_group_config.namespace }}
-  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
   S3ToGlueDestinationType: "Topic"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-s3-event-config-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}

config/develop/namespaced/sns-dispatch-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch-policy"
+dependencies:
+  - develop/namespaced/lambda-dispatch.yaml
+  - develop/namespaced/sns-dispatch.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+  LambdaSourceArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch::DispatchFunctionArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-dispatch.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-input-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input-policy"
+dependencies:
+  - develop/s3-input-bucket.yaml
+  - develop/namespaced/sns-input.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
+  S3SourceBucketArn: !stack_output_external "recover-dev-input-bucket::BucketArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-input.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sqs-input-to-raw.yaml → config/develop/namespaced/sqs-input-to-dispatch.yaml

@@ -4,9 +4,9 @@ parameters:
   MessageRetentionPeriod: "1209600"
   ReceiveMessageWaitTimeSeconds: "20"
   VisibilityTimeout: "120"
-  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
 dependencies:
-  - develop/namespaced/sns-topic.yaml
-stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw'
+  - develop/namespaced/sns-input.yaml
+stack_name: "{{ stack_group_config.namespace }}-sqs-input-to-dispatch"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sqs-input-to-intermediate.yaml

@@ -4,9 +4,9 @@ parameters:
   MessageRetentionPeriod: "1209600"
   ReceiveMessageWaitTimeSeconds: "20"
   VisibilityTimeout: "120"
-  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
 dependencies:
-  - develop/namespaced/sns-topic.yaml
+  - develop/namespaced/sns-input.yaml
 stack_name: "{{ stack_group_config.namespace }}-sqs-input-to-intermediate"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-dispatch-role.yaml

@@ -0,0 +1,13 @@
+template:
+  path: lambda-dispatch-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch-role"
+dependencies:
+  - prod/namespaced/sqs-input-to-dispatch.yaml
+  - prod/namespaced/sns-dispatch.yaml
+  - prod/s3-cloudformation-bucket.yaml
+parameters:
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-dispatch.yaml

@@ -0,0 +1,15 @@
+template:
+  type: sam
+  path: src/lambda_function/dispatch/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - prod/namespaced/lambda-dispatch-role.yaml
+  - prod/namespaced/sqs-input-to-dispatch.yaml
+  - prod/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch-role::RoleArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  DispatchSnsArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-s3-event-config.yaml

@@ -6,12 +6,12 @@ template:
 dependencies:
   - prod/namespaced/lambda-s3-event-config-role.yaml
   - prod/s3-cloudformation-bucket.yaml
-  - prod/namespaced/sns-topic.yaml
+  - prod/namespaced/sns-input.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
   Namespace: {{ stack_group_config.namespace }}
-  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
   S3ToGlueDestinationType: "Topic"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-s3-event-config-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}

config/prod/namespaced/sns-dispatch-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch-policy"
+dependencies:
+  - prod/namespaced/lambda-dispatch.yaml
+  - prod/namespaced/sns-dispatch.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+  LambdaSourceArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch::DispatchFunctionArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-dispatch.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-input-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input-policy"
+dependencies:
+  - prod/s3-input-bucket.yaml
+  - prod/namespaced/sns-input.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
+  S3SourceBucketArn: !stack_output_external "recover-input-bucket::BucketArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-input.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sqs-input-to-raw.yaml → config/prod/namespaced/sqs-input-to-dispatch.yaml

@@ -4,9 +4,9 @@ parameters:
   MessageRetentionPeriod: "1209600"
   ReceiveMessageWaitTimeSeconds: "20"
   VisibilityTimeout: "120"
-  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
 dependencies:
-  - prod/namespaced/sns-topic.yaml
-stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw'
+  - prod/namespaced/sns-input.yaml
+stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-dispatch'
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sqs-input-to-intermediate.yaml

@@ -4,9 +4,9 @@ parameters:
   MessageRetentionPeriod: "1209600"
   ReceiveMessageWaitTimeSeconds: "20"
   VisibilityTimeout: "120"
-  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input-to-sqs::SnsTopicArn"
+  SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
 dependencies:
-  - prod/namespaced/sns-topic.yaml
+  - prod/namespaced/sns-input.yaml
 stack_name: "{{ stack_group_config.namespace }}-sqs-input-to-intermediate"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

src/lambda_function/dispatch/README.md

@@ -0,0 +1,35 @@
+# Dispatch Lambda
+
+The dispatch Lambda polls the input-to-dispatch SQS queue and publishes to the dispatch SNS topic.
+Its purpose is to inspect an export and dispatch each file as a separate job to eventually be consumed
+by the dispatch-to-raw Lambda.
+
+## Development
+
+The Serverless Application Model Command Line Interface (SAM CLI) is an
+extension of the AWS CLI that adds functionality for building and testing
+Lambda applications.
+
+To use the SAM CLI, you need the following tools.
+
+* SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html)
+* Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community)
+
+You may need the following for local testing.
+* [Python 3 installed](https://www.python.org/downloads/)
+
+You will also need to configure your AWS credentials, if you have not already done so.
+
+## Creating a local build
+
+Use the SAM CLI to build and test your lambda locally.
+Build your application with the `sam build` command.
+
+```bash
+cd src/lambda_function/dispatch/
+sam build
+```
+
+## Tests
+
+Tests are available in `tests/test_dispatch_lambda.py`.