[ETL-611] Raw sync lambda (#141)

* initial commit raw sync lambda

* add stacks for raw sync lambda

* add publish to sns functionality to raw sync lambda

* Add cloudwatch events rule for raw sync lambda

* raw sync lambda minor improvements

.github/workflows/upload-and-deploy.yaml

@@ -136,12 +136,13 @@ jobs:
 
       - name: Test scripts with pytest (lambda, etc.)
         run: |
-          pipenv run python -m pytest \
+          pipenv run python -m pytest -v \
             tests/test_s3_event_config_lambda.py \
             tests/test_s3_to_glue_lambda.py \
             tests/test_lambda_dispatch.py \
             tests/test_consume_logs.py \
-            tests/test_lambda_raw.py -v
+            tests/test_lambda_raw.py \
+            tests/test_lambda_raw_sync.py
 
       - name: Test dev synapse folders for STS access with pytest
         run: >

config/develop/namespaced/events-rule-raw-sync.yaml

@@ -0,0 +1,12 @@
+template:
+  path: events-rule.yaml
+stack_name: "{{ stack_group_config.namespace }}-events-rule-raw-sync"
+dependencies:
+  - develop/namespaced/lambda-raw-sync.yaml
+parameters:
+  RuleName: "{{ stack_group_config.namespace }}-lambda-raw-sync-trigger"
+  RuleState: DISABLED
+  LambdaArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync::RawSyncFunctionArn"
+  CronSchedule: cron(0 0 * * ? *)
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-dispatch.yaml

@@ -6,6 +6,7 @@ template:
 dependencies:
   - develop/namespaced/lambda-dispatch-role.yaml
   - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/namespaced/sns-dispatch.yaml
   - develop/s3-cloudformation-bucket.yaml
 stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
 parameters:

config/develop/namespaced/lambda-raw-sync-role.yaml

@@ -0,0 +1,11 @@
+template:
+  path: lambda-raw-sync-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync-role"
+dependencies:
+  - develop/namespaced/sns-dispatch.yaml
+parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  S3TargetBucketName: {{ stack_group_config.raw_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-raw-sync.yaml

@@ -0,0 +1,18 @@
+template:
+  type: sam
+  path: src/lambda_function/raw_sync/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - develop/namespaced/lambda-raw-sync-role.yaml
+  - develop/namespaced/sns-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync-role::RoleArn"
+  S3InputBucket: {{ stack_group_config.input_bucket_name }}
+  S3InputKeyPrefix: "{{ stack_group_config.namespace }}/"
+  S3RawBucket: {{ stack_group_config.raw_bucket_name }}
+  S3RawKeyPrefix: "{{ stack_group_config.namespace }}/json/"
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/events-rule-raw-sync.yaml

@@ -0,0 +1,12 @@
+template:
+  path: events-rule.yaml
+stack_name: "{{ stack_group_config.namespace }}-events-rule-raw-sync"
+dependencies:
+  - prod/namespaced/lambda-raw-sync.yaml
+parameters:
+  RuleName: "{{ stack_group_config.namespace }}-lambda-raw-sync-trigger"
+  RuleState: DISABLED
+  LambdaArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync::RawSyncFunctionArn"
+  CronSchedule: cron(0 0 * * ? *)
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-dispatch.yaml

@@ -6,6 +6,7 @@ template:
 dependencies:
   - prod/namespaced/lambda-dispatch-role.yaml
   - prod/namespaced/sqs-input-to-dispatch.yaml
+  - prod/namespaced/sns-dispatch.yaml
   - prod/s3-cloudformation-bucket.yaml
 stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
 parameters:

config/prod/namespaced/lambda-raw-sync-role.yaml

@@ -0,0 +1,11 @@
+template:
+  path: lambda-raw-sync-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync-role"
+dependencies:
+  - prod/namespaced/sns-dispatch.yaml
+parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  S3TargetBucketName: {{ stack_group_config.raw_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-raw-sync.yaml

@@ -0,0 +1,18 @@
+template:
+  type: sam
+  path: src/lambda_function/raw_sync/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - prod/namespaced/lambda-raw-sync-role.yaml
+  - prod/namespaced/sns-dispatch.yaml
+  - prod/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync-role::RoleArn"
+  S3InputBucket: {{ stack_group_config.input_bucket_name }}
+  S3InputKeyPrefix: "{{ stack_group_config.namespace }}/"
+  S3RawBucket: {{ stack_group_config.raw_bucket_name }}
+  S3RawKeyPrefix: "{{ stack_group_config.namespace }}/json/"
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

src/lambda_function/raw_sync/README.md

@@ -0,0 +1,38 @@
+# Raw Sync Lambda
+
+The raw sync Lambda ensures that the input and raw S3 buckets are synchronized
+by verifying that all non-zero sized JSON in each export in the input bucket
+have a corresponding object in the raw bucket.
+
+If a JSON file from an export is found to not have a corresponding object in the raw bucket,
+the file is submitted to the raw Lambda (via the dispatch SNS topic) for processing.
+
+## Development
+
+The Serverless Application Model Command Line Interface (SAM CLI) is an
+extension of the AWS CLI that adds functionality for building and testing
+Lambda applications.
+
+To use the SAM CLI, you need the following tools.
+
+* SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html)
+* Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community)
+
+You may need the following for local testing.
+* [Python 3 installed](https://www.python.org/downloads/)
+
+You will also need to configure your AWS credentials, if you have not already done so.
+
+## Creating a local build
+
+Use the SAM CLI to build and test your lambda locally.
+Build your application with the `sam build` command.
+
+```bash
+cd src/lambda_function/raw_sync/
+sam build
+```
+
+## Tests
+
+Tests are available in `tests/test_lambda_raw_sync.py`.