Cirrus Lambda Fixes

CHANGELOG.md

@@ -9,10 +9,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- A custom Cirrus lambda dist ZIP can now be used instead of the default
+
 ### Changed
 
 ### Fixed
 
+- Fixed the Cirrus `update-state` lambda permissions to allow:
+  - Pushing messages to the Cirrus `publish` SNS topic
+  - Creating objects in the Cirrus `payload` S3 bucket
+- Fixed Cirrus workflow state machine permissions to allow creating state transition events
+
 ### Removed
 
 ## [2.30.0] - 2024-11-27

inputs.tf

@@ -352,6 +352,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = number
       timestream_memory_store_retention_period_in_hours  = number
     })
+    lambda_dist_zip_filepath = optional(string)
     api_lambda = object({
       timeout = number
       memory  = number
@@ -580,6 +581,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = 93
       timestream_memory_store_retention_period_in_hours  = 24
     }
+    lambda_dist_zip_filepath = null
     api_lambda = {
       timeout = 10
       memory  = 128

modules/base_infra/sns/inputs.tf

@@ -4,5 +4,5 @@ variable "sns_topics_map" {
 }
 
 locals {
-  default_sns_policy_file_path_name = "./modules/base_infra/sns/policy_files/sns_policy.json.tpl"
+  default_sns_policy_file_path_name = "${path.module}/policy_files/sns_policy.json.tpl"
 }

modules/cirrus/builtin-functions/api.tf

@@ -89,12 +89,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_api_lambda_role_policy_attachm
 }
 
 resource "aws_lambda_function" "cirrus_api" {
-  filename         = "${path.module}/../cirrus-lambda-dist.zip"
+  filename         = var.cirrus_lambda_dist_zip_filepath
   function_name    = "${var.cirrus_prefix}-api"
   description      = "Cirrus API Lambda"
   role             = aws_iam_role.cirrus_api_lambda_role.arn
   handler          = "api.lambda_handler"
-  source_code_hash = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
+  source_code_hash = filebase64sha256(var.cirrus_lambda_dist_zip_filepath)
   runtime          = "python3.12"
   timeout          = var.cirrus_api_lambda_timeout
   memory_size      = var.cirrus_api_lambda_memory

modules/cirrus/builtin-functions/inputs.tf

@@ -19,6 +19,15 @@ variable "cirrus_payload_bucket" {
   type        = string
 }
 
+variable "cirrus_lambda_dist_zip_filepath" {
+  description = <<-DESCRIPTION
+  (Optional) Filepath to a Cirrus Lambda Dist ZIP relative to the root module of
+  this Terraform deployment.
+  DESCRIPTION
+  type        = string
+  nullable    = false
+}
+
 variable "cirrus_api_lambda_timeout" {
   description = "Cirrus API lambda timeout (sec)"
   type        = number
@@ -106,6 +115,11 @@ variable "cirrus_workflow_event_sns_topic_arn" {
   type        = string
 }
 
+variable "cirrus_publish_sns_topic_arn" {
+  description = "Cirrus publish sns topic arn"
+  type        = string
+}
+
 variable "vpc_subnet_ids" {
   description = "List of subnet ids in the FilmDrop vpc"
   type        = list(string)

modules/cirrus/builtin-functions/process.tf

@@ -95,7 +95,7 @@ resource "aws_iam_policy" "cirrus_process_lambda_policy" {
       "Action": [
         "s3:PutObject"
       ],
-      "Resource": "arn:aws:s3:::${var.cirrus_payload_bucket}*"
+      "Resource": "arn:aws:s3:::${var.cirrus_payload_bucket}/*"
     },
     {
       "Effect": "Allow",
@@ -121,12 +121,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_process_lambda_role_policy_att
 }
 
 resource "aws_lambda_function" "cirrus_process" {
-  filename                       = "${path.module}/../cirrus-lambda-dist.zip"
+  filename                       = var.cirrus_lambda_dist_zip_filepath
   function_name                  = "${var.cirrus_prefix}-process"
   description                    = "Cirrus Process Lambda"
   role                           = aws_iam_role.cirrus_process_lambda_role.arn
   handler                        = "process.lambda_handler"
-  source_code_hash               = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
+  source_code_hash               = filebase64sha256(var.cirrus_lambda_dist_zip_filepath)
   runtime                        = "python3.12"
   timeout                        = var.cirrus_process_lambda_timeout
   memory_size                    = var.cirrus_process_lambda_memory

modules/cirrus/builtin-functions/update-state.tf

@@ -21,8 +21,6 @@ EOF
 resource "aws_iam_policy" "cirrus_update_state_lambda_policy" {
   name_prefix = "${var.cirrus_prefix}-process-policy-"
 
-  # TODO: the secret thing is probably not gonna work without some fixes in boto3utils...
-  # We should probably reconsider if this is the right solution.
   policy = <<EOF
 {
   "Version": "2012-10-17",
@@ -73,16 +71,20 @@ resource "aws_iam_policy" "cirrus_update_state_lambda_policy" {
     {
       "Effect": "Allow",
       "Action": [
-        "s3:GetObject"
+        "s3:GetObject",
+        "s3:PutObject"
       ],
-      "Resource": "arn:aws:s3:::${var.cirrus_payload_bucket}*"
+      "Resource": "arn:aws:s3:::${var.cirrus_payload_bucket}/*"
     },
     {
       "Effect": "Allow",
       "Action": [
         "sns:Publish"
       ],
-      "Resource": "${var.cirrus_workflow_event_sns_topic_arn}"
+      "Resource": [
+        "${var.cirrus_publish_sns_topic_arn}",
+        "${var.cirrus_workflow_event_sns_topic_arn}"
+      ]
     }
   ]
 }
@@ -101,12 +103,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_update_state_lambda_role_polic
 }
 
 resource "aws_lambda_function" "cirrus_update_state" {
-  filename         = "${path.module}/../cirrus-lambda-dist.zip"
+  filename         = var.cirrus_lambda_dist_zip_filepath
   function_name    = "${var.cirrus_prefix}-update-state"
   description      = "Cirrus Update-State Lambda"
   role             = aws_iam_role.cirrus_update_state_lambda_role.arn
   handler          = "update_state.lambda_handler"
-  source_code_hash = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
+  source_code_hash = filebase64sha256(var.cirrus_lambda_dist_zip_filepath)
   runtime          = "python3.12"
   timeout          = var.cirrus_update_state_lambda_timeout
   memory_size      = var.cirrus_update_state_lambda_memory
@@ -121,6 +123,7 @@ resource "aws_lambda_function" "cirrus_update_state" {
       CIRRUS_STATE_DB                 = var.cirrus_state_dynamodb_table_name
       CIRRUS_EVENT_DB_AND_TABLE       = "${var.cirrus_state_event_timestreamwrite_database_name}|${var.cirrus_state_event_timestreamwrite_table_name}"
       CIRRUS_WORKFLOW_EVENT_TOPIC_ARN = var.cirrus_workflow_event_sns_topic_arn
+      CIRRUS_PUBLISH_TOPIC_ARN        = var.cirrus_publish_sns_topic_arn
       CIRRUS_PROCESS_QUEUE_URL        = var.cirrus_process_sqs_queue_url
     }
   }
@@ -145,7 +148,7 @@ resource "aws_cloudwatch_event_rule" "cirrus_update_state_rule" {
   "detail": {
     "stateMachineArn": [
       {
-        "prefix": "arn:aws:states:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stateMachine:${var.cirrus_prefix}-*"
+        "prefix": "arn:aws:states:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stateMachine:${var.cirrus_prefix}-"
       }
     ],
     "status": [

modules/cirrus/builtin_functions.tf

@@ -12,6 +12,7 @@ module "builtin_functions" {
   cirrus_log_level                                 = var.cirrus_log_level
   cirrus_data_bucket                               = module.base.cirrus_data_bucket
   cirrus_payload_bucket                            = module.base.cirrus_payload_bucket
+  cirrus_lambda_dist_zip_filepath                  = local.cirrus_lambda_dist_zip_filepath
   cirrus_api_lambda_timeout                        = var.cirrus_api_lambda_timeout
   cirrus_api_lambda_memory                         = var.cirrus_api_lambda_memory
   cirrus_process_lambda_timeout                    = var.cirrus_process_lambda_timeout
@@ -25,6 +26,7 @@ module "builtin_functions" {
   cirrus_state_event_timestreamwrite_table_name    = module.base.cirrus_state_event_timestreamwrite_table_name
   cirrus_state_event_timestreamwrite_table_arn     = module.base.cirrus_state_event_timestreamwrite_table_arn
   cirrus_workflow_event_sns_topic_arn              = module.base.cirrus_workflow_event_sns_topic_arn
+  cirrus_publish_sns_topic_arn                     = module.base.cirrus_publish_sns_topic_arn
   cirrus_process_sqs_queue_arn                     = module.base.cirrus_process_sqs_queue_arn
   cirrus_process_sqs_queue_url                     = module.base.cirrus_process_sqs_queue_url
   cirrus_update_state_dead_letter_sqs_queue_arn    = module.base.cirrus_update_state_dead_letter_sqs_queue_arn

modules/cirrus/builtin_tasks.tf

@@ -5,7 +5,7 @@ locals {
     name = "pre-batch"
     lambda = {
       description     = "Cirrus Pre-Batch Lambda"
-      filename        = "${path.module}/cirrus-lambda-dist.zip"
+      filename        = local.cirrus_lambda_dist_zip_filepath
       handler         = "pre_batch.lambda_handler"
       runtime         = "python3.12"
       architectures   = ["arm64"]
@@ -77,7 +77,7 @@ locals {
     name = "post-batch"
     lambda = {
       description     = "Cirrus Post-Batch Lambda"
-      filename        = "${path.module}/cirrus-lambda-dist.zip"
+      filename        = local.cirrus_lambda_dist_zip_filepath
       handler         = "post_batch.lambda_handler"
       runtime         = "python3.12"
       architectures   = ["arm64"]

modules/cirrus/data.tf

@@ -8,4 +8,11 @@ locals {
 
   # All Cirrus-managed resources will be prefixed with this identifier
   cirrus_prefix = lower(substr(replace("fd-${var.project_name}-${var.environment}-cirrus", "_", "-"), 0, 63))
+
+  # Use a custom Cirrus Lambda Dist ZIP or accept the module's builtin version
+  cirrus_lambda_dist_zip_filepath = (
+    var.cirrus_lambda_dist_zip_filepath != null
+    ? "${path.root}/${var.cirrus_lambda_dist_zip_filepath}"
+    : "${path.module}/cirrus-lambda-dist.zip"
+  )
 }

modules/cirrus/inputs.tf

@@ -16,6 +16,18 @@ variable "project_name" {
   }
 }
 
+variable "cirrus_lambda_dist_zip_filepath" {
+  description = <<-DESCRIPTION
+  (Optional) Filepath to a Cirrus Lambda Dist ZIP relative to the root module
+  of this Terraform deployment. Used to override the ZIP that's included with
+  this module; only set if you're confident the replacement ZIP is compatible
+  with this module. If omitted, the default ZIP is used.
+  DESCRIPTION
+  type        = string
+  nullable    = true
+  default     = null
+}
+
 variable "cirrus_process_sqs_timeout" {
   description = "Cirrus Process SQS Visibility Timeout"
   type        = number

modules/cirrus/workflow/main.tf

@@ -90,6 +90,21 @@ resource "aws_iam_role" "workflow_machine" {
   description        = "State Machine execution role for Cirrus Workflow '${var.workflow_config.name}'"
   assume_role_policy = data.aws_iam_policy_document.workflow_machine_assume_role.json
 }
+
+data "aws_iam_policy_document" "workflow_machine_events" {
+  statement {
+    # Allow the state machine to push state transition events
+    effect    = "Allow"
+    actions   = ["events:PutEvents"]
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_role_policy" "workflow_machine_events" {
+  name_prefix = "${var.cirrus_prefix}-workflow-role-event-creation-"
+  role        = aws_iam_role.workflow_machine.name
+  policy      = data.aws_iam_policy_document.workflow_machine_events.json
+}
 # ==============================================================================
 
 

profiles/cirrus/inputs.tf

@@ -45,6 +45,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = number
       timestream_memory_store_retention_period_in_hours  = number
     })
+    lambda_dist_zip_filepath = optional(string)
     api_lambda = object({
       timeout = number
       memory  = number
@@ -273,6 +274,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = 93
       timestream_memory_store_retention_period_in_hours  = 24
     }
+    lambda_dist_zip_filepath = null
     api_lambda = {
       timeout = 10
       memory  = 128

profiles/cirrus/main.tf

@@ -5,6 +5,7 @@ module "cirrus" {
   environment                                               = var.environment
   vpc_subnet_ids                                            = var.private_subnet_ids
   vpc_security_group_ids                                    = [var.security_group_id]
+  cirrus_lambda_dist_zip_filepath                           = var.cirrus_inputs.lambda_dist_zip_filepath
   cirrus_process_sqs_timeout                                = var.cirrus_inputs.process.sqs_timeout
   cirrus_process_sqs_max_receive_count                      = var.cirrus_inputs.process.sqs_max_receive_count
   cirrus_timestream_magnetic_store_retention_period_in_days = var.cirrus_inputs.state.timestream_magnetic_store_retention_period_in_days

profiles/core/inputs.tf

@@ -352,6 +352,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = number
       timestream_memory_store_retention_period_in_hours  = number
     })
+    lambda_dist_zip_filepath = optional(string)
     api_lambda = object({
       timeout = number
       memory  = number
@@ -580,6 +581,7 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = 93
       timestream_memory_store_retention_period_in_hours  = 24
     }
+    lambda_dist_zip_filepath = null
     api_lambda = {
       timeout = 10
       memory  = 128