first commit

README.md

@@ -0,0 +1,55 @@
+`tf_lambda_s3_backup`
+---------------------
+
+This modules allows configure a ECS task with a given container to be triggered using
+a cron expression.
+
+Currently is designed to trigger backups using the container: https://github.com/mergermarket/docker-mysql-s3-backup,
+but it would be adapted to run any container.
+
+*Note*: since the release of [scheduled ECS tasks from AWS](https://aws.amazon.com/about-aws/whats-new/2017/06/amazon-ecs-now-supports-time-and-event-based-task-scheduling), there is no point to use a lambda for this anymore. But the code related to tasks can be reused.
+
+Usage:
+-----
+
+You can configure it with, for instance:
+
+```
+resource "aws_s3_bucket" "s3-backup" {
+  bucket = "${var.team}-${var.env}-${var.component}-mysql-s3-backup"
+}
+
+module "lambda_s3_backup" {
+  source = "github.com/mergermarket/tf_lambda_s3_backup?ref=PLAT-71_initial_implementation"
+
+  name                 = "${var.env}-${var.component}"
+  # Could be "${aws_s3_bucket.s3-backup.id}", but using this to avoid the
+  # count cannot be computer error
+  bucket_name          = "${var.team}-${var.env}-${var.component}-mysql-s3-backup"
+  bind_host_path       = "${var.data_volume_path}"
+  bind_container_path  = "/mnt/data"
+  cluster              = "atlassian"
+  lambda_cron_schedule = "rate(3 hours)"
+
+  backup_env = {
+    "DATABASE_TYPE"     = "mysql"
+    "DATABASE_HOSTNAME" = "${aws_db_instance.rds.address}"
+    "DATABASE_PORT"     = "3306"
+    "DATABASE_DB_NAME"  = "${aws_db_instance.rds.name}"
+    "DATABASE_USERNAME" = "${aws_db_instance.rds.username}"
+    "DATABASE_PASSWORD" = "${var.secrets["MYSQL_PASSWORD"]}"
+    "RETENTION"         = 12
+    "DUMPS_PATH"        = "/mnt/data/mysql"
+    "S3_BUCKET_NAME"    = "${var.team}-${var.env}-${var.component}-mysql-s3-backup"
+    "S3_BUCKET_PATH"    = "/backup/${var.component}/${var.env}"
+    "SYNC_ORIGIN_PATH"  = "/mnt/data"
+  }
+
+  metadata = {
+    component = "${var.component}"
+    env       = "${var.env}"
+  }
+}
+```
+
+

lambda-src/ecs_worker.py

@@ -0,0 +1,50 @@
+import os
+import sys
+import logging
+import shlex
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+try:
+    import boto3
+    client = boto3.client('ecs')
+except ImportError:
+    logger.error('boto3 is not installed. ECSTasks require boto3')
+    sys.exit(1)
+
+
+def trigger(event, context):
+    logger.info('got event{}'.format(event))
+
+    overrides = dict()
+
+    task_definition_arn = os.environ.get('TASK_DEFINITION_ARN')
+    if not task_definition_arn:
+        logger.error(
+            "'TASK_DEFINITION ENVIRONMENT' environment variable not set")
+        raise(Exception("task_definition environment variable not set"))
+
+    logger.info('Starting task {}'.format(task_definition_arn))
+
+    task_command = os.environ.get('TASK_COMMAND')
+    if task_command:
+        logger.info('Custom command: {}'.format(task_command))
+        overrides = {'containerOverrides': [{
+                    'name': os.environ.get('CONTAINER_NAME'),
+                    'command': shlex.split(task_command)
+                  }]}
+    response = client.run_task(
+            cluster=os.environ.get('CLUSTER', 'default'),
+            taskDefinition=task_definition_arn,
+            overrides=overrides
+            )
+
+    ids = ', '.join([task['taskArn'] for task in response['tasks']])
+
+    logger.info('Started tasks {}'.format(ids))
+
+    return {
+        'message': 'Started tasks {}'.format(ids)
+    }

lambda.tf

@@ -0,0 +1,141 @@
+# Archive with the code to upload
+data "archive_file" "lambda_zip" {
+  type        = "zip"
+  source_dir  = "${path.module}/lambda-src"
+  output_path = "${path.module}/lambda.zip"
+}
+
+# IAM roles and policy
+resource "aws_iam_role" "iam_for_lambda" {
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+
+}
+
+resource "aws_iam_role_policy" "lambda_policy" {
+  role = aws_iam_role.iam_for_lambda.id
+  name = "${var.name}-lambda-policy"
+
+  policy = data.aws_iam_policy_document.lambda_policy.json
+}
+
+data "aws_caller_identity" "current" {
+}
+
+data "aws_iam_policy_document" "lambda_policy" {
+  # Allow lambda to log
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+
+    resources = [
+      "arn:aws:logs:*:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${var.name}-s3-backup",
+      "arn:aws:logs:*:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${var.name}-s3-backup:*",
+    ]
+  }
+
+  # Allow lambda to run the task
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ecs:RunTask",
+    ]
+
+    # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
+    # force an interpolation expression to be interpreted as a list by wrapping it
+    # in an extra set of list brackets. That form was supported for compatibility in
+    # v0.11, but is no longer supported in Terraform v0.12.
+    #
+    # If the expression in the following list itself returns a list, remove the
+    # brackets to avoid interpretation as a list of lists. If the expression
+    # returns a single list item then leave it as-is and remove this TODO comment.
+    resources = [
+      module.s3_backup_taskdef.arn,
+    ]
+  }
+
+  # Allow lambda to assume the role of the task
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+
+    # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
+    # force an interpolation expression to be interpreted as a list by wrapping it
+    # in an extra set of list brackets. That form was supported for compatibility in
+    # v0.11, but is no longer supported in Terraform v0.12.
+    #
+    # If the expression in the following list itself returns a list, remove the
+    # brackets to avoid interpretation as a list of lists. If the expression
+    # returns a single list item then leave it as-is and remove this TODO comment.
+    resources = [
+      module.s3_backup_taskdef.task_role_arn,
+    ]
+  }
+
+  depends_on = [module.s3_backup_taskdef]
+}
+
+resource "aws_lambda_permission" "allow_cloudwatch" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.lambda_function.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.cron_schedule.arn
+}
+
+# Configure the lambda function
+resource "aws_lambda_function" "lambda_function" {
+  filename         = "${path.module}/lambda.zip"
+  source_code_hash = data.archive_file.lambda_zip.output_base64sha256
+  function_name    = "${var.name}-s3-backup"
+  role             = aws_iam_role.iam_for_lambda.arn
+  handler          = "ecs_worker.trigger"
+  runtime          = "python3.6"
+
+  environment {
+    variables = merge(
+      var.backup_env,
+      {
+        "TASK_DEFINITION_ARN" = module.s3_backup_taskdef.arn
+        "TASK_COMMAND"        = var.docker_command
+        "CONTAINER_NAME"      = "${var.name}-s3-backup"
+        "CLUSTER"             = var.cluster
+      },
+    )
+  }
+}
+
+# Configure cron
+resource "aws_cloudwatch_event_rule" "cron_schedule" {
+  name                = "${aws_lambda_function.lambda_function.function_name}-cron_schedule"
+  description         = "This event will run according to a schedule for lambda ${var.name}-s3-backup"
+  schedule_expression = var.lambda_cron_schedule
+}
+
+resource "aws_cloudwatch_event_target" "event_target" {
+  rule = aws_cloudwatch_event_rule.cron_schedule.name
+  arn  = aws_lambda_function.lambda_function.arn
+}
+

outputs.tf

@@ -0,0 +1,8 @@
+output "lambda_arn" {
+  value = aws_lambda_function.lambda_function.arn
+}
+
+output "task_role_arn" {
+  value = module.s3_backup_taskdef.task_role_arn
+}
+

task.tf

@@ -0,0 +1,69 @@
+# Create two Cloudwatch Log Groups for the backup container
+resource "aws_cloudwatch_log_group" "stdout" {
+  name              = "${var.name}-s3-backup-stdout"
+  retention_in_days = "7"
+}
+
+resource "aws_cloudwatch_log_group" "stderr" {
+  name              = "${var.name}-s3-backup-stderr"
+  retention_in_days = "7"
+}
+
+module "s3_backup_container_definition" {
+  source = "github.com/mergermarket/tf_ecs_container_definition?ref=no-secrets"
+
+  name   = "${var.name}-s3-backup"
+  image  = var.docker_image
+  cpu    = 512
+  memory = 512
+
+  container_env = merge(
+    var.backup_env,
+    {
+      "LOGSPOUT_CLOUDWATCHLOGS_LOG_GROUP_STDOUT" = "${var.name}-s3-backup-stdout"
+      "LOGSPOUT_CLOUDWATCHLOGS_LOG_GROUP_STDERR" = "${var.name}-s3-backup-stderr"
+    },
+  )
+
+  metadata = var.metadata
+
+  mountpoint = {
+    sourceVolume  = "s3_backup_volume"
+    containerPath = var.bind_container_path
+    readOnly      = "false"
+  }
+}
+
+module "s3_backup_taskdef" {
+  source = "github.com/mergermarket/tf_ecs_task_definition_with_task_role?ref=pre-assume-role"
+
+  family                = "${var.name}-s3-backup"
+  container_definitions = [module.s3_backup_container_definition.rendered]
+
+  policy = data.aws_iam_policy_document.s3_backup_policy.json
+
+  volume = {
+    name      = "s3_backup_volume"
+    host_path = var.bind_host_path
+  }
+}
+
+# Allow the task to sync files into the container
+data "aws_iam_policy_document" "s3_backup_policy" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:ListBucket",
+      "s3:PutObject",
+      "s3:PutObjectAcl",
+      "s3:DeleteObject",
+    ]
+
+    resources = [
+      "arn:aws:s3:::${var.bucket_name}",
+      "arn:aws:s3:::${var.bucket_name}/*",
+    ]
+  }
+}
+

variables.tf

@@ -0,0 +1,50 @@
+variable "name" {
+  description = "Name for this backup task"
+}
+
+variable "bucket_name" {
+  description = "Bucket to sync the files to"
+}
+
+variable "lambda_cron_schedule" {
+  description = "The scheduling expression for how often the lambda function runs."
+  default     = "rate(3 hours)"
+}
+
+variable "bind_host_path" {
+  description = "Host volume to mount into the container. Must be set together with bind_host_path"
+  default     = "/tmp/dummy"
+}
+
+variable "bind_container_path" {
+  description = "Container volume to mount into the container. Must be set together with bind_container_path"
+  default     = "/tmp/dummy"
+}
+
+variable "cluster" {
+  description = "Name of the ECS cluster where the ECS task would run"
+  default     = "default"
+}
+
+variable "docker_image" {
+  description = "Docker image to use for the task. It should contain all the logic to perform the dump and sync"
+  default     = "mergermarket/docker-mysql-s3-backup"
+}
+
+variable "docker_command" {
+  description = "Custom command to run in the container"
+  default     = ""
+}
+
+variable "backup_env" {
+  description = "Environment parameters passed to the lambda function and the container"
+  type        = map(string)
+  default     = {}
+}
+
+variable "metadata" {
+  description = "Metadata for the resources created by this module"
+  type        = map(string)
+  default     = {}
+}
+

versions.tf

@@ -0,0 +1,12 @@
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    archive = {
+      source = "hashicorp/archive"
+    }
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}