From 85830e5923085569c8424ab2a182c69ecd9e162d Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sun, 26 May 2024 10:54:38 -0700 Subject: [PATCH] Add ability to generate secrets for the application (#602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⚠️ breaking change (change to way secrets are defined in app-config in environment-variables.tf). secrets are now defined as a map: ``` secrets = { ENV_VAR_NAME = { manage_method = "code" or "manual" secret_store_path = "/ssm/param/name" } } ``` It was previously defined as a list: ``` secrets = [ { name = "ENV_VAR_NAME", ssm_param_name = "/ssm/param/name" } ] ``` * Add new module modules/secret for generating new secrets or referencing existing secrets * Refactor interface To migrate: * In app-config's environment-variables.tf, update secret definitions to use the new format. * For secrets managed outside of the project's codebase, set manage_method = "manual" * For secrets created within the project's codebase but defined elsewhere, move (using [terraform mv](https://developer.hashicorp.com/terraform/cli/commands/state/mv)) the aws_ssm_parameter to module.secret[ENV_VAR_NAME].aws_ssm_parameter.secret --- app/Dockerfile | 5 +++- app/app.py | 3 ++- .../environment-variables-and-secrets.md | 18 ++++++++----- .../env-config/environment-variables.tf | 25 +++++++++++++----- infra/app/app-config/env-config/outputs.tf | 2 +- infra/app/service/main.tf | 7 ++++- infra/app/service/secrets.tf | 16 ++++++++++++ infra/modules/secret/main.tf | 26 +++++++++++++++++++ infra/modules/secret/outputs.tf | 3 +++ infra/modules/secret/variables.tf | 22 ++++++++++++++++ infra/modules/service/access-control.tf | 2 +- infra/modules/service/main.tf | 2 +- infra/modules/service/secrets.tf | 14 ---------- infra/modules/service/variables.tf | 4 +-- 14 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 infra/app/service/secrets.tf create mode 100644 infra/modules/secret/main.tf create mode 100644 infra/modules/secret/outputs.tf create mode 100644 infra/modules/secret/variables.tf delete mode 100644 infra/modules/service/secrets.tf diff --git a/app/Dockerfile b/app/Dockerfile index 28293e281..295f9e884 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,4 +1,7 @@ -FROM python:3-alpine as release +# Pin to Alpine 3.19 since aws-cli was removed in Alpine 3.20 +# see https://github.com/alpinelinux/docker-alpine/issues/396 +# https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.20.0#aws-cli +FROM python:3-alpine3.19 as release RUN adduser --system --disabled-password --no-create-home app diff --git a/app/app.py b/app/app.py index 68972c4f8..0b23d0cd3 100644 --- a/app/app.py +++ b/app/app.py @@ -75,7 +75,8 @@ def document_upload(): @app.route("/secrets") def secrets(): secret_sauce = os.environ["SECRET_SAUCE"] - return f'The secret sauce is "{secret_sauce}"' + random_secret = os.environ["RANDOM_SECRET"] + return f'The secret sauce is "{secret_sauce}".
The random secret is "{random_secret}".' @app.cli.command("etl", help="Run ETL job") diff --git a/docs/infra/environment-variables-and-secrets.md b/docs/infra/environment-variables-and-secrets.md index f05d39df6..82ee62787 100644 --- a/docs/infra/environment-variables-and-secrets.md +++ b/docs/infra/environment-variables-and-secrets.md @@ -40,19 +40,23 @@ module "dev_config" { Secrets are a specific category of environment variables that need to be handled sensitively. Examples of secrets are authentication credentials such as API keys for external services. Secrets first need to be stored in AWS SSM Parameter Store as a `SecureString`. This section then describes how to make those secrets accessible to the ECS task as environment variables through the `secrets` configuration in the container definition (see AWS documentation on [retrieving Secrets Manager secrets through environment variables](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/secrets-envvar-secrets-manager.html)). -Secrets are defined in the same file that non-sensitive environment variables are defined, in the `app-config` module in the [environment-variables.tf file](/infra/app/app-config/env-config/environment-variables.tf). Modify the `secrets` list to define the secrets that the application will have access to. For each secret, `name` defines the environment variable name, and `ssm_param_name` defines the SSM parameter name that stores the secret value. For example: +Secrets are defined in the same file that non-sensitive environment variables are defined, in the `app-config` module in the [environment-variables.tf file](/infra/app/app-config/env-config/environment-variables.tf). Modify the `secrets` map to define the secrets that the application will have access to. For each secret, the map key defines the environment variable name. The `managed_by` property, which can be set to `"generated"` or `"manual"`, defines whether or not to generate a random secret or to reference an existing secret that was manually created and stored into AWS SSM. The `secret_store_name` property defines the SSM parameter name that stores the secret value. If `managed_by = "generated"`, then `secret_store_name` is where terraform will store the secret. If `managed_by = "manual"`, then `secret_store_name` is where terraform will look for the existing secret. For example: ```terraform # environment-variables.tf locals { - secrets = [ - { - name = "SOME_API_KEY" - ssm_param_name = "/${var.app_name}-${var.environment}/secret-sauce" + secrets = { + GENERATED_SECRET = { + manage_method = "generated" + secret_store_name = "/${var.app_name}-${var.environment}/generated-secret" } - ] + MANUALLY_CREATED_SECRET = { + manage_method = "manual" + secret_store_name = "/${var.app_name}-${var.environment}/manually-created-secret" + } + } } ``` -> ⚠️ Make sure you store the secret in SSM Parameter Store before you try to add secrets to your application service, or else the service won't be able to start since the ECS Task Executor won't be able to fetch the configured secret. +> ⚠️ For secrets with `managed_by = "manual"`, make sure you store the secret in SSM Parameter Store *before* you try to add configure your application service with the secrets, or else the service won't be able to start since the ECS Task Executor won't be able to fetch the configured secret. diff --git a/infra/app/app-config/env-config/environment-variables.tf b/infra/app/app-config/env-config/environment-variables.tf index f4225a7b7..d7354d09d 100644 --- a/infra/app/app-config/env-config/environment-variables.tf +++ b/infra/app/app-config/env-config/environment-variables.tf @@ -12,12 +12,23 @@ locals { # Configuration for secrets # List of configurations for defining environment variables that pull from SSM parameter # store. Configurations are of the format - # { name = "ENV_VAR_NAME", ssm_param_name = "/ssm/param/name" } - secrets = [ - # Example secret - # { - # name = "SECRET_SAUCE" - # ssm_param_name = "/${var.app_name}-${var.environment}/secret-sauce" + # { + # ENV_VAR_NAME = { + # manage_method = "generated" # or "manual" for a secret that was created and stored in SSM manually + # secret_store_name = "/ssm/param/name" + # } + # } + secrets = { + # Example generated secret + # RANDOM_SECRET = { + # manage_method = "generated" + # secret_store_name = "/${var.app_name}-${var.environment}/random-secret" # } - ] + + # Example secret that references a manually created secret + # SECRET_SAUCE = { + # manage_method = "manual" + # secret_store_name = "/${var.app_name}-${var.environment}/secret-sauce" + # } + } } diff --git a/infra/app/app-config/env-config/outputs.tf b/infra/app/app-config/env-config/outputs.tf index 360bc7e85..43b6d393a 100644 --- a/infra/app/app-config/env-config/outputs.tf +++ b/infra/app/app-config/env-config/outputs.tf @@ -30,7 +30,7 @@ output "service_config" { var.service_override_extra_environment_variables ) - secrets = toset(local.secrets) + secrets = local.secrets file_upload_jobs = { for job_name, job_config in local.file_upload_jobs : diff --git a/infra/app/service/main.tf b/infra/app/service/main.tf index f212e9b44..f06cc2d09 100644 --- a/infra/app/service/main.tf +++ b/infra/app/service/main.tf @@ -155,7 +155,12 @@ module "service" { BUCKET_NAME = local.storage_config.bucket_name }, local.service_config.extra_environment_variables) - secrets = local.service_config.secrets + secrets = [ + for secret_name in keys(local.service_config.secrets) : { + name = secret_name + valueFrom = module.secrets[secret_name].secret_arn + } + ] extra_policies = { feature_flags_access = module.feature_flags.access_policy_arn, diff --git a/infra/app/service/secrets.tf b/infra/app/service/secrets.tf new file mode 100644 index 000000000..e65eaa0cc --- /dev/null +++ b/infra/app/service/secrets.tf @@ -0,0 +1,16 @@ +module "secrets" { + for_each = local.service_config.secrets + + source = "../../modules/secret" + + # When generating secrets and storing them in parameter store, append the + # terraform workspace to the secret store path if the environment is temporary + # to avoid conflicts with existing environments. + # Don't do this for secrets that are managed manually since the temporary + # environments will need to share those secrets. + secret_store_name = (each.value.manage_method == "generated" && local.is_temporary ? + "${each.value.secret_store_name}/${terraform.workspace}" : + each.value.secret_store_name + ) + manage_method = each.value.manage_method +} diff --git a/infra/modules/secret/main.tf b/infra/modules/secret/main.tf new file mode 100644 index 000000000..8619c86e8 --- /dev/null +++ b/infra/modules/secret/main.tf @@ -0,0 +1,26 @@ +locals { + secret = var.manage_method == "generated" ? aws_ssm_parameter.secret[0] : data.aws_ssm_parameter.secret[0] + access_policy_name = "${trimprefix(replace(local.secret.name, "/", "-"), "/")}-access" +} + +resource "random_password" "secret" { + count = var.manage_method == "generated" ? 1 : 0 + + length = 64 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + +resource "aws_ssm_parameter" "secret" { + count = var.manage_method == "generated" ? 1 : 0 + + name = var.secret_store_name + type = "SecureString" + value = random_password.secret[0].result +} + +data "aws_ssm_parameter" "secret" { + count = var.manage_method == "manual" ? 1 : 0 + + name = var.secret_store_name +} diff --git a/infra/modules/secret/outputs.tf b/infra/modules/secret/outputs.tf new file mode 100644 index 000000000..57ebfcf82 --- /dev/null +++ b/infra/modules/secret/outputs.tf @@ -0,0 +1,3 @@ +output "secret_arn" { + value = local.secret.arn +} diff --git a/infra/modules/secret/variables.tf b/infra/modules/secret/variables.tf new file mode 100644 index 000000000..53d00e068 --- /dev/null +++ b/infra/modules/secret/variables.tf @@ -0,0 +1,22 @@ +variable "manage_method" { + type = string + description = <