Skip to content

Commit

Permalink
Add support for AWS China regions (closes #40)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbeemster
jbeemster committed Apr 3, 2024
1 parent 67f37c3 commit 1add395
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 130 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ module "enrich_kinesis" {
| <a name="input_kcl_write_min_capacity"></a> [kcl\_write\_min\_capacity](#input\_kcl\_write\_min\_capacity) | The minimum WRITE capacity for the KCL DynamoDB table | `number` | `1` | no |
| <a name="input_max_size"></a> [max\_size](#input\_max\_size) | The maximum number of servers in this server-group | `number` | `2` | no |
| <a name="input_min_size"></a> [min\_size](#input\_min\_size) | The minimum number of servers in this server-group | `number` | `1` | no |
| <a name="input_private_ecr_registry"></a> [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no |
| <a name="input_record_limit"></a> [record\_limit](#input\_record\_limit) | The number of events to buffer before pushing them to Kinesis | `number` | `500` | no |
| <a name="input_scale_down_cooldown_sec"></a> [scale\_down\_cooldown\_sec](#input\_scale\_down\_cooldown\_sec) | Time (in seconds) until another scale-down action can occur | `number` | `600` | no |
| <a name="input_scale_down_cpu_threshold_percentage"></a> [scale\_down\_cpu\_threshold\_percentage](#input\_scale\_down\_cpu\_threshold\_percentage) | The average CPU percentage that we must be below to scale-down | `number` | `20` | no |
Expand Down
281 changes: 152 additions & 129 deletions main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,25 @@ locals {
data "aws_region" "current" {}
data "aws_caller_identity" "current" {}

locals {
is_aws_global = replace(data.aws_region.current.name, "cn-", "") == data.aws_region.current.name
iam_partition = local.is_aws_global ? "aws" : "aws-cn"

is_private_ecr_registry = var.private_ecr_registry != ""
private_ecr_registry_statement = [{
Action = [
"ecr:GetAuthorizationToken",
"ecr:BatchGetImage",
"ecr:GetDownloadUrlForLayer"
]
Effect = "Allow"
Resource = [
"*"
]
}]
private_ecr_registry_statement_final = local.is_private_ecr_registry ? local.private_ecr_registry_statement : []
}

module "telemetry" {
source = "snowplow-devops/telemetry/snowplow"
version = "0.5.0"
Expand Down Expand Up @@ -112,22 +131,20 @@ resource "aws_cloudwatch_log_group" "log_group" {
# --- IAM: Roles & Permissions

locals {
custom_s3_hosted_assets_bucket_policy = <<EOF
,{
"Action": [
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket"
],
"Effect":"Allow",
"Resource": [
"arn:aws:s3:::${var.custom_s3_hosted_assets_bucket_name}",
"arn:aws:s3:::${var.custom_s3_hosted_assets_bucket_name}/*"
]
}
EOF

custom_s3_hosted_assets_bucket_policy_final = var.custom_s3_hosted_assets_bucket_name == "" ? "" : local.custom_s3_hosted_assets_bucket_policy
custom_s3_hosted_assets_bucket_statement = [{
Action = [
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket"
]
Effect = "Allow"
Resource = [
"arn:${local.iam_partition}:s3:::${var.custom_s3_hosted_assets_bucket_name}",
"arn:${local.iam_partition}:s3:::${var.custom_s3_hosted_assets_bucket_name}/*"
]
}]

custom_s3_hosted_assets_bucket_statement_final = var.custom_s3_hosted_assets_bucket_name != "" ? local.custom_s3_hosted_assets_bucket_statement : []
}

resource "aws_iam_role" "iam_role" {
Expand All @@ -154,120 +171,122 @@ EOF
resource "aws_iam_policy" "iam_policy" {
name = var.name

policy = <<EOF
{
"Version" : "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"kinesis:DescribeStream",
"kinesis:DescribeStreamSummary",
"kinesis:List*"
],
"Resource": [
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.in_stream_name}",
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.enriched_stream_name}",
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.bad_stream_name}"
]
},
{
"Effect": "Allow",
"Action": [
"kinesis:Get*"
],
"Resource": [
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.in_stream_name}"
]
},
{
"Effect": "Allow",
"Action": [
"kinesis:Put*"
],
"Resource": [
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.enriched_stream_name}",
"arn:aws:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.bad_stream_name}"
policy = jsonencode({
Version = "2012-10-17",
Statement = concat(
local.private_ecr_registry_statement_final,
local.custom_s3_hosted_assets_bucket_statement_final,
[
{
Effect = "Allow",
Action = [
"kinesis:DescribeStream",
"kinesis:DescribeStreamSummary",
"kinesis:List*"
],
Resource = [
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.in_stream_name}",
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.enriched_stream_name}",
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.bad_stream_name}"
]
},
{
Effect = "Allow",
Action = [
"kinesis:Get*"
],
Resource = [
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.in_stream_name}"
]
},
{
Effect = "Allow",
Action = [
"kinesis:Put*"
],
Resource = [
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.enriched_stream_name}",
"arn:${local.iam_partition}:kinesis:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:stream/${var.bad_stream_name}"
]
},
{
Effect = "Allow",
Action = [
"dynamodb:BatchWriteItem",
"dynamodb:PutItem",
"dynamodb:DescribeTable",
"dynamodb:DeleteItem",
"dynamodb:GetItem",
"dynamodb:Scan",
"dynamodb:UpdateItem"
],
Resource = [
"${aws_dynamodb_table.kcl.arn}"
]
},
{
Effect = "Allow",
Action = [
"dynamodb:DescribeTable",
"dynamodb:GetItem",
"dynamodb:Scan"
],
Resource = [
"${aws_dynamodb_table.config.arn}"
]
},
{
Effect = "Allow",
Action = [
"logs:PutLogEvents",
"logs:CreateLogStream",
"logs:DescribeLogStreams"
],
Resource = [
"arn:${local.iam_partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.cloudwatch_log_group_name}:*"
]
},
{
Effect = "Allow",
Action = [
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket"
],
Resource = [
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-east-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-east-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-west-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-west-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-west-2",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-west-2/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-sa-east-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-sa-east-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-eu-central-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-eu-central-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-southeast-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-southeast-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-southeast-2",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-southeast-2/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-northeast-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-northeast-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-south-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-south-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-east-2",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-us-east-2/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ca-central-1",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ca-central-1/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-eu-west-2",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-eu-west-2/*",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-northeast-2",
"arn:${local.iam_partition}:s3:::snowplow-hosted-assets-ap-northeast-2/*"
]
}
]
},
{
"Effect": "Allow",
"Action": [
"dynamodb:BatchWriteItem",
"dynamodb:PutItem",
"dynamodb:DescribeTable",
"dynamodb:DeleteItem",
"dynamodb:GetItem",
"dynamodb:Scan",
"dynamodb:UpdateItem"
],
"Resource": [
"${aws_dynamodb_table.kcl.arn}"
]
},
{
"Effect": "Allow",
"Action": [
"dynamodb:DescribeTable",
"dynamodb:GetItem",
"dynamodb:Scan"
],
"Resource": [
"${aws_dynamodb_table.config.arn}"
]
},
{
"Effect": "Allow",
"Action": [
"logs:PutLogEvents",
"logs:CreateLogStream",
"logs:DescribeLogStreams"
],
"Resource": [
"arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.cloudwatch_log_group_name}:*"
]
},
{
"Action": [
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket"
],
"Effect":"Allow",
"Resource": [
"arn:aws:s3:::snowplow-hosted-assets",
"arn:aws:s3:::snowplow-hosted-assets/*",
"arn:aws:s3:::snowplow-hosted-assets-us-east-1",
"arn:aws:s3:::snowplow-hosted-assets-us-east-1/*",
"arn:aws:s3:::snowplow-hosted-assets-us-west-1",
"arn:aws:s3:::snowplow-hosted-assets-us-west-1/*",
"arn:aws:s3:::snowplow-hosted-assets-us-west-2",
"arn:aws:s3:::snowplow-hosted-assets-us-west-2/*",
"arn:aws:s3:::snowplow-hosted-assets-sa-east-1",
"arn:aws:s3:::snowplow-hosted-assets-sa-east-1/*",
"arn:aws:s3:::snowplow-hosted-assets-eu-central-1",
"arn:aws:s3:::snowplow-hosted-assets-eu-central-1/*",
"arn:aws:s3:::snowplow-hosted-assets-ap-southeast-1",
"arn:aws:s3:::snowplow-hosted-assets-ap-southeast-1/*",
"arn:aws:s3:::snowplow-hosted-assets-ap-southeast-2",
"arn:aws:s3:::snowplow-hosted-assets-ap-southeast-2/*",
"arn:aws:s3:::snowplow-hosted-assets-ap-northeast-1",
"arn:aws:s3:::snowplow-hosted-assets-ap-northeast-1/*",
"arn:aws:s3:::snowplow-hosted-assets-ap-south-1",
"arn:aws:s3:::snowplow-hosted-assets-ap-south-1/*",
"arn:aws:s3:::snowplow-hosted-assets-us-east-2",
"arn:aws:s3:::snowplow-hosted-assets-us-east-2/*",
"arn:aws:s3:::snowplow-hosted-assets-ca-central-1",
"arn:aws:s3:::snowplow-hosted-assets-ca-central-1/*",
"arn:aws:s3:::snowplow-hosted-assets-eu-west-2",
"arn:aws:s3:::snowplow-hosted-assets-eu-west-2/*",
"arn:aws:s3:::snowplow-hosted-assets-ap-northeast-2",
"arn:aws:s3:::snowplow-hosted-assets-ap-northeast-2/*"
]
}${local.custom_s3_hosted_assets_bucket_policy_final}
]
}
EOF
)
})
}

resource "aws_iam_role_policy_attachment" "policy_attachment" {
Expand Down Expand Up @@ -387,6 +406,10 @@ locals {

container_memory = "${module.instance_type_metrics.memory_application_mb}m"
java_opts = var.java_opts

is_private_ecr_registry = local.is_private_ecr_registry
private_ecr_registry = var.private_ecr_registry
region = data.aws_region.current.name
})
}

Expand Down
6 changes: 5 additions & 1 deletion templates/user-data.sh.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ sudo base64 --decode << EOF > $${CONFIG_DIR}/enrich.hocon
${config_b64}
EOF

%{ if is_private_ecr_registry }
aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${private_ecr_registry}
%{ endif ~}

sudo docker run \
-d \
--name enrich \
Expand All @@ -23,7 +27,7 @@ sudo docker run \
--env JDK_JAVA_OPTIONS='${java_opts}' \
--env ACCEPT_LIMITED_USE_LICENSE=${accept_limited_use_license} \
--env INSTANCE_ID=$(get_instance_id) \
snowplow/snowplow-enrich-kinesis:${version} \
${private_ecr_registry}snowplow/snowplow-enrich-kinesis:${version} \
--config /snowplow/config/enrich.hocon \
--iglu-config ${resolver} \
--enrichments ${enrichments}
Expand Down
8 changes: 8 additions & 0 deletions variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,3 +402,11 @@ variable "user_provided_id" {
type = string
default = ""
}

# --- Image Repositories

variable "private_ecr_registry" {
description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')"
type = string
default = ""
}

0 comments on commit 1add395

Please sign in to comment.