From 793db6e9e1c492a44e6f48605f68bcde57a9839b Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 08:23:14 -0400 Subject: [PATCH 1/7] Copy egress proxy setup over from FAC Original implementation: https://github.com/GSA-TTS/FAC/tree/ad9a04d70b2701867c2a29d605ccc7894e00757b/terraform/shared/modules/https-proxy --- egress_proxy/acl.tftpl | 5 ++ egress_proxy/main.tf | 127 ++++++++++++++++++++++++++++++++++ egress_proxy/outputs.tf | 29 ++++++++ egress_proxy/prepare-proxy.sh | 25 +++++++ egress_proxy/providers.tf | 9 +++ egress_proxy/variables.tf | 64 +++++++++++++++++ 6 files changed, 259 insertions(+) create mode 100644 egress_proxy/acl.tftpl create mode 100644 egress_proxy/main.tf create mode 100644 egress_proxy/outputs.tf create mode 100644 egress_proxy/prepare-proxy.sh create mode 100644 egress_proxy/providers.tf create mode 100644 egress_proxy/variables.tf diff --git a/egress_proxy/acl.tftpl b/egress_proxy/acl.tftpl new file mode 100644 index 0000000..a109cb1 --- /dev/null +++ b/egress_proxy/acl.tftpl @@ -0,0 +1,5 @@ +%{ for app, dests in list ~} +%{ for dest in dests ~} +${ split(":", dest)[0] } +%{ endfor ~} +%{ endfor ~} diff --git a/egress_proxy/main.tf b/egress_proxy/main.tf new file mode 100644 index 0000000..dfe50f2 --- /dev/null +++ b/egress_proxy/main.tf @@ -0,0 +1,127 @@ +locals { + + # Make a clean list of the client apps for iteration purposes + clients = toset(keys(merge(var.allowlist, var.denylist))) + + # Generate Caddy-compatible allow and deny ACLs, one target per line. + # + # For now, there's just one consolidated allowlist and denylist, no matter + # what apps they were specified for. Future improvments could improve this, + # but it would mean also changing the proxy to be both more complex (in terms + # of how the Caddyfile is constructed) and more discriminating (in terms of + # recognizing client apps based on GUIDs supplied by Envoy in request headers, + # as well as the destination ports). However, adding these improvements won't + # require modifying the module's interface, since we're already collecting + # that refined information. + allowacl = templatefile("${path.module}/acl.tftpl", { list = var.allowlist }) + denyacl = templatefile("${path.module}/acl.tftpl", { list = var.denylist }) +} + +### +### Set up the authenticated egress application in the target space on apps.internal +### + +data "cloudfoundry_domain" "internal" { + name = "apps.internal" +} + +resource "cloudfoundry_route" "egress_route" { + space = data.cloudfoundry_space.egress_space.id + domain = data.cloudfoundry_domain.internal.id + hostname = "${var.cf_org_name}-${replace(var.cf_space_name, ".", "-")}-${var.name}" + # Yields something like: orgname-spacename-name.apps.internal +} + +resource "random_uuid" "username" {} +resource "random_password" "password" { + length = 16 + special = false +} + +data "cloudfoundry_space" "egress_space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +# This zips up just the depoyable files from the specified gitref in the +# cg-egress-proxy repository +data "external" "proxyzip" { + program = ["/bin/sh", "prepare-proxy.sh"] + working_dir = path.module + query = { + gitref = var.gitref + } +} + +resource "cloudfoundry_app" "egress_app" { + name = var.name + space = data.cloudfoundry_space.egress_space.id + path = "${path.module}/${data.external.proxyzip.result.path}" + source_code_hash = filesha256("${path.module}/${data.external.proxyzip.result.path}") + buildpack = "binary_buildpack" + command = "./caddy run --config Caddyfile" + memory = var.egress_memory + instances = var.instances + strategy = "rolling" + + routes { + route = cloudfoundry_route.egress_route.id + } + environment = { + PROXY_PORTS : join(" ", var.allowports) + PROXY_ALLOW : local.allowacl + PROXY_DENY : local.denyacl + PROXY_USERNAME : random_uuid.username.result + PROXY_PASSWORD : random_password.password.result + } +} + +### +### Set up network policies so that the clients can reach the proxy +### + +data "cloudfoundry_space" "client_space" { + org_name = var.cf_org_name + name = var.client_space +} + +data "cloudfoundry_app" "clients" { + for_each = local.clients + name_or_id = each.key + space = data.cloudfoundry_space.client_space.id +} + +resource "cloudfoundry_network_policy" "client_routing" { + for_each = local.clients + policy { + source_app = data.cloudfoundry_app.clients[each.key].id + destination_app = cloudfoundry_app.egress_app.id + port = "61443" + } +} + +### +### Create a credential service for bound clients to use when make requests of the proxy +### +locals { + https_proxy = "https://${random_uuid.username.result}:${random_password.password.result}@${cloudfoundry_route.egress_route.endpoint}:61443" + domain = cloudfoundry_route.egress_route.endpoint + username = random_uuid.username.result + password = random_password.password.result + protocol = "https" + port = 61443 + app_id = cloudfoundry_app.egress_app.id +} + +resource "cloudfoundry_user_provided_service" "credentials" { + name = "${var.name}-creds" + space = data.cloudfoundry_space.client_space.id + credentials = { + "uri" = local.https_proxy + "domain" = local.domain + "username" = local.username + "password" = local.password + "protocol" = local.protocol + "port" = local.port + } +} diff --git a/egress_proxy/outputs.tf b/egress_proxy/outputs.tf new file mode 100644 index 0000000..c567775 --- /dev/null +++ b/egress_proxy/outputs.tf @@ -0,0 +1,29 @@ +output "https_proxy" { + value = local.https_proxy + sensitive = true +} + +output "domain" { + value = local.domain +} + +output "username" { + value = local.username +} + +output "password" { + value = local.password + sensitive = true +} + +output "protocol" { + value = local.protocol +} + +output "app_id" { + value = local.app_id +} + +output "port" { + value = local.port +} diff --git a/egress_proxy/prepare-proxy.sh b/egress_proxy/prepare-proxy.sh new file mode 100644 index 0000000..12f6b8c --- /dev/null +++ b/egress_proxy/prepare-proxy.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Exit if any step fails +set -e + +eval "$(jq -r '@sh "GITREF=\(.gitref)"')" + +popdir=$(pwd) + +# Portable construct so this will work everywhere +# https://unix.stackexchange.com/a/84980 +tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') +cd "$tmpdir" + +# Grab a copy of the zip file for the specified ref +curl -s -L https://github.com/GSA-TTS/cg-egress-proxy/archive/${GITREF}.zip --output local.zip + +# Zip up just the proxy/ subdirectory for pushing +unzip -q -u local.zip \*/proxy/\* +zip -q -j -r ${popdir}/proxy.zip cg-egress-proxy-*/proxy + +# Tell Terraform where to find it +cat << EOF +{ "path": "proxy.zip" } +EOF diff --git a/egress_proxy/providers.tf b/egress_proxy/providers.tf new file mode 100644 index 0000000..aa59fd5 --- /dev/null +++ b/egress_proxy/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = ">=0.53.1" + } + } +} diff --git a/egress_proxy/variables.tf b/egress_proxy/variables.tf new file mode 100644 index 0000000..5e41839 --- /dev/null +++ b/egress_proxy/variables.tf @@ -0,0 +1,64 @@ +variable "cf_org_name" { + type = string + description = "cloud.gov organization name" +} + +variable "cf_space_name" { + type = string + description = "cloud.gov space name for egress (eg staging-egress or prod-egress)" +} + +variable "client_space" { + type = string + description = "cloud.gov space name for client apps (eg staging or prod)" +} + +variable "name" { + type = string + description = "name of the egress proxy application" +} + +variable "egress_memory" { + type = number + description = "Memory in MB to allocate to egress proxy app" + default = 64 +} + +variable "gitref" { + type = string + description = "gitref for the specific version of cg-egress-proxy that you want to use. Branch name should start with `refs/heads` while a git sha are given without a prefix" + default = "refs/heads/main" + # You can also specify a specific commit, eg "7487f882903b9e834a5133a883a88b16fb8b16c9" +} + +variable "allowports" { + type = list(number) + description = "Valid ports to proxy to" + default = [443] +} + +variable "allowlist" { + description = "Allowed egress for apps (applied first). A map where keys are app names, and the values are sets of acl strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "*.example.com:443", "example2.com:443" ] + } +} + +variable "denylist" { + description = "Denied egress for apps (applied second). A map where keys are app names, and the values are sets of host:port strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "bad.example.com:443" ] + } +} + +variable "instances" { + type = number + description = "the number of instances of the HTTPS proxy application to run (default: 2)" + default = 2 +} From 78a839f36ebff4b0c7d53b038773598242da0c97 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 09:48:51 -0400 Subject: [PATCH 2/7] Add creation and outputs test --- egress_proxy/main.tf | 4 +-- egress_proxy/outputs.tf | 4 +-- egress_proxy/tests/creation.tftest.hcl | 46 ++++++++++++++++++++++++++ egress_proxy/variables.tf | 4 +-- 4 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 egress_proxy/tests/creation.tftest.hcl diff --git a/egress_proxy/main.tf b/egress_proxy/main.tf index dfe50f2..b2953a5 100644 --- a/egress_proxy/main.tf +++ b/egress_proxy/main.tf @@ -28,8 +28,8 @@ data "cloudfoundry_domain" "internal" { resource "cloudfoundry_route" "egress_route" { space = data.cloudfoundry_space.egress_space.id domain = data.cloudfoundry_domain.internal.id - hostname = "${var.cf_org_name}-${replace(var.cf_space_name, ".", "-")}-${var.name}" - # Yields something like: orgname-spacename-name.apps.internal + hostname = substr("${var.cf_org_name}-${replace(var.cf_space_name, ".", "-")}-${var.name}", -63, -1) + # Yields something like: orgname-spacename-name.apps.internal, limited to the last 63 characters } resource "random_uuid" "username" {} diff --git a/egress_proxy/outputs.tf b/egress_proxy/outputs.tf index c567775..db9e66e 100644 --- a/egress_proxy/outputs.tf +++ b/egress_proxy/outputs.tf @@ -1,5 +1,5 @@ output "https_proxy" { - value = local.https_proxy + value = local.https_proxy sensitive = true } @@ -12,7 +12,7 @@ output "username" { } output "password" { - value = local.password + value = local.password sensitive = true } diff --git a/egress_proxy/tests/creation.tftest.hcl b/egress_proxy/tests/creation.tftest.hcl new file mode 100644 index 0000000..1098e30 --- /dev/null +++ b/egress_proxy/tests/creation.tftest.hcl @@ -0,0 +1,46 @@ +mock_provider "cloudfoundry" {} + +variables { + cf_org_name = "gsa-tts-devtools-prototyping" + cf_space_name = "terraform-cloudgov-ci-tests-egress" + client_space = "terraform-cloudgov-ci-tests" + name = "terraform-egress-app" + allowlist = { "continuous_monitoring-staging" = ["raw.githubusercontent.com:443"] } +} + +run "test_proxy_creation" { + assert { + condition = output.https_proxy == "https://${output.username}:${output.password}@${output.domain}:61443" + error_message = "HTTPS_PROXY output must match the correct form, got ${nonsensitive(output.https_proxy)}" + } + + assert { + condition = output.domain == cloudfoundry_route.egress_route.endpoint + error_message = "Output domain must match the route endpoint" + } + + assert { + condition = output.username == random_uuid.username.result + error_message = "Output username must come from the random_uuid resource" + } + + assert { + condition = output.password == random_password.password.result + error_message = "Output password must come from the random_password resource" + } + + assert { + condition = output.protocol == "https" + error_message = "protocol only supports https" + } + + assert { + condition = output.app_id == cloudfoundry_app.egress_app.id + error_message = "Output app_id is the egress_app's ID" + } + + assert { + condition = output.port == 61443 + error_message = "port only supports 61443 internal https listener" + } +} diff --git a/egress_proxy/variables.tf b/egress_proxy/variables.tf index 5e41839..1d8e3fd 100644 --- a/egress_proxy/variables.tf +++ b/egress_proxy/variables.tf @@ -32,9 +32,9 @@ variable "gitref" { } variable "allowports" { - type = list(number) + type = list(number) description = "Valid ports to proxy to" - default = [443] + default = [443] } variable "allowlist" { From 5430c07d2922e2f13f22cb8fe5a0144e8fc4c38d Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 10:18:50 -0400 Subject: [PATCH 3/7] Update docs for using egress_proxy module --- README.md | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7dea733..102d5c3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # terraform-cloudgov -Terraform modules for working with cloud.gov commonly used by [18f/rails-template](https://github.com/18f/rails-template) based apps +Terraform modules for working with cloud.gov commonly used by [GSA-TTS/rails-template](https://github.com/GSA-TTS/rails-template) based apps ## Module Examples @@ -10,7 +10,7 @@ Creates an RDS database based on the `rds_plan_name` variable and outputs the `i ``` module "database" { - source = "github.com/18f/terraform-cloudgov//database?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name @@ -32,7 +32,7 @@ Creates a Elasticache redis instance and outputs the `instance_id` for use elsew ``` module "redis" { - source = "github.com/18f/terraform-cloudgov//redis?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name @@ -54,7 +54,7 @@ Creates an s3 bucket and outputs the `bucket_id` for use elsewhere. ``` module "s3" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name @@ -79,7 +79,7 @@ Note that the domain must be created in cloud.gov by an OrgManager before this m ``` module "domain" { - source = "github.com/18f/terraform-cloudgov//domain?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//domain?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name @@ -101,7 +101,7 @@ Notes: ``` module "clamav" { - source = "github.com/18f/terraform-cloudgov//clamav?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//clamav?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name @@ -129,7 +129,7 @@ Creates a new cloud.gov space, such as when creating an egress space, and output ``` module "egress_space" { - source = "github.com/18f/terraform-cloudgov//cg_space?ref=v1.0.0" + source = "github.com/GSA-TTS/terraform-cloudgov//cg_space?ref=v1.1.0" cf_org_name = local.cf_org_name cf_space_name = "${local.cf_space_name}-egress" @@ -145,6 +145,30 @@ module "egress_space" { } ``` +### egress_proxy + +Creates and configures an instance of cg-egress-proxy to proxy traffic from your apps. + +Prerequities: + +* existing client_space with already deployed apps +* existing public-egress space to deploy the proxy into + +``` +module "egress_proxy" { + source = "github.com/GSA-TTS/terraform-cloudgov//egress_proxy?ref=v1.1.0" + + cf_org_name = local.cf_org_name + cf_space_name = "${local.cf_space_name}-egress" + client_space = local.cf_space_name + name = "egress-proxy" + allowlist = { + "source_app_name" = ["host.com:443", "otherhost.com:443"] + } + # see egress_proxy/variables.tf for full list of optional arguments +} +``` + ## Testing From ec560e0e0ca77020a113ada5e0a5382902afe6f2 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 10:21:34 -0400 Subject: [PATCH 4/7] Add proxy module to CI actions --- .github/workflows/lint.yml | 4 ++-- .github/workflows/test.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 119b957..cb81cca 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - path: ["cg_space", "clamav", "database", "domain", "redis", "s3"] + path: ["cg_space", "clamav", "database", "domain", "redis", "s3", "egress_proxy"] steps: - name: Checkout uses: actions/checkout@v4 @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - path: ["cg_space", "clamav", "database", "domain", "redis", "s3"] + path: ["cg_space", "clamav", "database", "domain", "redis", "s3", "egress_proxy"] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 455c96a..54f8a9f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false max-parallel: 2 matrix: - module: ["s3", "database", "redis", "cg_space", "domain", "clamav"] + module: ["s3", "database", "redis", "cg_space", "domain", "clamav", "egress_proxy"] steps: - uses: actions/checkout@v4 From 5dd540df33da0d571df2fe44bfd905d4b1a2ffc4 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 10:24:39 -0400 Subject: [PATCH 5/7] Add space_name as output from cg_space module --- cg_space/outputs.tf | 4 ++++ cg_space/tests/creation.tftest.hcl | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/cg_space/outputs.tf b/cg_space/outputs.tf index 7333cc4..0217e76 100644 --- a/cg_space/outputs.tf +++ b/cg_space/outputs.tf @@ -1,3 +1,7 @@ output "space_id" { value = cloudfoundry_space.space.id } + +output "space_name" { + value = cloudfoundry_space.space.name +} diff --git a/cg_space/tests/creation.tftest.hcl b/cg_space/tests/creation.tftest.hcl index 50a961a..6d010ce 100644 --- a/cg_space/tests/creation.tftest.hcl +++ b/cg_space/tests/creation.tftest.hcl @@ -34,6 +34,11 @@ run "test_space_creation" { condition = cloudfoundry_space.space.name == var.cf_space_name error_message = "Space name should match the cf_space_name variable" } + + assert { + condition = cloudfoundry_space.space.name == output.space_name + error_message = "Space name output must match the new space" + } } run "test_manager_only" { From fc94c677c37c177da1733d0aacc8742c6ed0f967 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 12:47:16 -0400 Subject: [PATCH 6/7] Test egress_proxy separately with zip isntalled --- .github/workflows/test.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54f8a9f..1412842 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false max-parallel: 2 matrix: - module: ["s3", "database", "redis", "cg_space", "domain", "clamav", "egress_proxy"] + module: ["s3", "database", "redis", "cg_space", "domain", "clamav"] steps: - uses: actions/checkout@v4 @@ -24,3 +24,17 @@ jobs: CF_PASSWORD: ${{ secrets.CF_PASSWORD }} with: path: ${{ matrix.module }} + + test-egress-proxy: + runs-on: ubuntu-latest + name: Egress proxy integration test + env: + TERRAFORM_PRE_RUN: | + apt-get update + apt-get install -y zip + steps: + - uses: actions/checkout@v4 + - name: terraform test egress_proxy + uses: dflook/terraform-test@v1 + with: + path: egress_proxy From 201d91d021c35c2d7926552bc2d92bd832cebff2 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 10 Oct 2024 14:51:10 -0400 Subject: [PATCH 7/7] Typo fix in egress_proxy/variables.tf Co-authored-by: Zachary Rollyson --- egress_proxy/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egress_proxy/variables.tf b/egress_proxy/variables.tf index 1d8e3fd..1a680a2 100644 --- a/egress_proxy/variables.tf +++ b/egress_proxy/variables.tf @@ -26,7 +26,7 @@ variable "egress_memory" { variable "gitref" { type = string - description = "gitref for the specific version of cg-egress-proxy that you want to use. Branch name should start with `refs/heads` while a git sha are given without a prefix" + description = "gitref for the specific version of cg-egress-proxy that you want to use. Branch name should start with `refs/heads` while a git sha should be given without a prefix" default = "refs/heads/main" # You can also specify a specific commit, eg "7487f882903b9e834a5133a883a88b16fb8b16c9" }