From 2ec43ebdb066adb887bd2b159cb8da2cce88f44e Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 3 May 2022 18:38:07 -0400 Subject: [PATCH 01/25] allow HEAD requests from different origins. --- terraform/modules/dandiset_bucket/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index c3948ff..1967041 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -34,6 +34,7 @@ resource "aws_s3_bucket_cors_configuration" "dandiset_bucket" { "PUT", "POST", "GET", + "HEAD", "DELETE", ] allowed_headers = [ From f0047e7aad947fc3b096f5f31c8479bc2ada9103 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com> Date: Mon, 18 Sep 2023 21:27:11 -0400 Subject: [PATCH 02/25] Add CI for terraform linting/validation --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0ed25c9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI +on: + push: + branches: + - master + pull_request: + +jobs: + lint-terraform: + name: Lint Terraform code + runs-on: ubuntu-latest + defaults: + run: + working-directory: terraform + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Terraform + uses: hashicorp/setup-terraform@v2.0.3 + + - name: Lint Terraform code + run: terraform fmt -check -diff -recursive + + - name: Initialize Terraform (offline only) + run: terraform init -backend=false + + - name: Validate Terraform code + run: terraform validate From b9b6ab27acb9f48a06dcb512ff1446480b778f70 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Mon, 26 Feb 2024 16:31:08 -0500 Subject: [PATCH 03/25] Fix linting --- terraform/modules/dandiset_bucket/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/dandiset_bucket/variables.tf b/terraform/modules/dandiset_bucket/variables.tf index 41915c5..b6dc9ad 100644 --- a/terraform/modules/dandiset_bucket/variables.tf +++ b/terraform/modules/dandiset_bucket/variables.tf @@ -38,6 +38,6 @@ variable "log_bucket_name" { # TODO: this can be inferred from the "versioning" variable once we're ready # to deploy this to the production bucket as well. variable "trailing_delete" { - type = bool + type = bool description = "Whether or not trailing delete should be enabled on the bucket." } From f7118e6af1f8f11fec21ec6e8e091cd4fe2e8468 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Tue, 27 Feb 2024 16:18:07 -0500 Subject: [PATCH 04/25] Update `hashicorp/setup-terraform` action This silences the warning about using Node 16. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ed25c9..bbab538 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Install Terraform - uses: hashicorp/setup-terraform@v2.0.3 + uses: hashicorp/setup-terraform@v3 - name: Lint Terraform code run: terraform fmt -check -diff -recursive From 2de08cd3b764fe935806a4e17e6ed54b70683b95 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Tue, 5 Mar 2024 15:23:17 -0500 Subject: [PATCH 05/25] Deploy `dandidav` service to Heroku --- terraform/webdav.tf | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 terraform/webdav.tf diff --git a/terraform/webdav.tf b/terraform/webdav.tf new file mode 100644 index 0000000..7a68a05 --- /dev/null +++ b/terraform/webdav.tf @@ -0,0 +1,39 @@ +resource "heroku_app" "webdav" { + name = "dandidav" + region = "us" + acm = true + + organization { + name = data.heroku_team.dandi.name + } + + buildpacks = [ + "https://buildpack-registry.s3.amazonaws.com/buildpacks/emk/rust.tgz" + ] +} + +resource "heroku_formation" "webdav_heroku_web" { + app_id = heroku_app.webdav.id + type = "web" + size = "basic" + quantity = 1 +} + +# Enable this feature so that the Rust application can access the git commit hash. +resource "heroku_app_feature" "webdav_runtime_dyno_metadata" { + app_id = heroku_app.webdav.id + name = "runtime-dyno-metadata" +} + +resource "heroku_domain" "webdav" { + app_id = heroku_app.webdav.id + hostname = "webdav.dandiarchive.org" +} + +resource "aws_route53_record" "heroku" { + zone_id = aws_route53_zone.dandi.zone_id + name = "webdav" + type = "CNAME" + ttl = "300" + records = [heroku_domain.webdav.cname] +} From bc9962d86feb792cb33e678039e15b5fc798e4b1 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Tue, 5 Mar 2024 20:04:10 -0500 Subject: [PATCH 06/25] Add import blocks We'll want to remove these in a follow-up PR after applying. --- terraform/webdav.tf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/terraform/webdav.tf b/terraform/webdav.tf index 7a68a05..1f0a779 100644 --- a/terraform/webdav.tf +++ b/terraform/webdav.tf @@ -12,6 +12,11 @@ resource "heroku_app" "webdav" { ] } +import { + to = heroku_app.webdav + id = "dandidav" +} + resource "heroku_formation" "webdav_heroku_web" { app_id = heroku_app.webdav.id type = "web" @@ -19,17 +24,32 @@ resource "heroku_formation" "webdav_heroku_web" { quantity = 1 } +import { + to = heroku_formation.webdav_heroku_web + id = "dandidav:web" +} + # Enable this feature so that the Rust application can access the git commit hash. resource "heroku_app_feature" "webdav_runtime_dyno_metadata" { app_id = heroku_app.webdav.id name = "runtime-dyno-metadata" } +import { + to = heroku_app_feature.webdav_runtime_dyno_metadata + id = "dandidav:runtime-dyno-metadata" +} + resource "heroku_domain" "webdav" { app_id = heroku_app.webdav.id hostname = "webdav.dandiarchive.org" } +import { + to = heroku_domain.webdav + id = "dandidav:webdav.dandiarchive.org" +} + resource "aws_route53_record" "heroku" { zone_id = aws_route53_zone.dandi.zone_id name = "webdav" @@ -37,3 +57,8 @@ resource "aws_route53_record" "heroku" { ttl = "300" records = [heroku_domain.webdav.cname] } + +import { + to = aws_route53_record.heroku + id = "Z02063701JNV8GCOUJIZZ_webdav.dandiarchive.org_CNAME" +} From b5eca955a33e793e1656910dc01792b05a38893e Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Mon, 11 Mar 2024 12:37:46 -0400 Subject: [PATCH 07/25] Remove import blocks in webdav.tf --- terraform/webdav.tf | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/terraform/webdav.tf b/terraform/webdav.tf index 1f0a779..7a68a05 100644 --- a/terraform/webdav.tf +++ b/terraform/webdav.tf @@ -12,11 +12,6 @@ resource "heroku_app" "webdav" { ] } -import { - to = heroku_app.webdav - id = "dandidav" -} - resource "heroku_formation" "webdav_heroku_web" { app_id = heroku_app.webdav.id type = "web" @@ -24,32 +19,17 @@ resource "heroku_formation" "webdav_heroku_web" { quantity = 1 } -import { - to = heroku_formation.webdav_heroku_web - id = "dandidav:web" -} - # Enable this feature so that the Rust application can access the git commit hash. resource "heroku_app_feature" "webdav_runtime_dyno_metadata" { app_id = heroku_app.webdav.id name = "runtime-dyno-metadata" } -import { - to = heroku_app_feature.webdav_runtime_dyno_metadata - id = "dandidav:runtime-dyno-metadata" -} - resource "heroku_domain" "webdav" { app_id = heroku_app.webdav.id hostname = "webdav.dandiarchive.org" } -import { - to = heroku_domain.webdav - id = "dandidav:webdav.dandiarchive.org" -} - resource "aws_route53_record" "heroku" { zone_id = aws_route53_zone.dandi.zone_id name = "webdav" @@ -57,8 +37,3 @@ resource "aws_route53_record" "heroku" { ttl = "300" records = [heroku_domain.webdav.cname] } - -import { - to = aws_route53_record.heroku - id = "Z02063701JNV8GCOUJIZZ_webdav.dandiarchive.org_CNAME" -} From 8701e2318c04bbc3f7a95c221c61523167859f7b Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Mon, 11 Mar 2024 21:54:05 -0400 Subject: [PATCH 08/25] Disable Rust buildpack and dyno-metadata feature Neither of these are needed anymore now that we are deploying by pushing directly to Heroku --- terraform/webdav.tf | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/terraform/webdav.tf b/terraform/webdav.tf index 7a68a05..d5006cb 100644 --- a/terraform/webdav.tf +++ b/terraform/webdav.tf @@ -8,7 +8,10 @@ resource "heroku_app" "webdav" { } buildpacks = [ - "https://buildpack-registry.s3.amazonaws.com/buildpacks/emk/rust.tgz" + # The Rust application is compiled and pushed to Heroku via a GitHub Action, so + # we don't need to specify a specific buildpack here. So, we just fall back to + # the Heroku CLI buildpack as a default. + "heroku-community/cli" ] } @@ -19,12 +22,6 @@ resource "heroku_formation" "webdav_heroku_web" { quantity = 1 } -# Enable this feature so that the Rust application can access the git commit hash. -resource "heroku_app_feature" "webdav_runtime_dyno_metadata" { - app_id = heroku_app.webdav.id - name = "runtime-dyno-metadata" -} - resource "heroku_domain" "webdav" { app_id = heroku_app.webdav.id hostname = "webdav.dandiarchive.org" From e60c8b5b55c60e1e497d75b2c1fec1397f965b30 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Mon, 25 Mar 2024 13:47:07 -0400 Subject: [PATCH 09/25] Increase quantity of web dynos to 3 --- terraform/api.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/api.tf b/terraform/api.tf index a8ea0a0..d3cc1d3 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -17,7 +17,7 @@ module "api" { heroku_cloudamqp_plan = "squirrel-1" heroku_papertrail_plan = "liatorp" - heroku_web_dyno_quantity = 1 + heroku_web_dyno_quantity = 3 heroku_worker_dyno_quantity = 1 django_default_from_email = "admin@api.dandiarchive.org" From ce5cbf84800819dc09e4d83aa73e2caf12050486 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Fri, 15 Mar 2024 13:22:13 -0400 Subject: [PATCH 10/25] Add bucket policy for embargoed objects --- terraform/modules/dandiset_bucket/main.tf | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 1967041..7208fb3 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -136,6 +136,42 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { } } +resource "aws_s3_bucket_policy" "dandiset_embargoed_objects" { + count = var.public ? 1 : 0 + provider = aws + bucket = aws_s3_bucket.dandiset_bucket.id + policy = data.aws_iam_policy_document.dandiset_embargoed_objects[0].json +} + +data "aws_iam_policy_document" "dandiset_embargoed_objects" { + count = var.public ? 1 : 0 + version = "2008-10-17" + + statement { + effect = "Deny" + principals { + identifiers = ["*"] + type = "*" + } + actions = [ + "s3:*", + ] + resources = [ + "${aws_s3_bucket.dandiset_bucket.arn}/*", + ] + condition { + test = "StringEquals" + variable = "s3:ExistingObjectTag/embargoed" + values = ["true"] + } + condition { + test = "ArnNotEquals" + variable = "aws:PrincipalArn" + values = [var.heroku_user.arn] + } + } +} + resource "aws_s3_bucket_policy" "dandiset_bucket_policy" { provider = aws From 1072e8822b7c46867476df4b5d7f90ac65958774 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Wed, 3 Apr 2024 11:05:34 -0400 Subject: [PATCH 11/25] Revert "Add embargoed tags bucket policy" --- terraform/modules/dandiset_bucket/main.tf | 36 ----------------------- 1 file changed, 36 deletions(-) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 7208fb3..1967041 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -136,42 +136,6 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { } } -resource "aws_s3_bucket_policy" "dandiset_embargoed_objects" { - count = var.public ? 1 : 0 - provider = aws - bucket = aws_s3_bucket.dandiset_bucket.id - policy = data.aws_iam_policy_document.dandiset_embargoed_objects[0].json -} - -data "aws_iam_policy_document" "dandiset_embargoed_objects" { - count = var.public ? 1 : 0 - version = "2008-10-17" - - statement { - effect = "Deny" - principals { - identifiers = ["*"] - type = "*" - } - actions = [ - "s3:*", - ] - resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}/*", - ] - condition { - test = "StringEquals" - variable = "s3:ExistingObjectTag/embargoed" - values = ["true"] - } - condition { - test = "ArnNotEquals" - variable = "aws:PrincipalArn" - values = [var.heroku_user.arn] - } - } -} - resource "aws_s3_bucket_policy" "dandiset_bucket_policy" { provider = aws From 6c4cad15d6e608777c1eb3779c277328ce3bdfd1 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Wed, 3 Apr 2024 11:24:12 -0400 Subject: [PATCH 12/25] Add bucket policy statement for embargoed objects --- terraform/modules/dandiset_bucket/main.tf | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 1967041..10d69cb 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -167,6 +167,33 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { } } + # Disallow access to embargoed objects, unless using the heroku user arn + dynamic "statement" { + for_each = var.public ? [1] : [] + + content { + effect = "Deny" + principals { + identifiers = ["*"] + type = "*" + } + actions = ["s3:*"] + resources = [ + "${aws_s3_bucket.dandiset_bucket.arn}/*", + ] + condition { + test = "StringEquals" + variable = "s3:ExistingObjectTag/embargoed" + values = ["true"] + } + condition { + test = "ArnNotEquals" + variable = "aws:PrincipalArn" + values = [var.heroku_user.arn] + } + } + } + dynamic "statement" { for_each = var.allow_cross_account_heroku_put_object ? [1] : [] From 6e56e06616df6916722da35c8b2c75a8e5641754 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:18:12 -0400 Subject: [PATCH 13/25] Use full URL for webdav buildpack Using just `heroku-community/cli` results in subsequent Terraform plans wanting to change it to the full URL. --- terraform/webdav.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/webdav.tf b/terraform/webdav.tf index d5006cb..2f2bc37 100644 --- a/terraform/webdav.tf +++ b/terraform/webdav.tf @@ -11,7 +11,7 @@ resource "heroku_app" "webdav" { # The Rust application is compiled and pushed to Heroku via a GitHub Action, so # we don't need to specify a specific buildpack here. So, we just fall back to # the Heroku CLI buildpack as a default. - "heroku-community/cli" + "https://buildpack-registry.s3.amazonaws.com/buildpacks/heroku-community/cli.tgz" ] } From 2de0231be39058deeadd138e68d62460b104ce66 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Thu, 4 Apr 2024 13:58:00 -0400 Subject: [PATCH 14/25] Put in a trailing dot --- terraform/domain.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/domain.tf b/terraform/domain.tf index 525ccd5..ec97f97 100644 --- a/terraform/domain.tf +++ b/terraform/domain.tf @@ -7,7 +7,7 @@ resource "aws_route53_record" "acm_validation" { name = "_cbe41dfe1888c2bb5c157cacc35e1722" type = "CNAME" ttl = "300" - records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws"] + records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws."] } resource "aws_route53_record" "gui" { From fe8f233cf01b02f2dc1e8d4b41e449ed975d9e6d Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 5 Apr 2024 13:40:29 -0400 Subject: [PATCH 15/25] WIP - remove trailing dot --- terraform/domain.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/domain.tf b/terraform/domain.tf index ec97f97..525ccd5 100644 --- a/terraform/domain.tf +++ b/terraform/domain.tf @@ -7,7 +7,7 @@ resource "aws_route53_record" "acm_validation" { name = "_cbe41dfe1888c2bb5c157cacc35e1722" type = "CNAME" ttl = "300" - records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws."] + records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws"] } resource "aws_route53_record" "gui" { From c37fd646b98a3af9818509b48b77de2bdfcc4027 Mon Sep 17 00:00:00 2001 From: Roni Choudhury <2903332+waxlamp@users.noreply.github.com> Date: Fri, 5 Apr 2024 15:29:07 -0400 Subject: [PATCH 16/25] Revert "Remove trailing dot for ACM CNAME entry" --- terraform/domain.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/domain.tf b/terraform/domain.tf index 525ccd5..ec97f97 100644 --- a/terraform/domain.tf +++ b/terraform/domain.tf @@ -7,7 +7,7 @@ resource "aws_route53_record" "acm_validation" { name = "_cbe41dfe1888c2bb5c157cacc35e1722" type = "CNAME" ttl = "300" - records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws"] + records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws."] } resource "aws_route53_record" "gui" { From 9a813476a3b8bbb87c9b95be3140f74bdc3dd02f Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:21:31 -0400 Subject: [PATCH 17/25] Give heroku user `s3:PutObjectTagging` permission --- terraform/modules/dandiset_bucket/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 10d69cb..3a2b2c7 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -116,7 +116,7 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { "${aws_s3_bucket.dandiset_bucket.arn}/*", ] - actions = ["s3:PutObject"] + actions = ["s3:PutObject", "s3:PutObjectTagging"] } } From 798f52db5a9e0487722bdab4ca1c27d4aeb51249 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Thu, 25 Apr 2024 16:43:13 -0400 Subject: [PATCH 18/25] Add dev_email variable and supply as env var --- terraform/api.tf | 1 + terraform/staging_pipeline.tf | 1 + terraform/variables.tf | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/terraform/api.tf b/terraform/api.tf index d3cc1d3..c736f5d 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -42,6 +42,7 @@ module "api" { DJANGO_DANDI_WEB_APP_URL = "https://dandiarchive.org" DJANGO_DANDI_API_URL = "https://api.dandiarchive.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.dandiarchive.org/" + DJANGO_DANDI_DEV_EMAIL = var.dev_email } additional_sensitive_django_vars = { DJANGO_DANDI_DOI_API_PASSWORD = var.doi_api_password diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index fdd3b0a..2de21de 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -41,6 +41,7 @@ module "api_staging" { DJANGO_DANDI_WEB_APP_URL = "https://gui-staging.dandiarchive.org" DJANGO_DANDI_API_URL = "https://api-staging.dandiarchive.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.dandiarchive.org/" + DJANGO_DANDI_DEV_EMAIL = var.dev_email } additional_sensitive_django_vars = { DJANGO_DANDI_DOI_API_PASSWORD = var.test_doi_api_password diff --git a/terraform/variables.tf b/terraform/variables.tf index 9d26cb7..b065f80 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -7,3 +7,8 @@ variable "test_doi_api_password" { type = string description = "The password for the Datacite Test API, used to mint new DOIs on staging during publish." } + +variable "dev_email" { + type = string + description = "The core developer email list." +} From 5de5bf43bcf9bc85910f0c42e9af8b06ba7546c7 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Mon, 29 Apr 2024 11:41:24 -0400 Subject: [PATCH 19/25] Allow PutObjectTagging for both production and staging buckets --- terraform/modules/dandiset_bucket/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 3a2b2c7..7707db6 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -108,7 +108,7 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { } dynamic "statement" { - for_each = var.allow_heroku_put_object ? [1] : [] + for_each = (var.allow_cross_account_heroku_put_object || var.allow_heroku_put_object) ? [1] : [] content { resources = [ From ef71895c9dd233b8d8f9adf239282acf37063432 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Mon, 29 Apr 2024 12:56:26 -0400 Subject: [PATCH 20/25] Add `s3:PutObjectTagging` permission to sponsored bucket --- terraform/modules/dandiset_bucket/main.tf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 7707db6..1ef7067 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -265,6 +265,23 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { } } + dynamic "statement" { + for_each = var.allow_cross_account_heroku_put_object ? [1] : [] + content { + resources = [ + "${aws_s3_bucket.dandiset_bucket.arn}", + "${aws_s3_bucket.dandiset_bucket.arn}/*", + ] + + actions = ["s3:PutObjectTagging"] + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + } + dynamic "statement" { for_each = var.trailing_delete ? [1] : [] From abb13a0b459f00eb4d469cd6e1114ae5309133a7 Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Fri, 26 Jul 2024 15:11:44 -0400 Subject: [PATCH 21/25] Update terraform to reflect correct heroku state --- terraform/staging_pipeline.tf | 2 +- terraform/webdav.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index 2de21de..8468157 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -12,7 +12,7 @@ module "api_staging" { heroku_web_dyno_size = "basic" heroku_worker_dyno_size = "basic" - heroku_postgresql_plan = "basic" + heroku_postgresql_plan = "essential-1" heroku_cloudamqp_plan = "tiger" heroku_papertrail_plan = "fixa" diff --git a/terraform/webdav.tf b/terraform/webdav.tf index 2f2bc37..50ec09e 100644 --- a/terraform/webdav.tf +++ b/terraform/webdav.tf @@ -18,7 +18,7 @@ resource "heroku_app" "webdav" { resource "heroku_formation" "webdav_heroku_web" { app_id = heroku_app.webdav.id type = "web" - size = "basic" + size = "standard-2x" quantity = 1 } From c17740202b5a6404e435573c8bce39482c6572d6 Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Fri, 26 Jul 2024 14:54:37 -0400 Subject: [PATCH 22/25] Add identity provider for terraform cloud This should allow terraform cloud to authenticate with AWS via a direct trust relationship instead of relying on a manual token placed in the environment variable of TFC. See https://developer.hashicorp.com/terraform/cloud-docs/workspaces/dynamic-provider-credentials#how-dynamic-credentials-work for documentation on how the dynamic credential system works. --- terraform/aws_oidc.tf | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 terraform/aws_oidc.tf diff --git a/terraform/aws_oidc.tf b/terraform/aws_oidc.tf new file mode 100644 index 0000000..dab09e3 --- /dev/null +++ b/terraform/aws_oidc.tf @@ -0,0 +1,43 @@ +data "tls_certificate" "tfc_certificate" { + url = "https://app.terraform.io" +} + +resource "aws_iam_openid_connect_provider" "tfc_provider" { + url = data.tls_certificate.tfc_certificate.url + client_id_list = ["aws.workload.identity"] + thumbprint_list = [data.tls_certificate.tfc_certificate.certificates[0].sha1_fingerprint] +} + +resource "aws_iam_role" "tfc_role" { + name = "terraform-cloud-role" + + assume_role_policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Federated" : "${aws_iam_openid_connect_provider.tfc_provider.arn}" + }, + "Action" : "sts:AssumeRoleWithWebIdentity", + "Condition" : { + "StringEquals" : { + "app.terraform.io:aud" : "${one(aws_iam_openid_connect_provider.tfc_provider.client_id_list)}" + }, + "StringLike" : { + "app.terraform.io:sub" : "organization:dandi:project:Default Project:workspace:dandi-prod:run_phase:*" + } + } + } + ] + }) +} + +data "aws_iam_policy" "administrator_access" { + arn = "arn:aws:iam::aws:policy/AdministratorAccess" +} + +resource "aws_iam_role_policy_attachment" "tfc_policy_attachment" { + role = aws_iam_role.tfc_role.name + policy_arn = data.aws_iam_policy.administrator_access.arn +} From de64322f0546f98ff5d165b6e39742883afb9aef Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Mon, 9 Sep 2024 14:23:54 -0400 Subject: [PATCH 23/25] Import new heroku postgres addon We recently upgraded our Heroku Postgres DB. This involved provisioning a new DB instance, which put our Terraform state out of sync. --- terraform/imports.tf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 terraform/imports.tf diff --git a/terraform/imports.tf b/terraform/imports.tf new file mode 100644 index 0000000..65b258b --- /dev/null +++ b/terraform/imports.tf @@ -0,0 +1,4 @@ +import { + to = module.api.module.heroku.heroku_addon.heroku_postgresql[0] + id = "ffbeccd0-d477-48a8-99fe-663aa123e318" +} From efdd8a83aa830590d3bfc53ec6467f81a9d99403 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:27:37 -0400 Subject: [PATCH 24/25] Remove import block This has been applied and can now be removed. --- terraform/imports.tf | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 terraform/imports.tf diff --git a/terraform/imports.tf b/terraform/imports.tf deleted file mode 100644 index 65b258b..0000000 --- a/terraform/imports.tf +++ /dev/null @@ -1,4 +0,0 @@ -import { - to = module.api.module.heroku.heroku_addon.heroku_postgresql[0] - id = "ffbeccd0-d477-48a8-99fe-663aa123e318" -} From 64fb49300eec21ca8f72998b6e87eea1e268cbdf Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Tue, 22 Oct 2024 13:47:34 -0400 Subject: [PATCH 25/25] Scope trailing delete policy to `blobs` prefix only --- terraform/modules/dandiset_bucket/main.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf index 1ef7067..3143fc1 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/dandiset_bucket/main.tf @@ -308,7 +308,7 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { # S3 lifecycle policy that permanently deletes objects with delete markers -# after 30 days. +# after 30 days. Note, this only applies to objects with the `blobs/` prefix. resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" { # Must have bucket versioning enabled first depends_on = [aws_s3_bucket_versioning.dandiset_bucket] @@ -320,7 +320,11 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" { # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7 rule { id = "ExpireOldDeleteMarkers" - filter {} + filter { + # We only want to expire objects with the `blobs/` prefix, i.e. Asset Blobs. + # Other objects in this bucket are not subject to this lifecycle policy. + prefix = "blobs/" + } # Expire objects with delete markers after 30 days noncurrent_version_expiration {