[Do not merge] PR for diff of DANDI vs. LINC

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  lint-terraform:
+    name: Lint Terraform code
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: terraform
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Lint Terraform code
+        run: terraform fmt -check -diff -recursive
+
+      - name: Initialize Terraform (offline only)
+        run: terraform init -backend=false
+
+      - name: Validate Terraform code
+        run: terraform validate

terraform/api.tf

@@ -17,7 +17,7 @@ module "api" {
   heroku_cloudamqp_plan   = "squirrel-1"
   heroku_papertrail_plan  = "liatorp"
 
-  heroku_web_dyno_quantity    = 1
+  heroku_web_dyno_quantity    = 3
   heroku_worker_dyno_quantity = 1
 
   django_default_from_email          = "[email protected]"
@@ -42,6 +42,7 @@ module "api" {
     DJANGO_DANDI_WEB_APP_URL                       = "https://dandiarchive.org"
     DJANGO_DANDI_API_URL                           = "https://api.dandiarchive.org"
     DJANGO_DANDI_JUPYTERHUB_URL                    = "https://hub.dandiarchive.org/"
+    DJANGO_DANDI_DEV_EMAIL                         = var.dev_email
   }
   additional_sensitive_django_vars = {
     DJANGO_DANDI_DOI_API_PASSWORD = var.doi_api_password

terraform/aws_oidc.tf

@@ -0,0 +1,43 @@
+data "tls_certificate" "tfc_certificate" {
+  url = "https://app.terraform.io"
+}
+
+resource "aws_iam_openid_connect_provider" "tfc_provider" {
+  url             = data.tls_certificate.tfc_certificate.url
+  client_id_list  = ["aws.workload.identity"]
+  thumbprint_list = [data.tls_certificate.tfc_certificate.certificates[0].sha1_fingerprint]
+}
+
+resource "aws_iam_role" "tfc_role" {
+  name = "terraform-cloud-role"
+
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Federated" : "${aws_iam_openid_connect_provider.tfc_provider.arn}"
+        },
+        "Action" : "sts:AssumeRoleWithWebIdentity",
+        "Condition" : {
+          "StringEquals" : {
+            "app.terraform.io:aud" : "${one(aws_iam_openid_connect_provider.tfc_provider.client_id_list)}"
+          },
+          "StringLike" : {
+            "app.terraform.io:sub" : "organization:dandi:project:Default Project:workspace:dandi-prod:run_phase:*"
+          }
+        }
+      }
+    ]
+  })
+}
+
+data "aws_iam_policy" "administrator_access" {
+  arn = "arn:aws:iam::aws:policy/AdministratorAccess"
+}
+
+resource "aws_iam_role_policy_attachment" "tfc_policy_attachment" {
+  role       = aws_iam_role.tfc_role.name
+  policy_arn = data.aws_iam_policy.administrator_access.arn
+}

terraform/domain.tf

@@ -7,7 +7,7 @@ resource "aws_route53_record" "acm_validation" {
   name    = "_cbe41dfe1888c2bb5c157cacc35e1722"
   type    = "CNAME"
   ttl     = "300"
-  records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws"]
+  records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws."]
 }
 
 resource "aws_route53_record" "gui" {

terraform/modules/dandiset_bucket/main.tf

@@ -34,6 +34,7 @@ resource "aws_s3_bucket_cors_configuration" "dandiset_bucket" {
       "PUT",
       "POST",
       "GET",
+      "HEAD",
       "DELETE",
     ]
     allowed_headers = [
@@ -107,15 +108,15 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" {
   }
 
   dynamic "statement" {
-    for_each = var.allow_heroku_put_object ? [1] : []
+    for_each = (var.allow_cross_account_heroku_put_object || var.allow_heroku_put_object) ? [1] : []
     content {
 
       resources = [
         "${aws_s3_bucket.dandiset_bucket.arn}",
         "${aws_s3_bucket.dandiset_bucket.arn}/*",
       ]
 
-      actions = ["s3:PutObject"]
+      actions = ["s3:PutObject", "s3:PutObjectTagging"]
     }
   }
 
@@ -166,6 +167,33 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
     }
   }
 
+  # Disallow access to embargoed objects, unless using the heroku user arn
+  dynamic "statement" {
+    for_each = var.public ? [1] : []
+
+    content {
+      effect = "Deny"
+      principals {
+        identifiers = ["*"]
+        type        = "*"
+      }
+      actions = ["s3:*"]
+      resources = [
+        "${aws_s3_bucket.dandiset_bucket.arn}/*",
+      ]
+      condition {
+        test     = "StringEquals"
+        variable = "s3:ExistingObjectTag/embargoed"
+        values   = ["true"]
+      }
+      condition {
+        test     = "ArnNotEquals"
+        variable = "aws:PrincipalArn"
+        values   = [var.heroku_user.arn]
+      }
+    }
+  }
+
   dynamic "statement" {
     for_each = var.allow_cross_account_heroku_put_object ? [1] : []
 
@@ -237,6 +265,23 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
     }
   }
 
+  dynamic "statement" {
+    for_each = var.allow_cross_account_heroku_put_object ? [1] : []
+    content {
+      resources = [
+        "${aws_s3_bucket.dandiset_bucket.arn}",
+        "${aws_s3_bucket.dandiset_bucket.arn}/*",
+      ]
+
+      actions = ["s3:PutObjectTagging"]
+
+      principals {
+        type        = "AWS"
+        identifiers = [var.heroku_user.arn]
+      }
+    }
+  }
+
   dynamic "statement" {
     for_each = var.trailing_delete ? [1] : []
 
@@ -263,7 +308,7 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
 
 
 # S3 lifecycle policy that permanently deletes objects with delete markers
-# after 30 days.
+# after 30 days. Note, this only applies to objects with the `blobs/` prefix.
 resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
   # Must have bucket versioning enabled first
   depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
@@ -275,7 +320,11 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
   # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7
   rule {
     id = "ExpireOldDeleteMarkers"
-    filter {}
+    filter {
+      # We only want to expire objects with the `blobs/` prefix, i.e. Asset Blobs.
+      # Other objects in this bucket are not subject to this lifecycle policy.
+      prefix = "blobs/"
+    }
 
     # Expire objects with delete markers after 30 days
     noncurrent_version_expiration {

terraform/modules/dandiset_bucket/variables.tf

@@ -38,6 +38,6 @@ variable "log_bucket_name" {
 # TODO: this can be inferred from the "versioning" variable once we're ready
 # to deploy this to the production bucket as well.
 variable "trailing_delete" {
-  type = bool
+  type        = bool
   description = "Whether or not trailing delete should be enabled on the bucket."
 }

terraform/staging_pipeline.tf

@@ -12,7 +12,7 @@ module "api_staging" {
 
   heroku_web_dyno_size    = "basic"
   heroku_worker_dyno_size = "basic"
-  heroku_postgresql_plan  = "basic"
+  heroku_postgresql_plan  = "essential-1"
   heroku_cloudamqp_plan   = "tiger"
   heroku_papertrail_plan  = "fixa"
 
@@ -41,6 +41,7 @@ module "api_staging" {
     DJANGO_DANDI_WEB_APP_URL                       = "https://gui-staging.dandiarchive.org"
     DJANGO_DANDI_API_URL                           = "https://api-staging.dandiarchive.org"
     DJANGO_DANDI_JUPYTERHUB_URL                    = "https://hub.dandiarchive.org/"
+    DJANGO_DANDI_DEV_EMAIL                         = var.dev_email
   }
   additional_sensitive_django_vars = {
     DJANGO_DANDI_DOI_API_PASSWORD = var.test_doi_api_password

terraform/variables.tf

@@ -7,3 +7,8 @@ variable "test_doi_api_password" {
   type        = string
   description = "The password for the Datacite Test API, used to mint new DOIs on staging during publish."
 }
+
+variable "dev_email" {
+  type        = string
+  description = "The core developer email list."
+}

terraform/webdav.tf

@@ -0,0 +1,36 @@
+resource "heroku_app" "webdav" {
+  name   = "dandidav"
+  region = "us"
+  acm    = true
+
+  organization {
+    name = data.heroku_team.dandi.name
+  }
+
+  buildpacks = [
+    # The Rust application is compiled and pushed to Heroku via a GitHub Action, so
+    # we don't need to specify a specific buildpack here. So, we just fall back to
+    # the Heroku CLI buildpack as a default.
+    "https://buildpack-registry.s3.amazonaws.com/buildpacks/heroku-community/cli.tgz"
+  ]
+}
+
+resource "heroku_formation" "webdav_heroku_web" {
+  app_id   = heroku_app.webdav.id
+  type     = "web"
+  size     = "standard-2x"
+  quantity = 1
+}
+
+resource "heroku_domain" "webdav" {
+  app_id   = heroku_app.webdav.id
+  hostname = "webdav.dandiarchive.org"
+}
+
+resource "aws_route53_record" "heroku" {
+  zone_id = aws_route53_zone.dandi.zone_id
+  name    = "webdav"
+  type    = "CNAME"
+  ttl     = "300"
+  records = [heroku_domain.webdav.cname]
+}