Merge branch 't-shirt' into veronica/tshirt-deployment

wandb · Jan 30, 2024 · dcaede9 · dcaede9
2 parents b0ea7c8 + 3a7767f
commit dcaede9
Show file tree

Hide file tree

Showing 23 changed files with 382 additions and 114 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,80 @@
 
 All notable changes to this project will be documented in this file.
 
+### [4.1.2](https://github.com/wandb/terraform-aws-wandb/compare/v4.1.1...v4.1.2) (2024-01-16)
+
+
+### Bug Fixes
+
+* Max LB name length ([#166](https://github.com/wandb/terraform-aws-wandb/issues/166)) ([85bd266](https://github.com/wandb/terraform-aws-wandb/commit/85bd266f5f0ce003f2d4e69f796a41df0ff9fb9c))
+
+### [4.1.1](https://github.com/wandb/terraform-aws-wandb/compare/v4.1.0...v4.1.1) (2024-01-11)
+
+
+### Bug Fixes
+
+* Update redis connection ttl ([#165](https://github.com/wandb/terraform-aws-wandb/issues/165)) ([f236b3b](https://github.com/wandb/terraform-aws-wandb/commit/f236b3b8c5f7d3fcece1a1d302276bde6bdd75d5))
+
+## [4.1.0](https://github.com/wandb/terraform-aws-wandb/compare/v4.0.2...v4.1.0) (2024-01-10)
+
+
+### Features
+
+* Adding flags to switch between LB ([#159](https://github.com/wandb/terraform-aws-wandb/issues/159)) ([ffa3778](https://github.com/wandb/terraform-aws-wandb/commit/ffa3778fe05da8681a828ce84f3f8291bb8fe5bd))
+
+### [4.0.2](https://github.com/wandb/terraform-aws-wandb/compare/v4.0.1...v4.0.2) (2024-01-09)
+
+
+### Bug Fixes
+
+* EFS index vs subnet for_each ([#163](https://github.com/wandb/terraform-aws-wandb/issues/163)) ([1e47177](https://github.com/wandb/terraform-aws-wandb/commit/1e47177a0017ef694e7667781111d9ce2d375f2b))
+
+### [4.0.1](https://github.com/wandb/terraform-aws-wandb/compare/v4.0.0...v4.0.1) (2024-01-09)
+
+
+### Bug Fixes
+
+* Disable gorilla glue tasks ([#161](https://github.com/wandb/terraform-aws-wandb/issues/161)) ([5d24bda](https://github.com/wandb/terraform-aws-wandb/commit/5d24bda4fead8d79b3e06d488ecb824980a3d15b))
+
+## [4.0.0](https://github.com/wandb/terraform-aws-wandb/compare/v3.4.2...v4.0.0) (2024-01-08)
+
+
+### ⚠ BREAKING CHANGES
+
+* Init operator (#154)
+
+### Features
+
+* Init operator ([#154](https://github.com/wandb/terraform-aws-wandb/issues/154)) ([95def33](https://github.com/wandb/terraform-aws-wandb/commit/95def33db96c55a640fba4df5bdfbcc3a179d8ac))
+
+### [3.4.2](https://github.com/wandb/terraform-aws-wandb/compare/v3.4.1...v3.4.2) (2023-12-07)
+
+
+### Bug Fixes
+
+* Switch to gp3 volumes on EKS nodes ([#146](https://github.com/wandb/terraform-aws-wandb/issues/146)) ([86dbc7d](https://github.com/wandb/terraform-aws-wandb/commit/86dbc7df0de1aa6d2bc69862770ea67010354c20))
+
+### [3.4.1](https://github.com/wandb/terraform-aws-wandb/compare/v3.4.0...v3.4.1) (2023-12-07)
+
+
+### Bug Fixes
+
+* Fix private access example ([#155](https://github.com/wandb/terraform-aws-wandb/issues/155)) ([f0745ea](https://github.com/wandb/terraform-aws-wandb/commit/f0745eaf3e2d7111b6becfccac3134b18961e862))
+
+## [3.4.0](https://github.com/wandb/terraform-aws-wandb/compare/v3.3.0...v3.4.0) (2023-11-13)
+
+
+### Features
+
+* Add support for AWS Secrets Manager ([#151](https://github.com/wandb/terraform-aws-wandb/issues/151)) ([aa64eb1](https://github.com/wandb/terraform-aws-wandb/commit/aa64eb146622132d9b70083094b3c60a728e6038))
+
+## [3.3.0](https://github.com/wandb/terraform-aws-wandb/compare/v3.2.0...v3.3.0) (2023-10-31)
+
+
+### Features
+
+* Remove vpc-cni EKS add-on in app_eks/main.tf ([#150](https://github.com/wandb/terraform-aws-wandb/issues/150)) ([9f01dde](https://github.com/wandb/terraform-aws-wandb/commit/9f01dde88971487622111e71ec2871b7445b5f57))
+
 ## [3.2.0](https://github.com/wandb/terraform-aws-wandb/compare/v3.1.0...v3.2.0) (2023-10-25)
 
 

diff --git a/README.md b/README.md
@@ -217,3 +217,28 @@ Upgrades must be executed in step-wise fashion from one version to the next. You
 | <a name="output_url"></a> [url](#output\_url) | The URL to the W&B application |
 
 <!-- END_TF_DOCS -->
+
+## Migrations
+
+#### Upgrading from 3.x -> 4.x
+
+- If egress access for retrieving the wandb/controller image is not available, Terraform apply may experience failures.
+- It's necessary to supply a license variable within the module, as shown:
+
+```hcl
+module "wandb" {
+  version = "4.x"
+
+  # ...
+  license = "<your license key>"
+  # ...
+}
+```
+
+### Upgrading from 2.x -> 3.x
+
+- No changes required by you
+
+### Upgrading from 1.x -> 2.x
+
+- ~>4.0 version required for AWS Provider
diff --git a/examples/private-access-only/main.tf b/examples/private-access-only/main.tf
@@ -61,18 +61,19 @@ module "standard" {
   namespace     = var.namespace
   public_access = false
 
-  wandb_license = var.wandb_license
-
   domain_name = aws_route53_zone.private.name
   zone_id     = aws_route53_zone.private.zone_id
 
   # Creating a custom VPC so that we can initalize a route53 zone first and configure a vpn
-  create_vpc              = false
-  network_id              = module.networking.vpc_id
-  network_private_subnets = module.networking.private_subnets
-  network_public_subnets  = module.networking.public_subnets
+  create_vpc                = false
+  network_id                = module.networking.vpc_id
+  network_private_subnets   = module.networking.private_subnets
+  network_public_subnets    = module.networking.public_subnets
+  allowed_inbound_cidr      = var.allowed_inbound_cidr
+  allowed_inbound_ipv6_cidr = var.allowed_inbound_ipv6_cidr
+  eks_cluster_version       = var.eks_cluster_version
 }
 
 output "url" {
   value = module.standard.url
-}
+}
diff --git a/examples/private-access-only/variables.tf b/examples/private-access-only/variables.tf
@@ -3,7 +3,22 @@ variable "namespace" {
   description = "Name prefix used for resources"
 }
 
-variable "wandb_license" {
-  type = string
+variable "allowed_inbound_cidr" {
+  default  = ["0.0.0.0/0"]
+  nullable = false
+  type     = list(string)
 }
 
+
+variable "allowed_inbound_ipv6_cidr" {
+  default  = ["::/0"]
+  nullable = false
+  type     = list(string)
+}
+
+variable "eks_cluster_version" {
+  description = "EKS cluster kubernetes version"
+  default     = "1.26"
+  nullable    = false
+  type        = string
+}
diff --git a/examples/private-access-only/versions.tf b/examples/private-access-only/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.60"
+      version = "~> 4.0"
     }
   }
-}
+}
diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf
@@ -1,5 +1,5 @@
 provider "aws" {
-  region  = "us-west-2"
+  region = "us-west-2"
 
   default_tags {
     tags = {
@@ -36,7 +36,7 @@ module "wandb_infra" {
   zone_id     = var.zone_id
   subdomain   = var.subdomain
 
-  # license = var.wandb_license
+  license = var.wandb_license
 
   bucket_name        = var.bucket_name
   bucket_kms_key_arn = var.bucket_kms_key_arn
@@ -87,6 +87,10 @@ module "wandb_app" {
   # If we dont wait, tf will start trying to deploy while the work group is
   # still spinning up
   depends_on = [module.wandb_infra]
+
+  other_wandb_env = merge({
+    "GORILLA_CUSTOMER_SECRET_STORE_SOURCE" = "aws-secretmanager://${var.namespace}?namespace=${var.namespace}"
+  }, var.other_wandb_env)
 }
 
 output "bucket_name" {

diff --git a/examples/public-dns-external/variables.tf b/examples/public-dns-external/variables.tf
@@ -83,3 +83,9 @@ variable "allowed_inbound_ipv6_cidr" {
   nullable = false
   type     = list(string)
 }
+
+variable "other_wandb_env" {
+  type        = map(string)
+  description = "Extra environment variables for W&B"
+  default     = {}
+}
diff --git a/main.tf b/main.tf
@@ -141,14 +141,15 @@ module "acm" {
 locals {
   acm_certificate_arn = local.create_certificate ? module.acm.acm_certificate_arn : var.acm_certificate_arn
   url                 = local.acm_certificate_arn == null ? "http://${local.fqdn}" : "https://${local.fqdn}"
+  domain_filter       = var.custom_domain_filter == null || var.custom_domain_filter == "" ? local.fqdn : var.custom_domain_filter
 
   internal_app_port = 32543
 }
 
 module "app_eks" {
   source = "./modules/app_eks"
 
-  fqdn = local.fqdn
+  fqdn = local.domain_filter
 
   namespace   = var.namespace
   kms_key_arn = local.kms_key_arn
@@ -187,7 +188,7 @@ module "app_lb" {
   acm_certificate_arn   = local.acm_certificate_arn
   zone_id               = var.zone_id
 
-  fqdn                      = local.fqdn
+  fqdn                      = var.enable_dummy_dns ? "old.${local.fqdn}" : local.fqdn
   extra_fqdn                = var.extra_fqdn
   allowed_inbound_cidr      = var.allowed_inbound_cidr
   allowed_inbound_ipv6_cidr = var.allowed_inbound_ipv6_cidr
@@ -216,63 +217,87 @@ module "redis" {
   kms_key_arn = local.kms_key_arn
 }
 
+locals {
+  max_lb_name_length = 32 - length("-alb-k8s")
+  lb_name_truncated  = "${substr(var.namespace, 0, local.max_lb_name_length)}-alb-k8s"
+}
+
+module "wandb" {
+  source  = "wandb/wandb/helm"
+  version = "1.2.0"
+
+  depends_on = [
+    module.database,
+    module.app_eks,
+    module.redis,
+  ]
+  operator_chart_version = "1.1.0"
+  controller_image_tag   = "1.10.1"
+
+  spec = {
+    values = {
+      global = {
+        host    = local.url
+        license = var.license
+
+        extraEnv = var.other_wandb_env
+
+        bucket = {
+          provider = "s3"
+          name     = local.bucket_name
+          region   = data.aws_s3_bucket.file_storage.region
+          kmsKey   = local.use_external_bucket ? var.bucket_kms_key_arn : local.kms_key_arn
+        }
+
+        mysql = {
+          host     = module.database.endpoint
+          password = module.database.password
+          user     = module.database.username
+          database = module.database.database_name
+          port     = module.database.port
+        }
+
+        redis = {
+          host = module.redis.0.host
+          port = "${module.redis.0.port}?tls=true&ttlInSeconds=604800"
+        }
+      }
+
+      ingress = {
+        class = "alb"
+
+        annotations = {
+          "alb.ingress.kubernetes.io/load-balancer-name"             = local.lb_name_truncated
+          "alb.ingress.kubernetes.io/inbound-cidrs"                  = <<-EOF
+            ${join("\\,", var.allowed_inbound_cidr)}
+          EOF
+          "external-dns.alpha.kubernetes.io/hostname"                = var.enable_operator_alb ? local.fqdn : ""
+          "external-dns.alpha.kubernetes.io/ingress-hostname-source" = "annotation-only"
+          "alb.ingress.kubernetes.io/scheme"                         = "internet-facing"
+          "alb.ingress.kubernetes.io/target-type"                    = "ip"
+          "alb.ingress.kubernetes.io/listen-ports"                   = "[{\\\"HTTPS\\\": 443}]"
+          "alb.ingress.kubernetes.io/certificate-arn"                = local.acm_certificate_arn
+        }
+      }
+
+      app = var.enable_operator_alb ? {} : {
+        extraEnv = {
+          "GORILLA_GLUE_LIST" = "true"
+        }
+      }
+
+      mysql = { install = false }
+      redis = { install = false }
+
+      weave = {
+        persistence = {
+          provider = "efs"
+          efs = {
+            fileSystemId = module.app_eks.efs_id
+          }
+        }
+      }
+    }
+  }
+}
 
-# Comming soon!
-# module "wandb" {
-#   source  = "wandb/wandb/helm"
-#   version = "1.2.0"
-
-#   depends_on = [
-#     module.database,
-#     module.app_eks,
-#     module.redis,
-#   ]
-
-#   operator_chart_version = "1.1.0"
-#   controller_image_tag   = "1.10.1"
-
-#   spec = {
-#     values = {
-#       global = {
-#         host    = local.url
-#         license = var.license
-
-#         bucket = {
-#           provider = "s3"
-#           name     = local.bucket_name
-#           region   = data.aws_s3_bucket.file_storage.region
-#           kmsKey   = local.kms_key_arn
-#         }
-
-#         mysql = {
-#           host     = module.database.endpoint
-#           password = module.database.password
-#           username = module.database.username
-#           database = module.database.database_name
-#           port     = module.database.port
-#         }
-
-#         redis = {
-#           host = module.redis.0.host
-#           port = "${module.redis.0.port}?tls=true"
-#         }
-#       }
-
-#       ingress = {
-#         class = "alb"
-
-#         annotations = {
-#           "alb.ingress.kubernetes.io/load-balancer-name" = "${var.namespace}-alb-k8s"
-#           "alb.ingress.kubernetes.io/inbound-cidrs"      = "0.0.0.0/0"
-#           "alb.ingress.kubernetes.io/scheme"             = "internet-facing"
-#           "alb.ingress.kubernetes.io/target-type"        = "ip"
-#           "alb.ingress.kubernetes.io/listen-ports"       = "[{\\\"HTTPS\\\": 443}]"
-#           "alb.ingress.kubernetes.io/certificate-arn"    = local.acm_certificate_arn
-#         }
-#       }
-
-#       mysql = { install = false }
-#       redis = { install = false }
-#     }
-#   }
-# }