Update ALB Controller to provision NLBs

README.md

@@ -30,7 +30,10 @@ Each brokered AWS EKS provides:
 - Bound credential permissions are limited by namespace
 - cluster-wide logging to AWS CloudWatch with fluent-bit
 - Automatic TLS configuration using AWS Certificate Manager (ACM)
-- Cluster-wide ingress load-balancing with AWS-specific features (eg WAFv2 integration) [via ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/)
+- Load Balancing Capabilities:
+  - [Current] Using the NLB functionality requires that you also install the [AWS VPC CNI add-on](https://github.com/GSA/datagov-brokerpak-eks/pull/69/files#diff-54dc6204ad9b3495e7157b5dab706ac9b1e4f19d69f127eec9959e80e2b0aa93R34-R37)
+    - This can be managed by this module through setting `install_vpc_cni` to `1`
+  - [Deprecated] Cluster-wide ingress load-balancing with AWS-specific features (eg WAFv2 integration) [via ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/)
 - nginx ingress controller for routing and IaaS-independent deployments
 - Automatic DNSSEC configuration for the cluster using AWS Route 53
 - Automatic DNS configuration for workloads using AWS Route53 [via ExternalDNS](https://github.com/kubernetes-sigs/external-dns)

docs/instance-cleanup.md

@@ -35,6 +35,8 @@ You might end up in a situation where the broker is failing to cleanup resources
     - Delete corresponding certificate (it should not be in use if you already deleted the Load Balancer)
 1. [EFS > Filesystems](https://console.aws.amazon.com/efs/home#/file-systems)
     - Delete corresponding EFS file system
+1. [EBS > Volumes](https://console.aws.amazon.com/ec2/v2/home?#Volumes:)
+    - Delete the volumes related to any k8s pods created on the corresponding k8s cluster
 1. [VPC > NAT Gateways](https://console.aws.amazon.com/vpc/home#NatGateways:)
     - Delete the one corresponding to your cluster
       - If you don't know which one it is, look for the one tagged with the k8s cluster name

eks-service-definition.yml

@@ -46,6 +46,10 @@ provision:
     required: false
     type: array
     details: "A list of the desired AWS Compute types that the nodes will be launched with (e.g. [\"m5.large\"])"
+  - field_name: install_vpc_cni
+    required: false
+    tyep: boolean
+    details: "Control input to specify whether this module creates the vpc cni addon as part of the EKS deployment"
 
   computed_inputs:
   - name: instance_name
@@ -86,6 +90,10 @@ provision:
     type: array
     default: ["m5.large"]
     overwrite: true
+  - name: install_vpc_cni
+    type: boolean
+    default: true
+    overwrite: true
 
   outputs:
   - field_name: domain_name

terraform/provision/eks.tf

@@ -72,7 +72,7 @@ resource "aws_iam_role_policy_attachment" "AmazonEKSFargatePodExecutionRolePolic
 # can only be one create or destroy operation in flight at a time. So we want to
 # create as few as possible, and do it sequentially rather than in parallel.
 resource "aws_eks_fargate_profile" "default_namespaces" {
-  count = 0
+  count                  = 0
   cluster_name           = local.cluster_name
   fargate_profile_name   = "default-namespaces-${local.cluster_name}"
   pod_execution_role_arn = aws_iam_role.iam_role_fargate.arn

terraform/provision/ingress.tf

@@ -17,8 +17,7 @@ resource "aws_iam_openid_connect_provider" "cluster" {
 
 # Use a convenient module to install the AWS Load Balancer controller
 module "aws_load_balancer_controller" {
-  # source                    = "/local/path/to/terraform-kubernetes-aws-load-balancer-controller"
-  source           = "github.com/GSA/terraform-kubernetes-aws-load-balancer-controller.git?ref=v4.3.0gsa"
+  source           = "github.com/GSA/terraform-kubernetes-aws-load-balancer-controller.git?ref=v5.0.0"
   k8s_cluster_type = "eks"
   k8s_namespace    = "kube-system"
   aws_region_name  = data.aws_region.current.name
@@ -30,6 +29,14 @@ module "aws_load_balancer_controller" {
   aws_tags = merge(var.labels, { "domain" = local.domain })
 }
 
+# To support the controller using NLBs the AWS VPC CNI add-on must be installed.
+# See https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.3/guide/service/nlb/#prerequisites
+resource "aws_eks_addon" "vpc-cni" {
+  count        = var.install_vpc_cni ? 1 : 0
+  cluster_name = module.eks.cluster_id
+  addon_name   = "vpc-cni"
+}
+
 # ---------------------------------------------------------
 # Provision the Ingress Controller using Helm
 # ---------------------------------------------------------
@@ -46,10 +53,33 @@ resource "helm_release" "ingress_nginx" {
 
   dynamic "set" {
     for_each = {
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-scheme"           = "internet-facing",
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-ports"        = "https",
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-cert"         = aws_acm_certificate.cert.arn,
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-nlb-target-type"  = "ip"
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-type"             = "external",
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-proxy-protocol"   = "*",
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-backend-protocol" = "ssl"
+
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-name" = local.subdomain,
+      # "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-alpn-policy"    = "HTTP2Preferred",
+      "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-negotiation-policy" = "ELBSecurityPolicy-TLS-1-2-2017-01"
+      # Enable this to restrict clients by CIDR range
+      # "controller.service.annotations.service\\.beta\\.kubernetes\\.io/load-balancer-source-ranges"     = var.client-cidrs
+
+      # Enable this to accept ipv6 connections (right now errors with "You must
+      # specify subnets with an associated IPv6 CIDR block.")
+      # "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ip-address-type"=
+      # "dualstack",
+
+
+      # TODO: AWS WAF doesn't work with NLBs. We probably have to set up Cloudfront in
+      # front of the NLB in order to get that functionality back
+      # "alb.ingress.kubernetes.io/wafv2-acl-arn"              = aws_wafv2_web_acl.waf_acl.arn
       "controller.service.externalTrafficPolicy"     = "Local",
-      "controller.service.type"                      = "NodePort",
+      "controller.service.type"                      = "LoadBalancer",
       "controller.config.server-tokens"              = false,
-      "controller.config.use-proxy-protocol"         = false,
+      "controller.config.use-proxy-protocol"         = true,
       "controller.config.compute-full-forwarded-for" = true,
       "controller.config.use-forwarded-headers"      = true,
       "controller.metrics.enabled"                   = true,
@@ -72,30 +102,21 @@ resource "helm_release" "ingress_nginx" {
   }
   values = [<<-VALUES
     controller: 
-      extraArgs: 
-        http-port: 8080 
-        https-port: 8543 
-      containerPort: 
-        http: 8080 
-        https: 8543 
-      service: 
-        ports: 
-          http: 80 
-          https: 443 
-        targetPorts: 
-          http: 8080 
-          https: 8543 
       image: 
         allowPrivilegeEscalation: false
     VALUES
   ]
   depends_on = [
     null_resource.cluster-functional,
+    module.aws_load_balancer_controller,
+    aws_eks_addon.vpc-cni
   ]
 }
 
-# Give the controller time to react to any recent events (eg an ingress was
-# removed and an ALB needs to be deleted) before actually removing it.
+# Give the AWS LB controller time to react to any recent events (eg an ingress was
+# removed and an ALB needs to be deleted) before actually removing it. Any
+# Ingress or Service:LoadBalancer resource created in future should add this as
+# a depends_on in order to ensure an orderly destroy!
 resource "time_sleep" "alb_controller_destroy_delay" {
   depends_on       = [module.aws_load_balancer_controller]
   destroy_duration = "30s"
@@ -154,61 +175,6 @@ resource "aws_wafv2_web_acl" "waf_acl" {
 }
 
 
-resource "kubernetes_ingress" "alb_to_nginx" {
-  wait_for_load_balancer = true
-  metadata {
-    name      = "alb-ingress-to-nginx-ingress"
-    namespace = "kube-system"
-
-    labels = {
-      app = "nginx"
-    }
-
-    annotations = {
-      "alb.ingress.kubernetes.io/actions.ssl-redirect"       = "{\"Type\": \"redirect\", \"RedirectConfig\": { \"Protocol\": \"HTTPS\", \"Port\": \"443\", \"StatusCode\": \"HTTP_301\"}}",
-      "alb.ingress.kubernetes.io/backend-protocol"           = "HTTPS",
-      "alb.ingress.kubernetes.io/certificate-arn"            = aws_acm_certificate.cert.arn,
-      "alb.ingress.kubernetes.io/healthcheck-path"           = "/",
-      "alb.ingress.kubernetes.io/listen-ports"               = "[{\"HTTP\":80}, {\"HTTPS\":443}]",
-      "alb.ingress.kubernetes.io/load-balancer-attributes"   = "routing.http2.enabled=true,idle_timeout.timeout_seconds=60",
-      "alb.ingress.kubernetes.io/scheme"                     = "internet-facing",
-      "alb.ingress.kubernetes.io/shield-advanced-protection" = "false",
-      "alb.ingress.kubernetes.io/ssl-policy"                 = "ELBSecurityPolicy-TLS-1-2-2017-01",
-      "alb.ingress.kubernetes.io/target-type"                = "ip",
-      "alb.ingress.kubernetes.io/wafv2-acl-arn"              = aws_wafv2_web_acl.waf_acl.arn,
-      "kubernetes.io/ingress.class"                          = "alb",
-    }
-  }
-
-  spec {
-    rule {
-      http {
-        path {
-          path = "/*"
-          backend {
-            service_name = "ssl-redirect"
-            service_port = "use-annotation"
-          }
-        }
-        path {
-          path = "/*"
-          backend {
-            service_name = "ingress-nginx-controller"
-            service_port = "443"
-          }
-        }
-      }
-    }
-  }
-
-  depends_on = [
-    helm_release.ingress_nginx,
-    time_sleep.alb_controller_destroy_delay,
-    module.aws_load_balancer_controller,
-  ]
-}
-
-
 # Create ACM certificate for the sub-domain
 resource "aws_acm_certificate" "cert" {
   domain_name = local.domain
@@ -237,18 +203,39 @@ resource "aws_acm_certificate_validation" "cert" {
   validation_record_fqdns = [aws_route53_record.cert_validation.fqdn]
 }
 
-# Get the Ingress for the ALB
-data "aws_elb_hosted_zone_id" "elb_zone_id" {}
+data "kubernetes_service" "ingress_service" {
+  metadata {
+    name      = "ingress-nginx-controller"
+    namespace = "kube-system"
+  }
+  depends_on = [
+    helm_release.ingress_nginx
+  ]
+}
+
+# # Create a local variable for the load balancer name.
+# locals {
+#   nlb_name = split("-", split(".", data.kubernetes_service.ingress_service.status.0.load_balancer.0.ingress.0.hostname).0).0
+# }
+
+# Read information about the NLB created for the ingress service
+data "aws_lb" "ingress_nlb" {
+  name = local.subdomain
+  depends_on = [
+    data.kubernetes_service.ingress_service,
+    aws_route53_record.cert_validation
+  ]
+}
 
-# Create CNAME record in sub-domain hosted zone for the ALB
-resource "aws_route53_record" "www" {
+# Create an A record in the subdomain zone aliased to the NLB
+resource "aws_route53_record" "nlb" {
   zone_id = aws_route53_zone.cluster.id
   name    = local.domain
   type    = "A"
 
   alias {
-    name                   = kubernetes_ingress.alb_to_nginx.status[0].load_balancer[0].ingress[0].hostname
-    zone_id                = data.aws_elb_hosted_zone_id.elb_zone_id.id
+    name                   = data.aws_lb.ingress_nlb.dns_name
+    zone_id                = data.aws_lb.ingress_nlb.zone_id
     evaluate_target_health = true
   }
 }

terraform/provision/persistent-storage.tf

@@ -163,6 +163,6 @@ resource "kubernetes_storage_class" "ebs-sc" {
   metadata {
     name = "ebs-sc"
   }
-  storage_provisioner = "ebs.csi.aws.com"
+  storage_provisioner    = "ebs.csi.aws.com"
   allow_volume_expansion = true
 }

terraform/provision/variables.tf

@@ -29,6 +29,11 @@ variable "mng_instance_types" {
   default = ["m4.xlarge"]
 }
 
+variable "install_vpc_cni" {
+  type    = bool
+  default = true
+}
+
 variable "labels" {
   type    = map(any)
   default = {}