fix: Rework the PSC connection to support SSL and be available globally

README.md

@@ -78,6 +78,7 @@ resources that lack official modules.
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.0 |
 | <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.30 |
+| <a name="requirement_google-beta"></a> [google-beta](#requirement\_google-beta) | ~> 5.30 |
 | <a name="requirement_helm"></a> [helm](#requirement\_helm) | ~> 2.10 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | ~> 2.23 |
 | <a name="requirement_time"></a> [time](#requirement\_time) | 0.11.2 |
@@ -106,7 +107,6 @@ resources that lack official modules.
 | <a name="module_project_factory_project_services"></a> [project\_factory\_project\_services](#module\_project\_factory\_project\_services) | terraform-google-modules/project-factory/google//modules/project_services | ~> 14.0 |
 | <a name="module_redis"></a> [redis](#module\_redis) | ./modules/redis | n/a |
 | <a name="module_service_accounts"></a> [service\_accounts](#module\_service\_accounts) | ./modules/service_accounts | n/a |
-| <a name="module_sleep"></a> [sleep](#module\_sleep) | matti/resource/shell | 1.5.0 |
 | <a name="module_storage"></a> [storage](#module\_storage) | ./modules/storage | n/a |
 | <a name="module_wandb"></a> [wandb](#module\_wandb) | wandb/wandb/helm | 1.2.0 |
 
@@ -115,7 +115,6 @@ resources that lack official modules.
 | Name | Type |
 |------|------|
 | [google_client_config.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source |
-| [google_compute_forwarding_rules.all](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_forwarding_rules) | data source |
 
 ## Inputs
 
@@ -206,7 +205,7 @@ resources that lack official modules.
 | <a name="output_gke_max_node_count"></a> [gke\_max\_node\_count](#output\_gke\_max\_node\_count) | n/a |
 | <a name="output_gke_node_count"></a> [gke\_node\_count](#output\_gke\_node\_count) | n/a |
 | <a name="output_gke_node_instance_type"></a> [gke\_node\_instance\_type](#output\_gke\_node\_instance\_type) | n/a |
-| <a name="output_private_attachement_id"></a> [private\_attachement\_id](#output\_private\_attachement\_id) | n/a |
+| <a name="output_private_attachment_id"></a> [private\_attachment\_id](#output\_private\_attachment\_id) | n/a |
 | <a name="output_sa_account_email"></a> [sa\_account\_email](#output\_sa\_account\_email) | This output provides the email address of the service account created for workload identity, if workload identity is enabled. Otherwise, it returns null |
 | <a name="output_service_account"></a> [service\_account](#output\_service\_account) | Weights & Biases service account used to manage resources. |
 | <a name="output_standardized_size"></a> [standardized\_size](#output\_standardized\_size) | n/a |

main.tf

@@ -129,10 +129,12 @@ module "app_gke" {
 }
 
 module "cloud_nat" {
-  count     = var.enable_private_gke_nodes ? 1 : 0
+  count     = var.enable_private_gke_nodes || var.create_private_link ? 1 : 0
   source    = "./modules/cloud_nat"
   network   = local.network
   namespace = var.namespace
+  vpc_nat   = var.enable_private_gke_nodes
+  proxy_nat = var.create_private_link
 }
 
 module "app_lb" {
@@ -254,7 +256,6 @@ locals {
     "OIDC_AUTH_METHOD" = var.oidc_auth_method
     "OIDC_SECRET"      = var.oidc_secret
   } : {}
-  internal_lb_name = "${var.namespace}-internal"
 }
 
 locals {
@@ -343,15 +344,6 @@ module "wandb" {
           "kubernetes.io/ingress.global-static-ip-name" = module.app_lb.address_operator_name
           "ingress.gcp.kubernetes.io/pre-shared-cert"   = module.app_lb.certificate
         }
-        ## In order to support secondary ingress required min version 0.13.0 of operator-wandb chart
-        secondary = {
-          create       = var.create_private_link # internal ingress for private link connections
-          nameOverride = local.internal_lb_name
-          annotations = {
-            "kubernetes.io/ingress.class"                   = "gce-internal"
-            "kubernetes.io/ingress.regional-static-ip-name" = var.create_private_link ? google_compute_address.default[0].name : null
-          }
-        }
       }
 
       # To support otel rds and redis metrics need operator-wandb chart minimum version 0.13.8 ( stackdriver subchart)
@@ -427,58 +419,16 @@ module "wandb" {
   ]
 }
 
-# proxy-only subnet used by internal load balancer
-resource "google_compute_subnetwork" "proxy" {
-  count         = var.create_private_link ? 1 : 0
-  name          = "${var.namespace}-proxy-subnet"
-  ip_cidr_range = var.ilb_proxynetwork_cidr
-  purpose       = "REGIONAL_MANAGED_PROXY"
-  role          = "ACTIVE"
-  network       = local.network.id
-  timeouts {
-    delete = "2m"
-  }
-}
-
-## This ensures that the private link resource does not fail during the provisioning process.
-module "sleep" {
-  count   = var.create_private_link ? 1 : 0
-  source  = "matti/resource/shell"
-  version = "1.5.0"
-
-  environment = {
-    TIME = timestamp()
-  }
-  command              = "sleep 400; date +%s"
-  command_when_destroy = "sleep 400"
-  trigger              = timestamp()
-  working_dir          = "/tmp"
-
-  depends = [
-    module.wandb
-  ]
-}
-
-data "google_compute_forwarding_rules" "all" {
-  depends_on = [module.sleep.stdout]
-}
-
-locals {
-  regex_pattern       = local.internal_lb_name
-  filtered_rule_names = [for rule in data.google_compute_forwarding_rules.all.rules : rule.name if can(regex(local.regex_pattern, rule.name))]
-  forwarding_rule     = join(", ", local.filtered_rule_names)
-}
-
 ## In order to support private link required min version 0.13.0 of operator-wandb chart
 module "private_link" {
   count                 = var.create_private_link ? 1 : 0
   source                = "./modules/private_link"
   namespace             = var.namespace
-  forwarding_rule       = local.forwarding_rule
   network               = local.network
   subnetwork            = local.subnetwork
   allowed_project_names = var.allowed_project_names
   psc_subnetwork        = var.psc_subnetwork_cidr
   proxynetwork_cidr     = var.ilb_proxynetwork_cidr
-  depends_on            = [google_compute_subnetwork.proxy, data.google_compute_forwarding_rules.all]
+  fqdn                  = local.fqdn
+  depends_on            = [module.wandb]
 }

modules/app_gke/main.tf

@@ -103,6 +103,12 @@ resource "google_container_node_pool" "default" {
       enable_secure_boot = true
     }
 
+    kubelet_config {
+      cpu_manager_policy = "none"
+      cpu_cfs_quota      = true
+      pod_pids_limit     = 0
+    }
+
     metadata = {
       disable-legacy-endpoints = "true"
     }

modules/cloud_nat/main.tf

@@ -14,10 +14,22 @@ resource "google_compute_address" "this" {
 
 # create cloud nat public gateway
 resource "google_compute_router_nat" "nat" {
+  count                              = var.vpc_nat ? 1 : 0
   name                               = "${var.namespace}-cloud-nat"
   router                             = google_compute_router.this.name
   region                             = google_compute_router.this.region
   nat_ip_allocate_option             = "MANUAL_ONLY"
   nat_ips                            = google_compute_address.this.*.self_link
   source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+}
+
+# create cloud nat public gateway for Private Service Connect LB
+resource "google_compute_router_nat" "nat_lb_proxy" {
+  count                              = var.proxy_nat ? 1 : 0
+  name                               = "${var.namespace}-cloud-nat-lb-proxy"
+  router                             = google_compute_router.this.name
+  region                             = google_compute_router.this.region
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  endpoint_types                     = ["ENDPOINT_TYPE_MANAGED_PROXY_LB"]
 }

modules/cloud_nat/variables.tf

@@ -5,4 +5,14 @@ variable "namespace" {
 variable "network" {
   description = "Google Compute Engine network to which the cluster is connected."
   type        = object({ self_link = string })
+}
+
+variable "proxy_nat" {
+  description = "Enable NAT for the Load Balancer Proxy Subnets"
+  type        = bool
+}
+
+variable "vpc_nat" {
+  description = "Enable NAT for the VPC"
+  type        = bool
 }

modules/private_link/main.tf

@@ -1,11 +1,67 @@
 data "google_client_config" "current" {}
 
+# proxy-only subnet used by internal load balancer
+resource "google_compute_subnetwork" "proxy" {
+  name          = "${var.namespace}-proxy-subnet"
+  ip_cidr_range = var.proxynetwork_cidr
+  purpose       = "REGIONAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+  network       = var.network.id
+  timeouts {
+    delete = "2m"
+  }
+}
+
+resource "google_compute_region_network_endpoint_group" "external_lb" {
+  name   = "${var.namespace}-psc-lb-neg"
+  region = data.google_client_config.current.region
+
+  network_endpoint_type = "INTERNET_FQDN_PORT"
+  network               = var.network.id
+}
+
+resource "google_compute_region_network_endpoint" "external_lb" {
+  region_network_endpoint_group = google_compute_region_network_endpoint_group.external_lb.name
+
+  fqdn = var.fqdn
+  port = 443
+}
+
+resource "google_compute_region_backend_service" "internal_nlb" {
+  name                  = "${var.namespace}-psc-nlb"
+  protocol              = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  backend {
+    group          = google_compute_region_network_endpoint_group.external_lb.id
+    balancing_mode = ""
+  }
+}
+
+resource "google_compute_region_target_tcp_proxy" "internal_nlb" {
+  name            = "${var.namespace}-psc-nlb"
+  backend_service = google_compute_region_backend_service.internal_nlb.id
+}
+
+resource "google_compute_forwarding_rule" "internal_nlb" {
+  name                  = "${var.namespace}-psc-nlb"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+
+  allow_global_access = true
+  ip_protocol         = "TCP"
+  port_range          = "443"
+
+  target = google_compute_region_target_tcp_proxy.internal_nlb.id
+
+  network    = var.network.id
+  subnetwork = var.subnetwork.self_link
+}
+
 resource "google_compute_service_attachment" "default" {
   name                  = "${var.namespace}-private-link"
-  enable_proxy_protocol = false
   connection_preference = "ACCEPT_MANUAL"
+  enable_proxy_protocol = false
   nat_subnets           = [google_compute_subnetwork.default.id]
-  target_service        = "https://www.googleapis.com/compute/v1/projects/${data.google_client_config.current.project}/regions/${data.google_client_config.current.region}/forwardingRules/${var.forwarding_rule}"
+  target_service        = google_compute_forwarding_rule.internal_nlb.self_link
 
   dynamic "consumer_accept_lists" {
     for_each = var.allowed_project_names != {} ? var.allowed_project_names : {}
@@ -14,7 +70,9 @@ resource "google_compute_service_attachment" "default" {
       connection_limit  = consumer_accept_lists.value
     }
   }
-  depends_on = [google_compute_subnetwork.default]
+  depends_on = [
+    google_compute_subnetwork.default
+  ]
 }
 
 resource "google_compute_subnetwork" "default" {

modules/private_link/outputs.tf

@@ -1,3 +1,3 @@
-output "private_attachement_id" {
+output "private_attachment_id" {
   value = try(google_compute_service_attachment.default.id, null)
 }

modules/private_link/variables.tf

@@ -3,18 +3,11 @@ variable "namespace" {
   description = "The name prefix for all resources created."
 }
 
-variable "labels" {
-  description = "Labels which will be applied to all applicable resources."
-  type        = map(string)
-  default     = {}
-}
-
 variable "network" {
   description = "Google Compute Engine network to which the cluster is connected."
   type        = object({ id = string })
 }
 
-
 variable "subnetwork" {
   type = object({
     self_link = string
@@ -38,7 +31,7 @@ variable "proxynetwork_cidr" {
   description = "Internal load balancer proxy subnetwork"
 }
 
-variable "forwarding_rule" {
+variable "fqdn" {
   type        = string
-  description = "forwarding rule name used in private service connect as a target"
+  description = "Fully qualified domain name or hostname"
 }

outputs.tf

@@ -92,8 +92,8 @@ output "database_instance_type" {
   value = local.database_machine_type
 }
 
-output "private_attachement_id" {
-  value = var.create_private_link ? module.private_link[0].private_attachement_id : null
+output "private_attachment_id" {
+  value = var.create_private_link ? module.private_link[0].private_attachment_id : null
 }
 
 output "sa_account_email" {

variables.tf

@@ -359,7 +359,7 @@ variable "public_access" {
 variable "allowed_project_names" {
   type = map(number)
   default = {
-    # "project_ID" = 4 
+    # "project_ID" = 10
   }
   description = "A map of allowed projects where each key is a project number and the value is the connection limit."
 }

versions.tf

@@ -9,6 +9,10 @@ terraform {
       source  = "hashicorp/google"
       version = "~> 5.30"
     }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 5.30"
+    }
     kubernetes = {
       source  = "hashicorp/kubernetes"
       version = "~> 2.23"