diff --git a/terraform/modules/account_mapping/account_mapping.tf b/terraform/modules/account_mapping/account_mapping.tf index 8ef8d3b03..0b5a3a163 100644 --- a/terraform/modules/account_mapping/account_mapping.tf +++ b/terraform/modules/account_mapping/account_mapping.tf @@ -14,9 +14,7 @@ variable "environment" { type = string } -variable "gcp_project" { - type = string -} +data "google_project" "current" {} resource "random_string" "account_id" { length = 16 @@ -63,7 +61,7 @@ resource "kubernetes_service_account" "account" { # account in GCP-level policies, below. See step 5 in # https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to locals { - service_account = "serviceAccount:${var.gcp_project}.svc.id.goog[${var.kubernetes_namespace}/${kubernetes_service_account.account.metadata[0].name}]" + service_account = "serviceAccount:${data.google_project.current.name}.svc.id.goog[${var.kubernetes_namespace}/${kubernetes_service_account.account.metadata[0].name}]" } # Allows the Kubernetes service account to impersonate the GCP service account. diff --git a/terraform/modules/kubernetes/kubernetes.tf b/terraform/modules/kubernetes/kubernetes.tf index 808717588..8e4eec607 100644 --- a/terraform/modules/kubernetes/kubernetes.tf +++ b/terraform/modules/kubernetes/kubernetes.tf @@ -146,7 +146,6 @@ module "account_mapping" { kubernetes_account_name = "${var.data_share_processor_name}-workflow-manager" kubernetes_namespace = var.kubernetes_namespace environment = var.environment - gcp_project = var.gcp_project } # Allows the Kubernetes service account to request auth tokens for the GCP diff --git a/terraform/modules/locality_kubernetes/kubernetes.tf b/terraform/modules/locality_kubernetes/kubernetes.tf index 52d184870..5514c6764 100644 --- a/terraform/modules/locality_kubernetes/kubernetes.tf +++ b/terraform/modules/locality_kubernetes/kubernetes.tf @@ -40,7 +40,6 @@ module "account_mapping" { kubernetes_account_name = "manifest-updater" kubernetes_namespace = var.kubernetes_namespace environment = var.environment - gcp_project = var.gcp_project } # Create a new manifest_updater role that is authorized to work with k8s secrets diff --git a/terraform/modules/monitoring/monitoring.tf b/terraform/modules/monitoring/monitoring.tf index e21340334..43d9619fb 100644 --- a/terraform/modules/monitoring/monitoring.tf +++ b/terraform/modules/monitoring/monitoring.tf @@ -133,6 +133,8 @@ resource "kubernetes_persistent_volume_claim" "prometheus_server" { } } +# Metrics are stored in Prometheus. Documentation and config reference: +# https://artifacthub.io/packages/helm/prometheus-community/prometheus resource "helm_release" "prometheus" { name = "prometheus" chart = "prometheus" @@ -196,6 +198,8 @@ DATASOURCE } } +# We use Grafana as a frontend to Prometheus for dashboards. Documentation and +# config reference: https://artifacthub.io/packages/helm/grafana/grafana resource "helm_release" "grafana" { name = "grafana" chart = "grafana" @@ -212,3 +216,73 @@ resource "helm_release" "grafana" { value = "grafana-datasources" } } + +data "google_project" "current" {} + +# The Stackdriver Prometheus exporter forwards metrics from Google Cloud +# Monitoring (aka Stackdriver) into Prometheus. Documentation and config +# reference: +# https://github.com/prometheus-community/stackdriver_exporter +# https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter +# https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-stackdriver-exporter +resource "helm_release" "stackdriver_exporter" { + name = "stackdriver-exporter" + chart = "prometheus-stackdriver-exporter" + repository = "https://prometheus-community.github.io/helm-charts" + namespace = kubernetes_namespace.monitoring.metadata[0].name + + set { + name = "stackdriver.projectId" + value = data.google_project.current.name + } + + set { + name = "serviceAccount.name" + value = module.account_mapping.kubernetes_account_name + } + + set { + name = "stackdriver.metrics.typePrefixes" + # The metrics to be forwarded from Stackdriver to Prometheus. The prefix + # list is comma separated, and we have to escape the comma to avoid + # confusing Helm: + # https://github.com/hashicorp/terraform-provider-helm/issues/226 + value = join("\\,", [ + # PubSub metrics + # https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub + "pubsub.googleapis.com/subscription/", + "pubsub.googleapis.com/topic/", + ]) + } + + # We must annotate the stackdriver exporter Kubernetes service to be + # discovered by Prometheus as a scrape target. It is surprisingly hard to get + # this represented just right so that the transformation from Terraform value + # to Helm argument to Kubernetes annotation comes out right, hence the YAML + # literal heredoc. + # https://github.com/hashicorp/terraform-provider-helm/issues/64 + values = [ + <