terraform: expose k8s services for metrics

In order for Prometheus to scrape metrics from the `intake-batch` and
`aggregate` workers, we must create a Kubernetes _service_ for each
_deployment_ which has _annotations_ on it marking it as a scrape target.

terraform/modules/kubernetes/kubernetes.tf

@@ -322,23 +322,48 @@ resource "kubernetes_cron_job" "workflow_manager" {
   }
 }
 
-resource "kubernetes_deployment" "intake_batch" {
+resource "kubernetes_service" "intake_batch" {
   metadata {
     name      = "intake-batch-${var.ingestor}"
     namespace = var.kubernetes_namespace
+    annotations = {
+      # Needed for discovery by Prometheus
+      "prometheus.io/scrape" = "true"
+    }
+  }
+  spec {
+    port {
+      name     = "metrics"
+      port     = 8080
+      protocol = "TCP"
+    }
+    type = "ClusterIP"
+    # Selector must match the label(s) on kubernetes_deployment.intake_batch
+    selector = {
+      app      = "intake-batch-worker"
+      ingestor = var.ingestor
+    }
   }
+}
 
+resource "kubernetes_deployment" "intake_batch" {
+  metadata {
+    name      = "intake-batch-${var.ingestor}"
+    namespace = var.kubernetes_namespace
+  }
   spec {
     replicas = var.intake_worker_count
     selector {
       match_labels = {
-        name = "intake-batch-${var.ingestor}"
+        app      = "intake-batch-worker"
+        ingestor = var.ingestor
       }
     }
     template {
       metadata {
         labels = {
-          name = "intake-batch-${var.ingestor}"
+          app      = "intake-batch-worker"
+          ingestor = var.ingestor
         }
       }
       spec {
@@ -347,6 +372,11 @@ resource "kubernetes_deployment" "intake_batch" {
           name  = "facile-container"
           image = "${var.container_registry}/${var.facilitator_image}:${var.facilitator_version}"
           args  = ["intake-batch-worker"]
+          # Prometheus metrics scrape endpoint
+          port {
+            container_port = 8080
+            protocol       = "TCP"
+          }
           resources {
             # Batch intake is single threaded, and we never expect to see
             # batches larger than 3-400 MB, so set the limits such that we can
@@ -395,6 +425,30 @@ resource "kubernetes_deployment" "intake_batch" {
   }
 }
 
+resource "kubernetes_service" "aggregate" {
+  metadata {
+    name      = "aggregate-${var.ingestor}"
+    namespace = var.kubernetes_namespace
+    annotations = {
+      # Needed for discovery by Prometheus
+      "prometheus.io/scrape" = "true"
+    }
+  }
+  spec {
+    port {
+      name     = "metrics"
+      port     = 8080
+      protocol = "TCP"
+    }
+    type = "ClusterIP"
+    # Selector must match the label(s) on kubernetes_deployment.aggregate
+    selector = {
+      app      = "aggregate-worker"
+      ingestor = var.ingestor
+    }
+  }
+}
+
 resource "kubernetes_deployment" "aggregate" {
   metadata {
     name      = "aggregate-${var.ingestor}"
@@ -405,13 +459,15 @@ resource "kubernetes_deployment" "aggregate" {
     replicas = var.aggregate_worker_count
     selector {
       match_labels = {
-        name = "aggregate-${var.ingestor}"
+        app      = "aggregate-worker"
+        ingestor = var.ingestor
       }
     }
     template {
       metadata {
         labels = {
-          name = "aggregate-${var.ingestor}"
+          app      = "aggregate-worker"
+          ingestor = var.ingestor
         }
       }
       spec {
@@ -420,6 +476,11 @@ resource "kubernetes_deployment" "aggregate" {
           name  = "facile-container"
           image = "${var.container_registry}/${var.facilitator_image}:${var.facilitator_version}"
           args  = ["aggregate-worker"]
+          # Prometheus metrics scrape endpoint
+          port {
+            container_port = 8080
+            protocol       = "TCP"
+          }
           resources {
             # As in the intake-batch case, aggregate jobs are single threaded
             # and need to fit whole ingestion batches into memory.