add service account and workload identity for GKE

gke/gke.tf

@@ -41,7 +41,7 @@ resource "google_container_cluster" "cluster" {
   }
 
   workload_identity_config {
-    workload_pool = null # Disable workload identity
+    workload_pool = "${var.project}.svc.id.goog"
   }
 
   network_policy {

kubernetes/drill.tf

@@ -1,13 +1,7 @@
-resource "kubernetes_namespace" "drill" {
-  metadata {
-    name = "drill"
-  }
-}
-
 resource "kubernetes_service_v1" "drill_service" {
   metadata {
     name      = "drill-service"
-    namespace = kubernetes_namespace.drill.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -42,7 +36,7 @@ resource "kubernetes_service_v1" "drill_service" {
 resource "kubernetes_service_v1" "drills" {
   metadata {
     name      = "drills"
-    namespace = kubernetes_namespace.drill.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -62,7 +56,7 @@ resource "kubernetes_service_v1" "drills" {
 resource "kubernetes_stateful_set_v1" "drill" {
   metadata {
     name      = local.drill.name
-    namespace = kubernetes_namespace.drill.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -83,6 +77,8 @@ resource "kubernetes_stateful_set_v1" "drill" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         container {
           name  = local.drill.name
           image = "${local.drill.image.name}:${local.drill.image.tag}"
@@ -123,8 +119,8 @@ resource "kubernetes_stateful_set_v1" "drill" {
 				echo "clientPortAddress=0.0.0.0" >> $ZOO_HOME/conf/zoo.cfg
 
 				for ((i=0;i<$REPLICAS;i++)); do
-				    echo "server.$i=${local.drill.name}-$i.${kubernetes_service_v1.drills.metadata.0.name}.${kubernetes_namespace.drill.metadata.0.name}.svc.cluster.local:2888:3888" >> $ZOO_HOME/conf/zoo.cfg
-				    NODES+="${local.drill.name}-$i.${kubernetes_service_v1.drills.metadata.0.name}.${kubernetes_namespace.drill.metadata.0.name}.svc.cluster.local:${local.drill.zookeeper.port},"
+				    echo "server.$i=${local.drill.name}-$i.${kubernetes_service_v1.drills.metadata.0.name}.${kubernetes_service_v1.drills.metadata.0.namespace}.svc.cluster.local:2888:3888" >> $ZOO_HOME/conf/zoo.cfg
+				    NODES+="${local.drill.name}-$i.${kubernetes_service_v1.drills.metadata.0.name}.${kubernetes_service_v1.drills.metadata.0.namespace}.svc.cluster.local:${local.drill.zookeeper.port},"
 				done
 
 				POD_NAME=$(hostname)
@@ -140,7 +136,7 @@ resource "kubernetes_stateful_set_v1" "drill" {
 					"storage": {
 					  dfs: {
 					    type : "file",
-					    connection : "hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}",
+					    connection : "hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}",
 					    workspaces : {
 					      "tmp" : {
 					        "location" : "/tmp",
@@ -309,9 +305,9 @@ resource "kubernetes_stateful_set_v1" "drill" {
 					  hive: {
 					    type: "hive",
 					    configProps: {
-					      "hive.metastore.uris": "thrift://${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_namespace.hive_metastore.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.hive_metastore.spec.0.port.0.target_port}",
+					      "hive.metastore.uris": "thrift://${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_service_v1.hive_metastore.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.hive_metastore.spec.0.port.0.target_port}",
 					      "hive.metastore.sasl.enabled": "false",
-					      "fs.default.name": "hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}/"
+					      "fs.default.name": "hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}/"
 					    },
 					    enabled: true
 					  }

kubernetes/hadoop.tf

@@ -134,7 +134,7 @@ resource "kubernetes_stateful_set_v1" "namenode" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           port {
@@ -231,7 +231,7 @@ resource "kubernetes_stateful_set_v1" "datanode" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           volume_mount {

kubernetes/hive_metastore.tf

@@ -1,14 +1,7 @@
-resource "kubernetes_namespace" "hive_metastore" {
-
-  metadata {
-    name = "hive-metastore"
-  }
-}
-
 resource "kubernetes_service_v1" "hive_metastore" {
   metadata {
     name      = "hive-metastore"
-    namespace = kubernetes_namespace.hive_metastore.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -32,7 +25,7 @@ resource "kubernetes_service_v1" "hive_metastore" {
 resource "kubernetes_service_v1" "hive_metastore_mysql" {
   metadata {
     name      = "hive-metastore-mysql"
-    namespace = kubernetes_namespace.hive_metastore.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -56,7 +49,7 @@ resource "kubernetes_service_v1" "hive_metastore_mysql" {
 resource "kubernetes_deployment_v1" "hive_metastore" {
   metadata {
     name      = "hive-metastore"
-    namespace = kubernetes_namespace.hive_metastore.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {
@@ -76,13 +69,15 @@ resource "kubernetes_deployment_v1" "hive_metastore" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         container {
           name  = "hive-metastore"
           image = "${local.hive_metastore.image.name}:${local.hive_metastore.image.tag}"
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
@@ -126,7 +121,7 @@ resource "kubernetes_deployment_v1" "hive_metastore" {
 resource "kubernetes_stateful_set_v1" "hive_metastore_mysql" {
   metadata {
     name      = "hive-metastore-mysql"
-    namespace = kubernetes_namespace.hive_metastore.metadata.0.name
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
   }
 
   spec {

kubernetes/hue.tf

@@ -115,7 +115,7 @@ resource "helm_release" "hue" {
   repository       = "https://helm.gethue.com"
   chart            = "hue"
   namespace        = kubernetes_namespace.hadoop.metadata.0.name
-  create_namespace = true
+  create_namespace = false
 
   values = [
     data.http.hue_values_yaml.response_body,
@@ -134,10 +134,10 @@ resource "helm_release" "hue" {
 		    [hadoop]
 		    [[hdfs_clusters]]
 		    [[[default]]]
-		    fs_defaultfs=hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}
-		    webhdfs_url=http://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode_ui.spec.0.port.0.target_port}/webhdfs/v1
+		    fs_defaultfs=hdfs://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}
+		    webhdfs_url=http://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.namenode_ui.spec.0.port.0.target_port}/webhdfs/v1
 		    [spark]
-		    sql_server_host=${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local
+		    sql_server_host=${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_service_v1.spark_thrift.metadata.0.namespace}.svc.cluster.local
 		    sql_server_port=${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}
 		  database:
 		    create: false

kubernetes/jupyter.tf

@@ -49,6 +49,8 @@ resource "kubernetes_stateful_set_v1" "jupyter" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         init_container {
           name  = "change-volume-owner"
           image = "busybox:latest"
@@ -76,7 +78,7 @@ resource "kubernetes_stateful_set_v1" "jupyter" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
@@ -86,7 +88,7 @@ resource "kubernetes_stateful_set_v1" "jupyter" {
 
           env {
             name  = "HIVE_METASTORE_HOSTNAME"
-            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_namespace.hive_metastore.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_service_v1.hive_metastore.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {

kubernetes/outputs.tf

@@ -1,19 +1,19 @@
 output "spark_thrift_server" {
-  value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}"
+  value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_service_v1.spark_thrift.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}"
 }
 
 output "hive_metastore" {
-  value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_namespace.hive_metastore.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.hive_metastore.spec.0.port.0.target_port}"
+  value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_service_v1.hive_metastore.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.hive_metastore.spec.0.port.0.target_port}"
 }
 
 output "namenode" {
-  value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}"
+  value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.namenode.spec.0.port.0.target_port}"
 }
 
 output "trino" {
   value = "trino.${helm_release.trino.namespace}.svc.cluster.local:8080"
 }
 
 output "drill" {
-  value = "${kubernetes_service_v1.drill_service.metadata.0.name}.drill.svc.cluster.local:${kubernetes_service_v1.drill_service.spec.0.port.0.target_port}"
+  value = "${kubernetes_service_v1.drill_service.metadata.0.name}.${kubernetes_service_v1.drill_service.metadata.0.namespace}.svc.cluster.local:${kubernetes_service_v1.drill_service.spec.0.port.0.target_port}"
 }

kubernetes/service_account.tf

@@ -0,0 +1,29 @@
+# Service Account
+resource "google_service_account" "storage_admin" {
+  account_id   = "storage-admin"
+  display_name = "Storage Admin"
+}
+
+resource "google_project_iam_member" "storage_admin" {
+  project = var.project
+  role    = "roles/storage.admin"
+  member  = "serviceAccount:${google_service_account.storage_admin.email}"
+}
+
+resource "google_service_account_iam_binding" "iam_binding" {
+  service_account_id = google_service_account.storage_admin.name
+  role               = "roles/iam.workloadIdentityUser"
+  members = [
+    "serviceAccount:${var.project}.svc.id.goog[${kubernetes_service_account.storage_admin.metadata.0.namespace}/${kubernetes_service_account.storage_admin.metadata.0.name}]",
+  ]
+}
+
+resource "kubernetes_service_account" "storage_admin" {
+  metadata {
+    name      = "storage-admin"
+    namespace = kubernetes_namespace.hadoop.metadata.0.name
+    annotations = {
+      "iam.gke.io/gcp-service-account" = google_service_account.storage_admin.email
+    }
+  }
+}

kubernetes/spark.tf

@@ -125,6 +125,8 @@ resource "kubernetes_stateful_set_v1" "spark_master" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         container {
           name  = kubernetes_service_v1.spark_master.metadata.0.name
           image = "${local.spark.image.name}:${local.spark.image.tag}"
@@ -136,17 +138,17 @@ resource "kubernetes_stateful_set_v1" "spark_master" {
 
           env {
             name  = "SPARK_MASTER_HOSTNAME"
-            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_service_v1.spark_master.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "HIVE_METASTORE_HOSTNAME"
-            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_namespace.hive_metastore.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_service_v1.hive_metastore.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
@@ -200,6 +202,8 @@ resource "kubernetes_deployment_v1" "spark_worker" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         container {
           name  = "spark-worker"
           image = "${local.spark.image.name}:${local.spark.image.tag}"
@@ -218,12 +222,12 @@ resource "kubernetes_deployment_v1" "spark_worker" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "SPARK_MASTER_HOSTNAME"
-            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_service_v1.spark_master.metadata.0.namespace}.svc.cluster.local"
           }
         }
       }
@@ -265,12 +269,12 @@ resource "kubernetes_deployment_v1" "spark_history" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "SPARK_MASTER_HOSTNAME"
-            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_service_v1.spark_master.metadata.0.namespace}.svc.cluster.local"
           }
 
           port {
@@ -315,6 +319,8 @@ resource "kubernetes_deployment_v1" "spark_thrift" {
       }
 
       spec {
+        service_account_name = kubernetes_service_account.storage_admin.metadata.0.name
+
         container {
           name  = "spark-thrift"
           image = "${local.spark.image.name}:${local.spark.image.tag}"
@@ -326,17 +332,17 @@ resource "kubernetes_deployment_v1" "spark_thrift" {
 
           env {
             name  = "NAMENODE_HOSTNAME"
-            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_service_v1.namenode.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "SPARK_MASTER_HOSTNAME"
-            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.spark_master.metadata.0.name}-0.${kubernetes_service_v1.spark_master.metadata.0.name}.${kubernetes_service_v1.spark_master.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {
             name  = "HIVE_METASTORE_HOSTNAME"
-            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_namespace.hive_metastore.metadata.0.name}.svc.cluster.local"
+            value = "${kubernetes_service_v1.hive_metastore.metadata.0.name}.${kubernetes_service_v1.hive_metastore.metadata.0.namespace}.svc.cluster.local"
           }
 
           env {