change to local parameters

tanawatpan · May 5, 2023 · 9416c53 · 9416c53
1 parent d6ef517
commit 9416c53
Show file tree

Hide file tree

Showing 9 changed files with 49 additions and 20 deletions.
diff --git a/kubernetes/hive_metastore.tf b/kubernetes/hive_metastore.tf
@@ -87,7 +87,7 @@ resource "kubernetes_deployment_v1" "hive_metastore" {
 
           env {
             name  = "HIVE_WAREHOUSE"
-            value = "user/hive/warehouse"
+            value = local.hive_metastore.warehouse
           }
 
           env {
@@ -102,16 +102,16 @@ resource "kubernetes_deployment_v1" "hive_metastore" {
 
           env {
             name  = "DATABASE_DB"
-            value = "metastore_db"
+            value = local.hive_metastore.mysql.database
           }
 
           env {
             name  = "DATABASE_USER"
-            value = "admin"
+            value = local.hive_metastore.mysql.user
           }
           env {
             name  = "DATABASE_PASSWORD"
-            value = "admin"
+            value = local.hive_metastore.mysql.password
           }
 
           port {
@@ -166,27 +166,27 @@ resource "kubernetes_stateful_set_v1" "hive_metastore_mysql" {
 
         container {
           name  = "hive-metastore-mysql"
-          image = "mariadb:latest"
+          image = "${local.hive_metastore.mysql.image.name}:${local.hive_metastore.mysql.image.tag}"
 
           port {
             container_port = 3306
           }
 
           env {
             name  = "MYSQL_ROOT_PASSWORD"
-            value = "admin"
+            value = local.hive_metastore.mysql.root_password
           }
           env {
             name  = "MYSQL_USER"
-            value = "admin"
+            value = local.hive_metastore.mysql.user
           }
           env {
             name  = "MYSQL_PASSWORD"
-            value = "admin"
+            value = local.hive_metastore.mysql.password
           }
           env {
             name  = "MYSQL_DATABASE"
-            value = "metastore_db"
+            value = local.hive_metastore.mysql.database
           }
 
           volume_mount {

diff --git a/kubernetes/hue.tf b/kubernetes/hue.tf
@@ -138,7 +138,7 @@ resource "helm_release" "hue" {
 		    webhdfs_url=http://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode_ui.spec.0.port.0.target_port}/webhdfs/v1
 		    [spark]
 		    sql_server_host=${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local
-		    sql_server_port=10000
+		    sql_server_port=${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}
 		  database:
 		    create: false
 		    name: "${local.hue.postgres.database}"
@@ -169,7 +169,7 @@ resource "helm_release" "hue" {
 
   set {
     name  = "image.tag"
-    value = "4.11.0"
+    value = local.hue.image.tag
   }
 
   set {
@@ -184,7 +184,7 @@ resource "helm_release" "hue" {
 
   set {
     name  = "hue.replicas"
-    value = "1"
+    value = local.hue.replicas
   }
 
   set_sensitive {

diff --git a/kubernetes/jupyter.tf b/kubernetes/jupyter.tf
@@ -91,7 +91,7 @@ resource "kubernetes_stateful_set_v1" "jupyter" {
 
           env {
             name  = "HIVE_WAREHOUSE"
-            value = "user/hive/warehouse"
+            value = local.hive_metastore.warehouse
           }
 
           port {

diff --git a/kubernetes/main.tf b/kubernetes/main.tf
@@ -65,6 +65,12 @@ locals {
     }
   }
 
+  spark = {
+    worker = {
+      replicas = 4
+    }
+  }
+
   jupyter = {
     image = {
       name = "gcr.io/${var.project}/jupyter"
@@ -73,10 +79,21 @@ locals {
   }
 
   hive_metastore = {
+    warehouse = "user/hive/warehouse"
     image = {
       name = "gcr.io/${var.project}/hive-metastore"
       tag  = "1.0"
     }
+    mysql = {
+      image = {
+        name = "mariadb"
+        tag  = "10.11"
+      }
+      root_password = var.hive_metastore_mysql_password
+      user          = "admin"
+      password      = var.hive_metastore_mysql_password
+      database      = "metastore_db"
+    }
   }
 
   drill = {
@@ -95,13 +112,20 @@ locals {
     mongodb_driver_version = "4.4.2"
   }
 
+  trino = {
+    worker = {
+      replicas = 2
+    }
+  }
+
   hue = {
+    replicas = 1
     image = {
       name = "gcr.io/${var.project}/hue"
       tag  = "4.11.0"
     }
     postgres = {
-      version  = "9.5"
+      version  = "14.7"
       hostname = "postgres-hue"
       user     = "hue"
       password = "${var.hue_postgres_password}"

diff --git a/kubernetes/outputs.tf b/kubernetes/outputs.tf
@@ -1,5 +1,5 @@
 output "spark_thrift_server" {
-  value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:10000"
+  value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}"
 }
 
 output "hive_metastore" {

diff --git a/kubernetes/spark.tf b/kubernetes/spark.tf
@@ -151,7 +151,7 @@ resource "kubernetes_stateful_set_v1" "spark_master" {
 
           env {
             name  = "HIVE_WAREHOUSE"
-            value = "user/hive/warehouse"
+            value = local.hive_metastore.warehouse
           }
 
           port {
@@ -184,7 +184,7 @@ resource "kubernetes_deployment_v1" "spark_worker" {
   }
 
   spec {
-    replicas = 4
+    replicas = local.spark.worker.replicas
 
     selector {
       match_labels = {
@@ -334,7 +334,7 @@ resource "kubernetes_deployment_v1" "spark_thrift" {
 
           env {
             name  = "HIVE_WAREHOUSE"
-            value = "user/hive/warehouse"
+            value = local.hive_metastore.warehouse
           }
 
           port {

diff --git a/kubernetes/superset.tf b/kubernetes/superset.tf
@@ -27,7 +27,7 @@ resource "helm_release" "superset" {
 		      allow_dml: false
 		    - allow_file_upload: false
 		      database_name: "Apache Spark SQL"
-		      sqlalchemy_uri: "hive://hadoop@${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:10000"
+		      sqlalchemy_uri: "hive://hadoop@${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}"
 		      expose_in_sqllab: true
 		      allow_dml: false
 		service:

diff --git a/kubernetes/trino.tf b/kubernetes/trino.tf
@@ -31,7 +31,7 @@ resource "helm_release" "trino" {
 
   set {
     name  = "server.workers"
-    value = 2
+    value = local.trino.worker.replicas
   }
 
   cleanup_on_fail = true

diff --git a/kubernetes/variables.tf b/kubernetes/variables.tf
@@ -51,3 +51,8 @@ variable "hue_postgres_password" {
   description = "Hue Postgres password"
 }
 
+variable "hive_metastore_mysql_password" {
+  type        = string
+  sensitive   = true
+  description = "Hive Metastore Mysql Password"
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -91,7 +91,7 @@ resource "kubernetes_stateful_set_v1" "jupyter" { @@
               env {
                 name  = "HIVE_WAREHOUSE"
-                value = "user/hive/warehouse"
+                value = local.hive_metastore.warehouse
               }
               port {
@@ Expand Down @@