diff --git a/kubernetes/hive_metastore.tf b/kubernetes/hive_metastore.tf index 9b9411f..a0c884a 100644 --- a/kubernetes/hive_metastore.tf +++ b/kubernetes/hive_metastore.tf @@ -87,7 +87,7 @@ resource "kubernetes_deployment_v1" "hive_metastore" { env { name = "HIVE_WAREHOUSE" - value = "user/hive/warehouse" + value = local.hive_metastore.warehouse } env { @@ -102,16 +102,16 @@ resource "kubernetes_deployment_v1" "hive_metastore" { env { name = "DATABASE_DB" - value = "metastore_db" + value = local.hive_metastore.mysql.database } env { name = "DATABASE_USER" - value = "admin" + value = local.hive_metastore.mysql.user } env { name = "DATABASE_PASSWORD" - value = "admin" + value = local.hive_metastore.mysql.password } port { @@ -166,7 +166,7 @@ resource "kubernetes_stateful_set_v1" "hive_metastore_mysql" { container { name = "hive-metastore-mysql" - image = "mariadb:latest" + image = "${local.hive_metastore.mysql.image.name}:${local.hive_metastore.mysql.image.tag}" port { container_port = 3306 @@ -174,19 +174,19 @@ resource "kubernetes_stateful_set_v1" "hive_metastore_mysql" { env { name = "MYSQL_ROOT_PASSWORD" - value = "admin" + value = local.hive_metastore.mysql.root_password } env { name = "MYSQL_USER" - value = "admin" + value = local.hive_metastore.mysql.user } env { name = "MYSQL_PASSWORD" - value = "admin" + value = local.hive_metastore.mysql.password } env { name = "MYSQL_DATABASE" - value = "metastore_db" + value = local.hive_metastore.mysql.database } volume_mount { diff --git a/kubernetes/hue.tf b/kubernetes/hue.tf index 3baafbc..feb5966 100644 --- a/kubernetes/hue.tf +++ b/kubernetes/hue.tf @@ -138,7 +138,7 @@ resource "helm_release" "hue" { webhdfs_url=http://${kubernetes_service_v1.namenode.metadata.0.name}-0.${kubernetes_service_v1.namenode.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.namenode_ui.spec.0.port.0.target_port}/webhdfs/v1 [spark] sql_server_host=${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local - sql_server_port=10000 + sql_server_port=${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port} database: create: false name: "${local.hue.postgres.database}" @@ -169,7 +169,7 @@ resource "helm_release" "hue" { set { name = "image.tag" - value = "4.11.0" + value = local.hue.image.tag } set { @@ -184,7 +184,7 @@ resource "helm_release" "hue" { set { name = "hue.replicas" - value = "1" + value = local.hue.replicas } set_sensitive { diff --git a/kubernetes/jupyter.tf b/kubernetes/jupyter.tf index 139416e..8a6b8c6 100644 --- a/kubernetes/jupyter.tf +++ b/kubernetes/jupyter.tf @@ -91,7 +91,7 @@ resource "kubernetes_stateful_set_v1" "jupyter" { env { name = "HIVE_WAREHOUSE" - value = "user/hive/warehouse" + value = local.hive_metastore.warehouse } port { diff --git a/kubernetes/main.tf b/kubernetes/main.tf index 16878d1..c5e029a 100644 --- a/kubernetes/main.tf +++ b/kubernetes/main.tf @@ -65,6 +65,12 @@ locals { } } + spark = { + worker = { + replicas = 4 + } + } + jupyter = { image = { name = "gcr.io/${var.project}/jupyter" @@ -73,10 +79,21 @@ locals { } hive_metastore = { + warehouse = "user/hive/warehouse" image = { name = "gcr.io/${var.project}/hive-metastore" tag = "1.0" } + mysql = { + image = { + name = "mariadb" + tag = "10.11" + } + root_password = var.hive_metastore_mysql_password + user = "admin" + password = var.hive_metastore_mysql_password + database = "metastore_db" + } } drill = { @@ -95,13 +112,20 @@ locals { mongodb_driver_version = "4.4.2" } + trino = { + worker = { + replicas = 2 + } + } + hue = { + replicas = 1 image = { name = "gcr.io/${var.project}/hue" tag = "4.11.0" } postgres = { - version = "9.5" + version = "14.7" hostname = "postgres-hue" user = "hue" password = "${var.hue_postgres_password}" diff --git a/kubernetes/outputs.tf b/kubernetes/outputs.tf index fa78f65..507a9d7 100644 --- a/kubernetes/outputs.tf +++ b/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "spark_thrift_server" { - value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:10000" + value = "${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}" } output "hive_metastore" { diff --git a/kubernetes/spark.tf b/kubernetes/spark.tf index dd7719f..f7dd36e 100644 --- a/kubernetes/spark.tf +++ b/kubernetes/spark.tf @@ -151,7 +151,7 @@ resource "kubernetes_stateful_set_v1" "spark_master" { env { name = "HIVE_WAREHOUSE" - value = "user/hive/warehouse" + value = local.hive_metastore.warehouse } port { @@ -184,7 +184,7 @@ resource "kubernetes_deployment_v1" "spark_worker" { } spec { - replicas = 4 + replicas = local.spark.worker.replicas selector { match_labels = { @@ -334,7 +334,7 @@ resource "kubernetes_deployment_v1" "spark_thrift" { env { name = "HIVE_WAREHOUSE" - value = "user/hive/warehouse" + value = local.hive_metastore.warehouse } port { diff --git a/kubernetes/superset.tf b/kubernetes/superset.tf index eda78bb..a1add3a 100644 --- a/kubernetes/superset.tf +++ b/kubernetes/superset.tf @@ -27,7 +27,7 @@ resource "helm_release" "superset" { allow_dml: false - allow_file_upload: false database_name: "Apache Spark SQL" - sqlalchemy_uri: "hive://hadoop@${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:10000" + sqlalchemy_uri: "hive://hadoop@${kubernetes_service_v1.spark_thrift.metadata.0.name}.${kubernetes_namespace.hadoop.metadata.0.name}.svc.cluster.local:${kubernetes_service_v1.spark_thrift.spec.0.port.0.target_port}" expose_in_sqllab: true allow_dml: false service: diff --git a/kubernetes/trino.tf b/kubernetes/trino.tf index 03ea584..da4f025 100644 --- a/kubernetes/trino.tf +++ b/kubernetes/trino.tf @@ -31,7 +31,7 @@ resource "helm_release" "trino" { set { name = "server.workers" - value = 2 + value = local.trino.worker.replicas } cleanup_on_fail = true diff --git a/kubernetes/variables.tf b/kubernetes/variables.tf index 0c86e92..6d8feec 100644 --- a/kubernetes/variables.tf +++ b/kubernetes/variables.tf @@ -51,3 +51,8 @@ variable "hue_postgres_password" { description = "Hue Postgres password" } +variable "hive_metastore_mysql_password" { + type = string + sensitive = true + description = "Hive Metastore Mysql Password" +}