chore: update bootstrap (#23)

* chore: update bootstrap

* chore: update bootstrap

bootstrap/instance/dbsync.tf

@@ -117,7 +117,7 @@ resource "kubernetes_deployment_v1" "db_sync" {
             value_from {
               secret_key_ref {
                 key  = "password"
-                name = "${var.postgres_secret_name}"
+                name = var.postgres_secret_name
               }
             }
           }
@@ -143,7 +143,7 @@ resource "kubernetes_deployment_v1" "db_sync" {
           }
 
           env {
-            name = "NETWORK"
+            name  = "NETWORK"
             value = ""
           }
 

bootstrap/pgbouncer/main.tf

@@ -1,4 +1,4 @@
-variable namespace {
+variable "namespace" {
   type = string
 }
 
@@ -14,7 +14,7 @@ resource "kubernetes_service_v1" "dbsync_pgbouncer_elb" {
   }
 
   spec {
-    type = "LoadBalancer"
+    type                = "LoadBalancer"
     load_balancer_class = "service.k8s.aws/nlb"
 
     port {

bootstrap/postgres/main.tf

@@ -47,16 +47,16 @@ variable "postgres_secret_name" {
 variable "postgres_settings" {
   default = {
     listen_addresses                 = "*"
-    max_connections                  = 1000
-    shared_buffers                   = "8GB"
-    effective_cache_size             = "24GB"
+    max_connections                  = 101
+    shared_buffers                   = "12GB"
+    effective_cache_size             = "36GB"
     maintenance_work_mem             = "2GB"
     checkpoint_completion_target     = 0.9
     wal_buffers                      = "16MB"
     default_statistics_target        = 500
     random_page_cost                 = 1.1
     effective_io_concurrency         = 200
-    work_mem                         = "1048kB"
+    work_mem                         = "15728kB"
     huge_pages                       = "try"
     min_wal_size                     = "4GB"
     max_wal_size                     = "16GB"
@@ -65,6 +65,8 @@ variable "postgres_settings" {
     max_parallel_workers             = 8
     max_parallel_maintenance_workers = 4
     ssl                              = "off"
+    shared_preload_libraries         = "pg_stat_statements"
+    max_pred_locks_per_transaction   = 256
   }
 }
 
@@ -78,4 +80,17 @@ variable "dbsync_probe_image_tag" {
 
 variable "pg_bouncer_replicas" {
   default = 1
-}
+}
+
+variable "pg_bouncer_user_settings" {
+  default = []
+  type = list(object({
+    name            = string
+    password        = string
+    max_connections = number
+  }))
+}
+
+variable "pg_bouncer_auth_user_password" {
+  type = string
+}

bootstrap/postgres/monitor.tf

@@ -25,3 +25,31 @@ resource "kubernetes_manifest" "postgres_podmonitor" {
     }
   }
 }
+
+resource "kubernetes_manifest" "pgbouncer_podmonitor" {
+  manifest = {
+    "apiVersion" = "monitoring.coreos.com/v1"
+    "kind"       = "PodMonitor"
+    "metadata" = {
+      "labels" = {
+        "app.kubernetes.io/component" = "o11y"
+        "app.kubernetes.io/part-of"   = "demeter"
+      }
+      "name"      = "${var.instance_name}-pgbouncer"
+      "namespace" = var.namespace
+    }
+    "spec" = {
+      podMetricsEndpoints = [
+        {
+          port = "metrics",
+          path = "/metrics"
+        }
+      ]
+      "selector" = {
+        "matchLabels" = {
+          "demeter.run/instance" = "${var.instance_name}-pgbouncer"
+        }
+      }
+    }
+  }
+}

bootstrap/postgres/pg-bouncer.tf

@@ -2,7 +2,8 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
   wait_for_rollout = false
   metadata {
     labels = {
-      role = "pgbouncer"
+      role                   = "pgbouncer"
+      "demeter.run/instance" = "${var.instance_name}-pgbouncer"
     }
     name      = "${var.instance_name}-pgbouncer"
     namespace = var.namespace
@@ -20,14 +21,16 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
 
     selector {
       match_labels = {
-        role = "pgbouncer"
+        role                   = "pgbouncer"
+        "demeter.run/instance" = "${var.instance_name}-pgbouncer"
       }
     }
 
     template {
       metadata {
         labels = {
-          role = "pgbouncer"
+          role                   = "pgbouncer"
+          "demeter.run/instance" = "${var.instance_name}-pgbouncer"
         }
       }
 
@@ -46,12 +49,6 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
             }
           }
 
-          port {
-            container_port = 9930
-            name           = "metrics"
-            protocol       = "TCP"
-          }
-
           port {
             container_port = 6432
             name           = "pgbouncer"
@@ -89,50 +86,24 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
           }
 
           env {
-            name  = "PGBOUNCER_DSN_0"
-            value = "mainnet=host=${var.instance_name} port=5432 dbname=dbsync-mainnet auth_user=pgbouncer"
-          }
-
-          env {
-            name  = "PGBOUNCER_DSN_1"
-            value = "preview=host=${var.instance_name} port=5432 dbname=dbsync-preview auth_user=pgbouncer"
-          }
-
-          env {
-            name  = "PGBOUNCER_DSN_2"
-            value = "preprod=host=${var.instance_name} port=5432 dbname=dbsync-preprod auth_user=pgbouncer"
-          }
-
-          env {
-            name = "PGBOUNCER_AUTH_USER"
-            value = "pgbouncer"
-          }
-
-          env {
-            name  = "PGBOUNCER_AUTH_QUERY"
-            value = "SELECT usename, passwd FROM user_search($1)"
-          }
-
-          env {
-            name  = "PGBOUNCER_IGNORE_STARTUP_PARAMETERS"
-            value = "ignore_startup_parameters = extra_float_digits"
-          }
-
-
-          env {
-            name = "PGBOUNCER_USERLIST_FILE"
+            name  = "PGBOUNCER_USERLIST_FILE"
             value = "/etc/pgbouncer/users.txt"
           }
 
           volume_mount {
-            name       = "pgbouncer-config"
+            name       = "pgbouncer-users"
             mount_path = "/etc/pgbouncer"
           }
 
+          volume_mount {
+            name       = "pgbouncer-ini"
+            mount_path = "/bitnami/pgbouncer/conf"
+          }
+
         }
 
         container {
-          name = "readiness"
+          name  = "readiness"
           image = "ghcr.io/demeter-run/cardano-dbsync-probe:${var.dbsync_probe_image_tag}"
           env {
             name  = "PGHOST"
@@ -153,7 +124,7 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
             name = "PGPASSWORD"
             value_from {
               secret_key_ref {
-                name = "${var.postgres_secret_name}"
+                name = var.postgres_secret_name
                 key  = "password"
               }
             }
@@ -166,10 +137,31 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
           }
         }
 
+        container {
+          name  = "exporter"
+          image = "prometheuscommunity/pgbouncer-exporter:v0.7.0"
+          port {
+            container_port = 9127
+            name           = "metrics"
+            protocol       = "TCP"
+          }
+          args = [
+            "--pgBouncer.connectionString=postgres://pgbouncer:${var.pg_bouncer_auth_user_password}@localhost:6432/pgbouncer?sslmode=disable",
+          ]
+
+        }
+
         volume {
-          name = "pgbouncer-config"
+          name = "pgbouncer-users"
           config_map {
-            name = "${var.instance_name}-pgbouncer-config"
+            name = "${var.instance_name}-pgbouncer-users"
+          }
+        }
+
+        volume {
+          name = "pgbouncer-ini"
+          config_map {
+            name = "${var.instance_name}-pgbouncer-ini"
           }
         }
 
@@ -197,13 +189,26 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
   }
 }
 
-resource "kubernetes_config_map" "dbsync_pgbouncer_config" {
+
+resource "kubernetes_config_map" "dbsync_pgbouncer_users" {
+  metadata {
+    namespace = var.namespace
+    name      = "${var.instance_name}-pgbouncer-users"
+  }
+
+  data = {
+    "users.txt" = "${templatefile("${path.module}/users.txt.tftpl", { auth_user_password = "${var.pg_bouncer_auth_user_password}", users = var.pg_bouncer_user_settings })}"
+  }
+}
+
+
+resource "kubernetes_config_map" "dbsync_pgbouncer_ini_config" {
   metadata {
     namespace = var.namespace
-    name      = "${var.instance_name}-pgbouncer-config"
+    name      = "${var.instance_name}-pgbouncer-ini"
   }
 
   data = {
-    "users.txt" = "${file("${path.module}/users.txt")}"
+    "pgbouncer.ini" = "${templatefile("${path.module}/pgbouncer.ini.tftpl", { db_host = "${var.instance_name}", users = var.pg_bouncer_user_settings })}"
   }
 }

bootstrap/postgres/pgbouncer.ini.tftpl

@@ -0,0 +1,31 @@
+[databases]
+*=host=${db_host} port=5432 auth_user=pgbouncer
+mainnet=host=${db_host} port=5432 dbname=dbsync-mainnet auth_user=pgbouncer
+preview=host=${db_host} port=5432 dbname=dbsync-preview auth_user=pgbouncer
+preprod=host=${db_host} port=5432 dbname=dbsync-preprod auth_user=pgbouncer
+
+[pgbouncer]
+listen_port=6432
+listen_addr=0.0.0.0
+unix_socket_dir=/tmp/
+unix_socket_mode=0777
+auth_file=/opt/bitnami/pgbouncer/conf/userlist.txt
+auth_type=scram-sha-256
+auth_query=SELECT usename, passwd FROM user_search($1)
+pidfile=/opt/bitnami/pgbouncer/tmp/pgbouncer.pid
+logfile=/opt/bitnami/pgbouncer/logs/pgbouncer.log
+admin_users=postgres
+client_tls_sslmode=disable
+server_tls_sslmode=disable
+ignore_startup_parameters=ignore_startup_parameters = extra_float_digits
+stats_period=60
+max_user_connections=5
+tcp_keepcnt=9
+tcp_keepidle=300
+tcp_keepintvl=75
+admin_users=pgbouncer
+[users]
+%{ for user in users ~}
+${user.name} = max_user_connections=${user.max_connections}
+%{ endfor ~}
+

bootstrap/postgres/postgres.tf

@@ -56,7 +56,7 @@ resource "kubernetes_stateful_set_v1" "postgres" {
             name = "POSTGRES_PASSWORD"
             value_from {
               secret_key_ref {
-                name = "${var.postgres_secret_name}"
+                name = var.postgres_secret_name
                 key  = "password"
               }
             }
@@ -88,6 +88,11 @@ resource "kubernetes_stateful_set_v1" "postgres" {
             name       = "config"
             sub_path   = "postgresql.conf"
           }
+
+          volume_mount {
+            mount_path = "/dev/shm"
+            name       = "dshm"
+          }
         }
 
         container {
@@ -105,7 +110,7 @@ resource "kubernetes_stateful_set_v1" "postgres" {
             name = "DATA_SOURCE_PASS"
             value_from {
               secret_key_ref {
-                name = "${var.postgres_secret_name}"
+                name = var.postgres_secret_name
                 key  = "password"
               }
             }
@@ -135,6 +140,14 @@ resource "kubernetes_stateful_set_v1" "postgres" {
           }
         }
 
+        volume {
+          name = "dshm"
+          empty_dir {
+            medium     = "Memory"
+            size_limit = "1Gi"
+          }
+        }
+
         toleration {
           effect   = "NoSchedule"
           key      = "demeter.run/compute-profile"

bootstrap/postgres/postgresql.conf

@@ -1,5 +1,5 @@
 listen_addresses = '*'
-max_connections = 500
+max_connections = 100
 shared_buffers = 12GB
 effective_cache_size = 36GB
 maintenance_work_mem = 2GB
@@ -8,7 +8,7 @@ wal_buffers = 16MB
 default_statistics_target = 500
 random_page_cost = 1.1
 effective_io_concurrency = 200
-work_mem = 3145kB
+work_mem = 15728kB
 huge_pages = try
 min_wal_size = 4GB
 max_wal_size = 16GB

bootstrap/postgres/queries.sql

@@ -1,6 +1,6 @@
 -- create pg bouncer role and user_search function - add to each database
 
-CREATE ROLE pgbouncer WITH PASSWORD 'pgbouncertest';
+CREATE ROLE pgbouncer WITH PASSWORD 'pgbounceruserpass';
 
 CREATE OR REPLACE FUNCTION user_search(uname TEXT) RETURNS TABLE (usename name, passwd text) as
 $$

bootstrap/postgres/users.txt.tftpl

@@ -0,0 +1,4 @@
+"pgbouncer" "${auth_user_password}"
+%{ for user in users ~}
+"${user.name}" "${user.password}"
+%{ endfor ~}