Set configmap for tiler-db's postgresql.config (#337)

* Set configmap for postgresql.config tiler-db

* Remove image values

* Add resource allocation for cronjob taginfo

osm-seed/templates/taginfo/taginfo-cronJob.yaml

@@ -10,7 +10,7 @@ metadata:
     environment: {{ .Values.environment }}
     release: {{ .Release.Name }}
 spec:
-  schedule: {{ .Values.taginfo.cronjob.schedule | quote}}
+  schedule: {{ .Values.taginfo.cronjob.schedule | quote }}
   concurrencyPolicy: Forbid
   startingDeadlineSeconds: 10
   successfulJobsHistoryLimit: 2
@@ -31,10 +31,19 @@ spec:
             envFrom:
               - configMapRef:
                   name: {{ .Release.Name }}-taginfo-common-env
+            {{- if .Values.taginfo.cronjob.resources.enabled }}
+            resources:
+              requests:
+                memory: {{ .Values.taginfo.cronjob.resources.requests.memory }}
+                cpu: {{ .Values.taginfo.cronjob.resources.requests.cpu }}
+              limits:
+                memory: {{ .Values.taginfo.cronjob.resources.limits.memory }}
+                cpu: {{ .Values.taginfo.cronjob.resources.limits.cpu }}
+            {{- end }}
           restartPolicy: Never
           {{- if .Values.taginfo.cronjob.nodeSelector.enabled }}
           nodeSelector:
-            {{ .Values.taginfo.cronjob.nodeSelector.label_key }} : {{ .Values.taginfo.cronjob.nodeSelector.label_value }}
+            {{ .Values.taginfo.cronjob.nodeSelector.label_key }}: {{ .Values.taginfo.cronjob.nodeSelector.label_value }}
           {{- end }}
 {{- end }}
-{{- end }}
+{{- end }}

osm-seed/templates/tiler-db/tiler-db-configmap.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.tilerDb.enabled -}}
+{{- if .Values.tilerDb.postgresqlConfig.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-tiler-db-postgresql-config
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    release: {{ .Release.Name }}
+data:
+  postgresql.conf: |
+{{ .Values.tilerDb.postgresqlConfig.values | indent 4 }}
+{{- end }}
+{{- end }}

osm-seed/templates/tiler-db/tiler-db-statefulset.yaml

@@ -45,18 +45,14 @@ spec:
               value: {{ default "postgres" .Values.tilerDb.env.POSTGRES_USER | quote }}
             - name: PGDATA
               value: {{ .Values.tilerDb.persistenceDisk.mountPath }}
-            - name: POSTGRES_DB_MAX_CONNECTIONS
-              value: {{ default 100 .Values.tilerDb.env.POSTGRES_DB_MAX_CONNECTIONS | quote }}
-            - name: POSTGRES_DB_SHARED_BUFFERS
-              value: {{ default "128MB" .Values.tilerDb.env.POSTGRES_DB_SHARED_BUFFERS | quote }}
-            - name: POSTGRES_DB_WORK_MEM
-              value: {{ default "4MB" .Values.tilerDb.env.POSTGRES_DB_WORK_MEM | quote }}
-            - name: POSTGRES_DB_MAINTENANCE_WORK_MEM
-              value: {{ default "64MB" .Values.tilerDb.env.POSTGRES_DB_MAINTENANCE_WORK_MEM | quote }}
-            - name: POSTGRES_DB_EFFECTIVE_CACHE_SIZE
-              value: {{ default "4GB" .Values.tilerDb.env.POSTGRES_DB_EFFECTIVE_CACHE_SIZE | quote }}
             - name: POD_IP
               valueFrom: { fieldRef: { fieldPath: status.podIP } }
+           {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }}
+          command:
+            - "sh"
+            - "-c"
+            - "su postgres -c 'postgres -c config_file=/etc/postgresql/postgresql.conf'"
+           {{- end }}
           livenessProbe:
             exec:
               command:
@@ -79,6 +75,10 @@ spec:
             - name: postgres-storage
               mountPath: {{ .Values.tilerDb.persistenceDisk.mountPath }}
               subPath: {{ .Values.tilerDb.persistenceDisk.subPath }}
+            {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }}
+            - name: tiler-db-postgresql-config
+              mountPath: /etc/postgresql/
+            {{- end }}
           {{- if .Values.tilerDb.resources.enabled }}
           resources:
             requests:
@@ -96,6 +96,11 @@ spec:
         {{- else }}
           emptyDir: {}
         {{- end }}
+        {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }}
+        - name: tiler-db-postgresql-config
+          configMap:
+            name: {{ .Release.Name }}-tiler-db-postgresql-config
+        {{- end }}
       {{- if .Values.tilerDb.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }}

osm-seed/values.yaml

@@ -40,8 +40,8 @@ AWS_SSL_ARN: false
 # as the SSL certificate for your services. Else, you will need to configure
 # SSL separately.
 # serviceType: NodePort
-serviceType: clusterIP
-createClusterIssuer: true
+serviceType: ClusterIP
+createClusterIssuer: false
 # Domain that is pointed to the clusterIP
 # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP
 # Then, the cluster configuration will setup services at their respective subdomains:
@@ -67,7 +67,7 @@ adminEmail: [email protected]
 # Variables for osm-seed database
 # ====================================================================================================
 db:
-  enabled: true
+  enabled: false
   image:
     name: ""
     tag: ""
@@ -359,11 +359,11 @@ tilerDb:
     POSTGRES_USER: postgres
     POSTGRES_PASSWORD: "1234"
     POSTGRES_PORT: 5432
-    POSTGRES_DB_MAX_CONNECTIONS: 100
-    POSTGRES_DB_SHARED_BUFFERS: 128MB
-    POSTGRES_DB_WORK_MEM: 4MB
-    POSTGRES_DB_MAINTENANCE_WORK_MEM: 64MB
-    POSTGRES_DB_EFFECTIVE_CACHE_SIZE: 4GB
+    # POSTGRES_DB_MAX_CONNECTIONS: 100
+    # POSTGRES_DB_SHARED_BUFFERS: 128MB
+    # POSTGRES_DB_WORK_MEM: 4MB
+    # POSTGRES_DB_MAINTENANCE_WORK_MEM: 64MB
+    # POSTGRES_DB_EFFECTIVE_CACHE_SIZE: 4GB
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce
@@ -373,8 +373,8 @@ tilerDb:
     localVolumeHostPath: /mnt/tiler-db-data
     localVolumeSize: 10Gi
     # AWS
-    AWS_ElasticBlockStore_volumeID: vol-1234bcd
-    AWS_ElasticBlockStore_size: 50Gi
+    AWS_ElasticBlockStore_volumeID: vol-0cf316caac04fc623
+    AWS_ElasticBlockStore_size: 20Gi
     # GCP
     GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1
     GCP_gcePersistentDisk_size: 50Gi
@@ -390,6 +390,26 @@ tilerDb:
     enabled: false
     label_key: nodegroup_type
     label_value: tiler
+  # To use postgresqlConfig, you need to have already $PGDATA set up in a persistent disk
+  postgresqlConfig:
+    enabled: false
+    values: |
+      listen_addresses = '*'
+      max_connections = 100
+      shared_buffers = 128MB
+      effective_cache_size = 256MB
+      maintenance_work_mem = 64MB
+      checkpoint_completion_target = 0.7
+      wal_buffers = 16MB
+      default_statistics_target = 100
+      random_page_cost = 1.1
+      effective_io_concurrency = 200
+      work_mem = 4MB
+      min_wal_size = 80MB
+      max_wal_size = 1GB
+      max_worker_processes = 4
+      max_parallel_workers_per_gather = 2
+      max_parallel_workers = 4
 # ====================================================================================================
 # Variables for tiler-imposm
 # ====================================================================================================
@@ -708,7 +728,7 @@ taginfo:
     name: ""
     tag: ""
   serviceAnnotations: {}
-  ingressDomain: taginfo.dev.osmseed.org
+  ingressDomain: taginfo-dev.staging.openhistoricalmap.org
   env:
     URL_PLANET_FILE_STATE: https://planet.osm.org/pbf/state.txt
     URL_HISTORY_PLANET_FILE_STATE: https://planet.osm.org/pbf/full-history/state.txt
@@ -734,7 +754,24 @@ taginfo:
       memory: "2Gi"
       cpu: "2"
   nodeSelector:
-    enabled: false
+    enabled: true
+    label_key: nodegroup_type
+    label_value: web
+  cronjob:
+      enabled: true
+      schedule: "0 2 */3 * *"
+      nodeSelector:
+        enabled: true
+        label_key: nodegroup_type
+        label_value: job_xlarge
+      resources:
+        enabled: false
+        requests:
+          memory: "13Gi"
+          cpu: "3600m"
+        limits:
+          memory: "14Gi"
+          cpu: "3800m"
 # ====================================================================================================
 # Variables for osm-simple-metrics
 # ====================================================================================================