From a6887ec7af10e62f75234938b462f574e85a46bf Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Thu, 8 Aug 2024 11:16:20 -0500 Subject: [PATCH] Set configmap for tiler-db's postgresql.config (#337) * Set configmap for postgresql.config tiler-db * Remove image values * Add resource allocation for cronjob taginfo --- .../{db-mapconfig.yaml => db-configmap.yaml} | 0 .../templates/taginfo/taginfo-cronJob.yaml | 15 ++++- .../tiler-db/tiler-db-configmap.yaml | 14 +++++ .../tiler-db/tiler-db-statefulset.yaml | 25 +++++--- osm-seed/values.yaml | 61 +++++++++++++++---- 5 files changed, 90 insertions(+), 25 deletions(-) rename osm-seed/templates/db/{db-mapconfig.yaml => db-configmap.yaml} (100%) create mode 100644 osm-seed/templates/tiler-db/tiler-db-configmap.yaml diff --git a/osm-seed/templates/db/db-mapconfig.yaml b/osm-seed/templates/db/db-configmap.yaml similarity index 100% rename from osm-seed/templates/db/db-mapconfig.yaml rename to osm-seed/templates/db/db-configmap.yaml diff --git a/osm-seed/templates/taginfo/taginfo-cronJob.yaml b/osm-seed/templates/taginfo/taginfo-cronJob.yaml index 6fb5bc38..ea06ed5b 100644 --- a/osm-seed/templates/taginfo/taginfo-cronJob.yaml +++ b/osm-seed/templates/taginfo/taginfo-cronJob.yaml @@ -10,7 +10,7 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: - schedule: {{ .Values.taginfo.cronjob.schedule | quote}} + schedule: {{ .Values.taginfo.cronjob.schedule | quote }} concurrencyPolicy: Forbid startingDeadlineSeconds: 10 successfulJobsHistoryLimit: 2 @@ -31,10 +31,19 @@ spec: envFrom: - configMapRef: name: {{ .Release.Name }}-taginfo-common-env + {{- if .Values.taginfo.cronjob.resources.enabled }} + resources: + requests: + memory: {{ .Values.taginfo.cronjob.resources.requests.memory }} + cpu: {{ .Values.taginfo.cronjob.resources.requests.cpu }} + limits: + memory: {{ .Values.taginfo.cronjob.resources.limits.memory }} + cpu: {{ .Values.taginfo.cronjob.resources.limits.cpu }} + {{- end }} restartPolicy: Never {{- if .Values.taginfo.cronjob.nodeSelector.enabled }} nodeSelector: - {{ .Values.taginfo.cronjob.nodeSelector.label_key }} : {{ .Values.taginfo.cronjob.nodeSelector.label_value }} + {{ .Values.taginfo.cronjob.nodeSelector.label_key }}: {{ .Values.taginfo.cronjob.nodeSelector.label_value }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/tiler-db/tiler-db-configmap.yaml b/osm-seed/templates/tiler-db/tiler-db-configmap.yaml new file mode 100644 index 00000000..efef635a --- /dev/null +++ b/osm-seed/templates/tiler-db/tiler-db-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.tilerDb.enabled -}} +{{- if .Values.tilerDb.postgresqlConfig.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-tiler-db-postgresql-config + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} +data: + postgresql.conf: | +{{ .Values.tilerDb.postgresqlConfig.values | indent 4 }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml b/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml index 4353110c..724f1f04 100644 --- a/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml +++ b/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml @@ -45,18 +45,14 @@ spec: value: {{ default "postgres" .Values.tilerDb.env.POSTGRES_USER | quote }} - name: PGDATA value: {{ .Values.tilerDb.persistenceDisk.mountPath }} - - name: POSTGRES_DB_MAX_CONNECTIONS - value: {{ default 100 .Values.tilerDb.env.POSTGRES_DB_MAX_CONNECTIONS | quote }} - - name: POSTGRES_DB_SHARED_BUFFERS - value: {{ default "128MB" .Values.tilerDb.env.POSTGRES_DB_SHARED_BUFFERS | quote }} - - name: POSTGRES_DB_WORK_MEM - value: {{ default "4MB" .Values.tilerDb.env.POSTGRES_DB_WORK_MEM | quote }} - - name: POSTGRES_DB_MAINTENANCE_WORK_MEM - value: {{ default "64MB" .Values.tilerDb.env.POSTGRES_DB_MAINTENANCE_WORK_MEM | quote }} - - name: POSTGRES_DB_EFFECTIVE_CACHE_SIZE - value: {{ default "4GB" .Values.tilerDb.env.POSTGRES_DB_EFFECTIVE_CACHE_SIZE | quote }} - name: POD_IP valueFrom: { fieldRef: { fieldPath: status.podIP } } + {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }} + command: + - "sh" + - "-c" + - "su postgres -c 'postgres -c config_file=/etc/postgresql/postgresql.conf'" + {{- end }} livenessProbe: exec: command: @@ -79,6 +75,10 @@ spec: - name: postgres-storage mountPath: {{ .Values.tilerDb.persistenceDisk.mountPath }} subPath: {{ .Values.tilerDb.persistenceDisk.subPath }} + {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }} + - name: tiler-db-postgresql-config + mountPath: /etc/postgresql/ + {{- end }} {{- if .Values.tilerDb.resources.enabled }} resources: requests: @@ -96,6 +96,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }} + - name: tiler-db-postgresql-config + configMap: + name: {{ .Release.Name }}-tiler-db-postgresql-config + {{- end }} {{- if .Values.tilerDb.nodeSelector.enabled }} nodeSelector: {{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }} diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index d7e303b6..f3516517 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -40,8 +40,8 @@ AWS_SSL_ARN: false # as the SSL certificate for your services. Else, you will need to configure # SSL separately. # serviceType: NodePort -serviceType: clusterIP -createClusterIssuer: true +serviceType: ClusterIP +createClusterIssuer: false # Domain that is pointed to the clusterIP # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP # Then, the cluster configuration will setup services at their respective subdomains: @@ -67,7 +67,7 @@ adminEmail: admin@osmseed.org # Variables for osm-seed database # ==================================================================================================== db: - enabled: true + enabled: false image: name: "" tag: "" @@ -359,11 +359,11 @@ tilerDb: POSTGRES_USER: postgres POSTGRES_PASSWORD: "1234" POSTGRES_PORT: 5432 - POSTGRES_DB_MAX_CONNECTIONS: 100 - POSTGRES_DB_SHARED_BUFFERS: 128MB - POSTGRES_DB_WORK_MEM: 4MB - POSTGRES_DB_MAINTENANCE_WORK_MEM: 64MB - POSTGRES_DB_EFFECTIVE_CACHE_SIZE: 4GB + # POSTGRES_DB_MAX_CONNECTIONS: 100 + # POSTGRES_DB_SHARED_BUFFERS: 128MB + # POSTGRES_DB_WORK_MEM: 4MB + # POSTGRES_DB_MAINTENANCE_WORK_MEM: 64MB + # POSTGRES_DB_EFFECTIVE_CACHE_SIZE: 4GB persistenceDisk: enabled: false accessMode: ReadWriteOnce @@ -373,8 +373,8 @@ tilerDb: localVolumeHostPath: /mnt/tiler-db-data localVolumeSize: 10Gi # AWS - AWS_ElasticBlockStore_volumeID: vol-1234bcd - AWS_ElasticBlockStore_size: 50Gi + AWS_ElasticBlockStore_volumeID: vol-0cf316caac04fc623 + AWS_ElasticBlockStore_size: 20Gi # GCP GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1 GCP_gcePersistentDisk_size: 50Gi @@ -390,6 +390,26 @@ tilerDb: enabled: false label_key: nodegroup_type label_value: tiler + # To use postgresqlConfig, you need to have already $PGDATA set up in a persistent disk + postgresqlConfig: + enabled: false + values: | + listen_addresses = '*' + max_connections = 100 + shared_buffers = 128MB + effective_cache_size = 256MB + maintenance_work_mem = 64MB + checkpoint_completion_target = 0.7 + wal_buffers = 16MB + default_statistics_target = 100 + random_page_cost = 1.1 + effective_io_concurrency = 200 + work_mem = 4MB + min_wal_size = 80MB + max_wal_size = 1GB + max_worker_processes = 4 + max_parallel_workers_per_gather = 2 + max_parallel_workers = 4 # ==================================================================================================== # Variables for tiler-imposm # ==================================================================================================== @@ -708,7 +728,7 @@ taginfo: name: "" tag: "" serviceAnnotations: {} - ingressDomain: taginfo.dev.osmseed.org + ingressDomain: taginfo-dev.staging.openhistoricalmap.org env: URL_PLANET_FILE_STATE: https://planet.osm.org/pbf/state.txt URL_HISTORY_PLANET_FILE_STATE: https://planet.osm.org/pbf/full-history/state.txt @@ -734,7 +754,24 @@ taginfo: memory: "2Gi" cpu: "2" nodeSelector: - enabled: false + enabled: true + label_key: nodegroup_type + label_value: web + cronjob: + enabled: true + schedule: "0 2 */3 * *" + nodeSelector: + enabled: true + label_key: nodegroup_type + label_value: job_xlarge + resources: + enabled: false + requests: + memory: "13Gi" + cpu: "3600m" + limits: + memory: "14Gi" + cpu: "3800m" # ==================================================================================================== # Variables for osm-simple-metrics # ====================================================================================================