Add configMap for tiler (#351)

* Increase initialDelaySeconds for tiler db deployment

* Add configmap for tiler server

* Add service for prometheus - tiler server

* Remove service for prometheus - tiler server

* Update tiler db templates - adding sharedMemorySize

* Set default sharedMemorySize

* Add ttlSecondsAfterFinished for cronjob - osmcha

* Add env var for imposm - tiler

* Add env vars for tiler server

* Add default values for EXECUTE_REINDEX and EXECUTE_VACUUM_ANALYZE

osm-seed/templates/osmcha-app/cronJob.yaml

@@ -11,10 +11,11 @@ metadata:
 spec:
   schedule: "*/2 * * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 120
       template:
         spec:
           containers:
@@ -32,12 +33,12 @@ spec:
           restartPolicy: Never
           {{- if .Values.osmchaApi.nodeSelector.enabled }}
           nodeSelector:
-            {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
+            {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
           {{- end }}
 
 ---
 apiVersion: batch/v1
-kind: CronJob
+kind: CronJob 
 metadata:
   name: {{ .Release.Name }}-osmcha-process-changesets-cronjob
   labels:
@@ -48,10 +49,11 @@ metadata:
 spec:
   schedule: "0 * * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 120
       template:
         spec:
           containers:
@@ -62,13 +64,19 @@ spec:
             - /bin/sh
             - -c
             - |
-              python manage.py backfill_changesets $(date -d "yesterday" +%Y-%m-%d) $(date +%Y-%m-%d)
+              YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
+              TODAY=$(date +%Y-%m-%d)
+
+              python manage.py backfill_changesets "$YESTERDAY" "$TODAY"
+              EXIT_CODE=$?
+
+              exit $EXIT_CODE
             envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-osmcha-common-env
           restartPolicy: Never
           {{- if .Values.osmchaApi.nodeSelector.enabled }}
           nodeSelector:
-            {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
+            {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
           {{- end }}
 {{- end }}

osm-seed/templates/tiler-db/tiler-db-statefulset.yaml

@@ -79,6 +79,8 @@ spec:
             - name: tiler-db-postgresql-config
               mountPath: /etc/postgresql/
             {{- end }}
+            - name: shared-memory
+              mountPath: /dev/shm
           {{- if .Values.tilerDb.resources.enabled }}
           resources:
             requests:
@@ -101,6 +103,10 @@ spec:
           configMap:
             name: {{ .Release.Name }}-tiler-db-postgresql-config
         {{- end }}
+        - name: shared-memory
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ .Values.tilerDb.sharedMemorySize | default "64Mi" }}
       {{- if .Values.tilerDb.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }}

osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml

@@ -69,6 +69,8 @@ spec:
             value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }}
           - name: IMPORT_OSM_LAND
             value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }}
+          - name: IMPOSM3_IMPORT_LAYERS
+            value: {{ .Values.tilerImposm.env.IMPOSM3_IMPORT_LAYERS | default "all" | quote }}
           - name: CLOUDPROVIDER
             value: {{ .Values.cloudProvider }}
           # In case cloudProvider=aws

osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml

@@ -33,53 +33,9 @@ spec:
               memory: {{ .Values.tilerServerCacheCleaner.resources.limits.memory }}
               cpu: {{ .Values.tilerServerCacheCleaner.resources.limits.cpu }}
           {{- end }}
-          env:
-              # tiler-db env variables
-            - name: POSTGRES_HOST
-              value: {{ .Release.Name }}-tiler-db
-            - name: POSTGRES_DB
-              value: {{ .Values.tilerDb.env.POSTGRES_DB }}
-            - name: POSTGRES_PASSWORD
-              value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
-            - name: POSTGRES_USER
-              value: {{ .Values.tilerDb.env.POSTGRES_USER }}
-            - name: POSTGRES_PORT
-              value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
-              # tiler-server env variables
-            - name: TILER_SERVER_PORT
-              value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
-            - name: TILER_CACHE_TYPE
-              value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
-            - name: TILER_CACHE_REGION
-              value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
-            - name: TILER_CACHE_BUCKET
-              value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
-            - name: TILER_CACHE_BASEPATH
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
-            - name: TILER_CACHE_AWS_ACCESS_KEY_ID
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
-            - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
-            - name: TILER_CACHE_MAX_ZOOM
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
-            - name: MAX_NUM_PS
-              value: {{ quote .Values.tilerServerCacheCleaner.env.MAX_NUM_PS }}
-            - name: PROCESS_NAME
-              value: {{ .Values.tilerServerCacheCleaner.env.PROCESS_NAME }}
-            - name: KILL_PROCESS
-              value: {{ .Values.tilerServerCacheCleaner.env.KILL_PROCESS }}
-            - name: CLOUDPROVIDER
-              value: {{ .Values.cloudProvider }}
-            # In case cloudProvider=aws
-            {{- if eq .Values.cloudProvider "aws" }}
-            - name: AWS_S3_BUCKET
-              value: {{ .Values.AWS_S3_BUCKET }}
-            {{- end }}
-            # In case cloudProvider=gcp
-            {{- if eq .Values.cloudProvider "gcp" }}
-            - name: GCP_STORAGE_BUCKET
-              value: {{ .Values.GCP_STORAGE_BUCKET }}
-            {{- end }}
+          envFrom:
+          - configMapRef:
+              name: {{ .Release.Name }}-tiler-server-cm
       {{- if .Values.tilerServerCacheCleaner.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerServerCacheCleaner.nodeSelector.label_key }} : {{ .Values.tilerServerCacheCleaner.nodeSelector.label_value }}

osm-seed/templates/tiler-server/tiler-server-configMap.yaml

@@ -0,0 +1,29 @@
+{{- if .Values.tilerServer.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-tiler-server-cm
+data:
+    POSTGRES_HOST: {{ .Release.Name }}-tiler-db
+    POSTGRES_DB: {{ .Values.tilerDb.env.POSTGRES_DB | quote }}
+    POSTGRES_PASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }}
+    POSTGRES_USER: {{ .Values.tilerDb.env.POSTGRES_USER | quote }}
+    POSTGRES_PORT: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
+    TILER_SERVER_PORT: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
+    TILER_CACHE_TYPE: {{ .Values.tilerServer.env.TILER_CACHE_TYPE | quote }}
+    TILER_CACHE_REGION: {{ .Values.tilerServer.env.TILER_CACHE_REGION | quote }}
+    TILER_CACHE_BUCKET: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET | quote }}
+    TILER_CACHE_BASEPATH: {{ .Values.tilerServer.env.TILER_CACHE_BASEPATH | quote }}
+    TILER_CACHE_AWS_ACCESS_KEY_ID: {{ .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID | quote }}
+    TILER_CACHE_AWS_SECRET_ACCESS_KEY: {{ .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY | quote }}
+    TILER_CACHE_MAX_ZOOM: {{ .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM | quote }}
+    EXECUTE_REINDEX: {{ .Values.tilerServer.env.EXECUTE_REINDEX | default "false" | quote }}
+    EXECUTE_VACUUM_ANALYZE: {{ .Values.tilerServer.env.EXECUTE_VACUUM_ANALYZE | default "false" | quote }}
+    CLOUDPROVIDER: {{ .Values.cloudProvider | quote }}
+    {{- if eq .Values.cloudProvider "aws" }}
+    AWS_S3_BUCKET: {{ .Values.AWS_S3_BUCKET | quote }}
+    {{- end }}
+    {{- if eq .Values.cloudProvider "gcp" }}
+    GCP_STORAGE_BUCKET: {{ .Values.GCP_STORAGE_BUCKET | quote }}
+    {{- end }}
+{{- end }}

osm-seed/templates/tiler-server/tiler-server-deployment.yaml

@@ -35,7 +35,7 @@ spec:
             httpGet:
               path: /
               port: {{ .Values.tilerServer.env.TILER_SERVER_PORT }}
-            initialDelaySeconds: 600 # 10 min, because the compile process takes time.
+            initialDelaySeconds: 1500
             timeoutSeconds: 30
           {{- if .Values.tilerServer.resources.enabled }}
           resources:
@@ -46,47 +46,9 @@ spec:
               memory: {{ .Values.tilerServer.resources.limits.memory }}
               cpu: {{ .Values.tilerServer.resources.limits.cpu }}
           {{- end }}
-          env:
-              # tiler-db env variables
-            - name: POSTGRES_HOST
-              value: {{ .Release.Name }}-tiler-db
-            - name: POSTGRES_DB
-              value: {{ .Values.tilerDb.env.POSTGRES_DB }}
-            - name: POSTGRES_PASSWORD
-              value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
-            - name: POSTGRES_USER
-              value: {{ .Values.tilerDb.env.POSTGRES_USER }}
-            - name: POSTGRES_PORT
-              value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
-              # tiler-server env variables
-            - name: TILER_SERVER_PORT
-              value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
-            - name: TILER_CACHE_TYPE
-              value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
-            - name: TILER_CACHE_REGION
-              value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
-            - name: TILER_CACHE_BUCKET
-              value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
-            - name: TILER_CACHE_BASEPATH
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
-            - name: TILER_CACHE_AWS_ACCESS_KEY_ID
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
-            - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
-            - name: TILER_CACHE_MAX_ZOOM
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
-            - name: CLOUDPROVIDER
-              value: {{ .Values.cloudProvider }}
-            # In case cloudProvider=aws
-            {{- if eq .Values.cloudProvider "aws" }}
-            - name: AWS_S3_BUCKET
-              value: {{ .Values.AWS_S3_BUCKET }}
-            {{- end }}
-            # In case cloudProvider=gcp
-            {{- if eq .Values.cloudProvider "gcp" }}
-            - name: GCP_STORAGE_BUCKET
-              value: {{ .Values.GCP_STORAGE_BUCKET }}
-            {{- end }}
+          envFrom:
+          - configMapRef:
+              name: {{ .Release.Name }}-tiler-server-cm
       {{- if .Values.tilerServer.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerServer.nodeSelector.label_key }} : {{ .Values.tilerServer.nodeSelector.label_value }}

osm-seed/values.yaml

@@ -376,6 +376,7 @@ tilerDb:
     # GCP
     GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1
     GCP_gcePersistentDisk_size: 50Gi
+  sharedMemorySize: 64Mi
   resources:
     enabled: false
     requests:
@@ -423,6 +424,7 @@ tilerImposm:
     UPLOAD_EXPIRED_FILES: true
     IMPORT_NATURAL_EARTH: true
     IMPORT_OSM_LAND: true
+    IMPOSM3_IMPORT_LAYERS: all
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce
@@ -469,6 +471,8 @@ tilerServer:
     TILER_CACHE_REGION: us-east-1
     TILER_CACHE_AWS_ACCESS_KEY_ID: xyz
     TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q
+    EXECUTE_REINDEX: false
+    EXECUTE_VACUUM_ANALYZE: false
   command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
   # In case you use TILER_CACHE_TYPE: file with  persistenceDisk
   persistenceDisk:
@@ -526,6 +530,21 @@ tilerServerCacheCleaner:
     minReplicas: 1
     maxReplicas: 2
     cpuUtilization: 60
+
+tilerCacheCleanerJob:
+  enabled: false
+  tilesFile: imposm/imposm3_expire_dir/latest.tiles
+  minZoom: 0
+  maxZoom: 20
+  overwrite: true
+  resources:
+    enabled: false
+    requests:
+      memory: "1Gi"
+      cpu: "500m"
+    limits:
+      memory: "2Gi"
+      cpu: "1"
 # ====================================================================================================
 # Variables for Tasking Manager DB
 # ====================================================================================================