diff --git a/osm-seed/templates/osmcha-app/cronJob.yaml b/osm-seed/templates/osmcha-app/cronJob.yaml index acb11030..5bb9871e 100644 --- a/osm-seed/templates/osmcha-app/cronJob.yaml +++ b/osm-seed/templates/osmcha-app/cronJob.yaml @@ -11,10 +11,11 @@ metadata: spec: schedule: "*/2 * * * *" concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 2 - failedJobsHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 jobTemplate: spec: + ttlSecondsAfterFinished: 120 template: spec: containers: @@ -32,12 +33,12 @@ spec: restartPolicy: Never {{- if .Values.osmchaApi.nodeSelector.enabled }} nodeSelector: - {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }} {{- end }} --- apiVersion: batch/v1 -kind: CronJob +kind: CronJob metadata: name: {{ .Release.Name }}-osmcha-process-changesets-cronjob labels: @@ -48,10 +49,11 @@ metadata: spec: schedule: "0 * * * *" concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 2 - failedJobsHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 jobTemplate: spec: + ttlSecondsAfterFinished: 120 template: spec: containers: @@ -62,13 +64,19 @@ spec: - /bin/sh - -c - | - python manage.py backfill_changesets $(date -d "yesterday" +%Y-%m-%d) $(date +%Y-%m-%d) + YESTERDAY=$(date -d "yesterday" +%Y-%m-%d) + TODAY=$(date +%Y-%m-%d) + + python manage.py backfill_changesets "$YESTERDAY" "$TODAY" + EXIT_CODE=$? + + exit $EXIT_CODE envFrom: - configMapRef: name: {{ .Release.Name }}-osmcha-common-env restartPolicy: Never {{- if .Values.osmchaApi.nodeSelector.enabled }} nodeSelector: - {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }} {{- end }} {{- end }} diff --git a/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml b/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml index 724f1f04..c1b473e5 100644 --- a/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml +++ b/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml @@ -79,6 +79,8 @@ spec: - name: tiler-db-postgresql-config mountPath: /etc/postgresql/ {{- end }} + - name: shared-memory + mountPath: /dev/shm {{- if .Values.tilerDb.resources.enabled }} resources: requests: @@ -101,6 +103,10 @@ spec: configMap: name: {{ .Release.Name }}-tiler-db-postgresql-config {{- end }} + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: {{ .Values.tilerDb.sharedMemorySize | default "64Mi" }} {{- if .Values.tilerDb.nodeSelector.enabled }} nodeSelector: {{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }} diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml index a5834332..f765c005 100644 --- a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml +++ b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml @@ -69,6 +69,8 @@ spec: value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }} - name: IMPORT_OSM_LAND value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }} + - name: IMPOSM3_IMPORT_LAYERS + value: {{ .Values.tilerImposm.env.IMPOSM3_IMPORT_LAYERS | default "all" | quote }} - name: CLOUDPROVIDER value: {{ .Values.cloudProvider }} # In case cloudProvider=aws diff --git a/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml b/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml index f6e7995d..285715da 100644 --- a/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml +++ b/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml @@ -33,53 +33,9 @@ spec: memory: {{ .Values.tilerServerCacheCleaner.resources.limits.memory }} cpu: {{ .Values.tilerServerCacheCleaner.resources.limits.cpu }} {{- end }} - env: - # tiler-db env variables - - name: POSTGRES_HOST - value: {{ .Release.Name }}-tiler-db - - name: POSTGRES_DB - value: {{ .Values.tilerDb.env.POSTGRES_DB }} - - name: POSTGRES_PASSWORD - value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }} - - name: POSTGRES_USER - value: {{ .Values.tilerDb.env.POSTGRES_USER }} - - name: POSTGRES_PORT - value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }} - # tiler-server env variables - - name: TILER_SERVER_PORT - value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }} - - name: TILER_CACHE_TYPE - value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }} - - name: TILER_CACHE_REGION - value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }} - - name: TILER_CACHE_BUCKET - value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }} - - name: TILER_CACHE_BASEPATH - value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }} - - name: TILER_CACHE_AWS_ACCESS_KEY_ID - value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }} - - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY - value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }} - - name: TILER_CACHE_MAX_ZOOM - value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }} - - name: MAX_NUM_PS - value: {{ quote .Values.tilerServerCacheCleaner.env.MAX_NUM_PS }} - - name: PROCESS_NAME - value: {{ .Values.tilerServerCacheCleaner.env.PROCESS_NAME }} - - name: KILL_PROCESS - value: {{ .Values.tilerServerCacheCleaner.env.KILL_PROCESS }} - - name: CLOUDPROVIDER - value: {{ .Values.cloudProvider }} - # In case cloudProvider=aws - {{- if eq .Values.cloudProvider "aws" }} - - name: AWS_S3_BUCKET - value: {{ .Values.AWS_S3_BUCKET }} - {{- end }} - # In case cloudProvider=gcp - {{- if eq .Values.cloudProvider "gcp" }} - - name: GCP_STORAGE_BUCKET - value: {{ .Values.GCP_STORAGE_BUCKET }} - {{- end }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-tiler-server-cm {{- if .Values.tilerServerCacheCleaner.nodeSelector.enabled }} nodeSelector: {{ .Values.tilerServerCacheCleaner.nodeSelector.label_key }} : {{ .Values.tilerServerCacheCleaner.nodeSelector.label_value }} diff --git a/osm-seed/templates/tiler-server/tiler-server-configMap.yaml b/osm-seed/templates/tiler-server/tiler-server-configMap.yaml new file mode 100644 index 00000000..06c858a7 --- /dev/null +++ b/osm-seed/templates/tiler-server/tiler-server-configMap.yaml @@ -0,0 +1,29 @@ +{{- if .Values.tilerServer.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-tiler-server-cm +data: + POSTGRES_HOST: {{ .Release.Name }}-tiler-db + POSTGRES_DB: {{ .Values.tilerDb.env.POSTGRES_DB | quote }} + POSTGRES_PASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }} + POSTGRES_USER: {{ .Values.tilerDb.env.POSTGRES_USER | quote }} + POSTGRES_PORT: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }} + TILER_SERVER_PORT: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }} + TILER_CACHE_TYPE: {{ .Values.tilerServer.env.TILER_CACHE_TYPE | quote }} + TILER_CACHE_REGION: {{ .Values.tilerServer.env.TILER_CACHE_REGION | quote }} + TILER_CACHE_BUCKET: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET | quote }} + TILER_CACHE_BASEPATH: {{ .Values.tilerServer.env.TILER_CACHE_BASEPATH | quote }} + TILER_CACHE_AWS_ACCESS_KEY_ID: {{ .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID | quote }} + TILER_CACHE_AWS_SECRET_ACCESS_KEY: {{ .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY | quote }} + TILER_CACHE_MAX_ZOOM: {{ .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM | quote }} + EXECUTE_REINDEX: {{ .Values.tilerServer.env.EXECUTE_REINDEX | default "false" | quote }} + EXECUTE_VACUUM_ANALYZE: {{ .Values.tilerServer.env.EXECUTE_VACUUM_ANALYZE | default "false" | quote }} + CLOUDPROVIDER: {{ .Values.cloudProvider | quote }} + {{- if eq .Values.cloudProvider "aws" }} + AWS_S3_BUCKET: {{ .Values.AWS_S3_BUCKET | quote }} + {{- end }} + {{- if eq .Values.cloudProvider "gcp" }} + GCP_STORAGE_BUCKET: {{ .Values.GCP_STORAGE_BUCKET | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/tiler-server/tiler-server-deployment.yaml b/osm-seed/templates/tiler-server/tiler-server-deployment.yaml index 3c99bfc3..bc7641c2 100644 --- a/osm-seed/templates/tiler-server/tiler-server-deployment.yaml +++ b/osm-seed/templates/tiler-server/tiler-server-deployment.yaml @@ -35,7 +35,7 @@ spec: httpGet: path: / port: {{ .Values.tilerServer.env.TILER_SERVER_PORT }} - initialDelaySeconds: 600 # 10 min, because the compile process takes time. + initialDelaySeconds: 1500 timeoutSeconds: 30 {{- if .Values.tilerServer.resources.enabled }} resources: @@ -46,47 +46,9 @@ spec: memory: {{ .Values.tilerServer.resources.limits.memory }} cpu: {{ .Values.tilerServer.resources.limits.cpu }} {{- end }} - env: - # tiler-db env variables - - name: POSTGRES_HOST - value: {{ .Release.Name }}-tiler-db - - name: POSTGRES_DB - value: {{ .Values.tilerDb.env.POSTGRES_DB }} - - name: POSTGRES_PASSWORD - value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }} - - name: POSTGRES_USER - value: {{ .Values.tilerDb.env.POSTGRES_USER }} - - name: POSTGRES_PORT - value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }} - # tiler-server env variables - - name: TILER_SERVER_PORT - value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }} - - name: TILER_CACHE_TYPE - value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }} - - name: TILER_CACHE_REGION - value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }} - - name: TILER_CACHE_BUCKET - value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }} - - name: TILER_CACHE_BASEPATH - value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }} - - name: TILER_CACHE_AWS_ACCESS_KEY_ID - value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }} - - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY - value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }} - - name: TILER_CACHE_MAX_ZOOM - value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }} - - name: CLOUDPROVIDER - value: {{ .Values.cloudProvider }} - # In case cloudProvider=aws - {{- if eq .Values.cloudProvider "aws" }} - - name: AWS_S3_BUCKET - value: {{ .Values.AWS_S3_BUCKET }} - {{- end }} - # In case cloudProvider=gcp - {{- if eq .Values.cloudProvider "gcp" }} - - name: GCP_STORAGE_BUCKET - value: {{ .Values.GCP_STORAGE_BUCKET }} - {{- end }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-tiler-server-cm {{- if .Values.tilerServer.nodeSelector.enabled }} nodeSelector: {{ .Values.tilerServer.nodeSelector.label_key }} : {{ .Values.tilerServer.nodeSelector.label_value }} diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 2a345d6d..b32db40c 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -376,6 +376,7 @@ tilerDb: # GCP GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1 GCP_gcePersistentDisk_size: 50Gi + sharedMemorySize: 64Mi resources: enabled: false requests: @@ -423,6 +424,7 @@ tilerImposm: UPLOAD_EXPIRED_FILES: true IMPORT_NATURAL_EARTH: true IMPORT_OSM_LAND: true + IMPOSM3_IMPORT_LAYERS: all persistenceDisk: enabled: false accessMode: ReadWriteOnce @@ -469,6 +471,8 @@ tilerServer: TILER_CACHE_REGION: us-east-1 TILER_CACHE_AWS_ACCESS_KEY_ID: xyz TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q + EXECUTE_REINDEX: false + EXECUTE_VACUUM_ANALYZE: false command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh # In case you use TILER_CACHE_TYPE: file with persistenceDisk persistenceDisk: @@ -526,6 +530,21 @@ tilerServerCacheCleaner: minReplicas: 1 maxReplicas: 2 cpuUtilization: 60 + +tilerCacheCleanerJob: + enabled: false + tilesFile: imposm/imposm3_expire_dir/latest.tiles + minZoom: 0 + maxZoom: 20 + overwrite: true + resources: + enabled: false + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1" # ==================================================================================================== # Variables for Tasking Manager DB # ====================================================================================================