Add UPLOAD_EXPIRED_FILES env var for imposm

developmentseed · Feb 16, 2024 · 4381661 · 4381661
1 parent 19f2d34
commit 4381661
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 18 deletions.
diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -49,18 +49,41 @@ function getData() {
     fi
 }
 
+getFormattedDate() {
+    local file_path="$1"
+    if command -v stat >/dev/null 2>&1; then
+        local modification_date=$(stat -c %Y "$file_path")
+        if [ $? -eq 0 ]; then
+            local formatted_date=$(date -d "@$modification_date" "+%Y-%m-%d:%H:%M:%S")
+            echo "Created/Updated date of $file_path: $formatted_date"
+        else
+            echo "Error: Unable to get file modification date for file ${file_path}"
+        fi
+    else
+        echo "Error: 'stat' command not found. Unable to get file modification date, for file ${file_path}"
+    fi
+}
+
 function uploadExpiredFiles() {
-    ### Upload the expired files to the cloud provider
-    for file in $(find $IMPOSM3_EXPIRE_DIR -type f -cmin -1); do
+    # Upload the expired files to the cloud provider
+    for file in $(find "$IMPOSM3_EXPIRE_DIR" -type f -cmin -1); do
         bucketFile=${file#*"$WORKDIR"}
-        echo $(date +%F_%H:%M:%S)":" $file
-        # AWS
-        if [ "$CLOUDPROVIDER" == "aws" ]; then
-            aws s3 cp $file ${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketFile} --acl public-read
-        fi
-        # Google Storage
-        if [ "$CLOUDPROVIDER" == "gcp" ]; then
-            gsutil cp -a public-read $file ${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketFile}
+        getFormattedDate "$file"
+        # UPLOAD_EXPIRED_FILES=true to upload the expired to cloud provider
+        if [ "$UPLOAD_EXPIRED_FILES" == "true" ]; then
+            echo "Uploading expired file ${file} to ${AWS_S3_BUCKET}"
+
+            # AWS
+            if [ "$CLOUDPROVIDER" == "aws" ]; then
+                aws s3 cp "$file" "${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketFile}" --acl public-read
+            fi
+
+            # Google Storage
+            if [ "$CLOUDPROVIDER" == "gcp" ]; then
+                gsutil cp -a public-read "$file" "${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketFile}"
+            fi
+        else
+            echo "Expired files were not uploaded because UPLOAD_EXPIRED_FILES=${UPLOAD_EXPIRED_FILES}"
         fi
     done
 }

diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml
@@ -47,23 +47,25 @@ spec:
           - name: POSTGRES_HOST
             value: {{ .Release.Name }}-tiler-db
           - name: POSTGRES_DB
-            value: {{ .Values.tilerDb.env.POSTGRES_DB }}
+            value: {{ .Values.tilerDb.env.POSTGRES_DB | quote }}
           - name: POSTGRES_PASSWORD
-            value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
+            value: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }}
           - name: POSTGRES_USER
-            value: {{ .Values.tilerDb.env.POSTGRES_USER }}
+            value: {{ .Values.tilerDb.env.POSTGRES_USER | quote }}
           - name: POSTGRES_PORT
             value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
           - name: TILER_IMPORT_FROM
-            value: {{ .Values.tilerImposm.env.TILER_IMPORT_FROM }}
+            value: {{ .Values.tilerImposm.env.TILER_IMPORT_FROM | quote }}
           - name: TILER_IMPORT_PBF_URL
-            value: {{ .Values.tilerImposm.env.TILER_IMPORT_PBF_URL }}
+            value: {{ .Values.tilerImposm.env.TILER_IMPORT_PBF_URL | quote }}
           - name: REPLICATION_URL
-            value: {{ .Values.tilerImposm.env.REPLICATION_URL }}
+            value: {{ .Values.tilerImposm.env.REPLICATION_URL | quote }}
           - name: SEQUENCE_NUMBER
-            value: {{ .Values.tilerImposm.env.SEQUENCE_NUMBER | quote}}
+            value: {{ .Values.tilerImposm.env.SEQUENCE_NUMBER | quote }}
           - name: OVERWRITE_STATE
-            value: {{ quote .Values.tilerImposm.env.OVERWRITE_STATE }}
+            value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }}
+          - name: UPLOAD_EXPIRED_FILES
+            value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | quote }}
           - name: CLOUDPROVIDER
             value: {{ .Values.cloudProvider }}
           # In case cloudProvider=aws