Add function to clean up backup older than x days

developmentseed · Nov 1, 2023 · 236fa2e · 236fa2e
1 parent f35b460
commit 236fa2e
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 41 deletions.
diff --git a/images/backup-restore/start.sh b/images/backup-restore/start.sh
@@ -64,6 +64,28 @@ restoreDB() {
 	done
 }
 
+delete_old_s3_files() {
+    # Use RETENTION_DAYS from environment variable or default to 30 days
+    if [ -z "${RETENTION_DAYS}" ]; then
+        DAYS_AGO=30
+    else
+        DAYS_AGO="${RETENTION_DAYS}"
+    fi
+
+    echo "Files older than $DAYS_AGO days will be deleted."
+    echo "Processing s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/"
+    TARGET_DATE=$(date -d "${DAYS_AGO} days ago" +%Y-%m-%d)
+    aws s3 ls "s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" --recursive | while read -r line; do
+        FILE_DATE=$(echo "$line" | awk '{print $1}')
+        FILE_PATH=$(echo "$line" | awk '{print $4}')
+        if [[ "$FILE_DATE" < "$TARGET_DATE" && ! -z "$FILE_PATH" ]]; then
+            echo "Deleting ${FILE_PATH} which was modified on ${FILE_DATE}"
+            aws s3 rm "s3://${AWS_S3_BUCKET}/${FILE_PATH}"
+        fi
+    done
+}
+
+
 # Main logic
 case "${DB_ACTION}" in
 backup)
@@ -77,3 +99,10 @@ restore)
 	exit 1
 	;;
 esac
+
+# Check for the CLEAN_BACKUPS var
+if [ "$CLEAN_BACKUPS" == "true" ]; then
+    delete_old_s3_files
+else
+    echo "CLEAN_BACKUPS is not set to true. Skipping deletion."
+fi
diff --git a/osm-seed/templates/db/db-backup-job.yaml b/osm-seed/templates/db/db-backup-job.yaml
@@ -43,6 +43,8 @@ spec:
                 value: {{ $job.env.RESTORE_URL_FILE}}
               - name: CLEAN_BACKUPS
                 value: {{ quote $job.env.CLEAN_BACKUPS }}
+              - name: RETENTION_DAYS
+                value: {{ quote $job.env.RETENTION_DAYS }}
             {{- end }}
             # Env vars for osmcha database
             {{ if eq $job.name "osmcha-db" }}
@@ -64,6 +66,8 @@ spec:
                 value: {{ $job.env.RESTORE_URL_FILE}}
               - name: CLEAN_BACKUPS
                 value: {{ quote $job.env.CLEAN_BACKUPS }}
+              - name: RETENTION_DAYS
+                value: {{ quote $job.env.RETENTION_DAYS }}
             {{- end }}
               # Cloud provider
               - name: CLOUDPROVIDER

diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml
@@ -200,47 +200,49 @@ fullHistory:
   # ====================================================================================================
 dbBackupRestore:
   image:
-    name: ''
-    tag: ''
+    name: 'developmentseed/osmseed-backup-restore'
+    tag: '0.1.0-0.dev.git.942.h994af82'
   cronjobs:
-  - name: web-db
-    enabled: true
-    schedule: '* * * * *'
-    env:
-      DB_ACTION: backup
-      SET_DATE: true
-      BACKUP_CLOUD_FOLDER: database/web-db
-      BACKUP_CLOUD_FILE: osmseed-api-web-db
-      AWS_S3_BUCKET: secure-bucket
-    resources:
-      enabled: false
-      requests:
-        memory: '300Mi'
-        cpu: '0.5'
-      limits:
-        memory: '400Mi'
-        cpu: '0.6'
-    nodeSelector:
-      enabled: false
-  - name: osmcha-db
-    enabled: true
-    schedule: '* * * * *'
-    env:
-      DB_ACTION: backup
-      SET_DATE: true
-      BACKUP_CLOUD_FOLDER: database/osmcha-db
-      BACKUP_CLOUD_FILE: osmseed-osmcha-db
-      AWS_S3_BUCKET: secure-bucket
-    resources:
-      enabled: false
-      requests:
-        memory: '300Mi'
-        cpu: '0.5'
-      limits:
-        memory: '400Mi'
-        cpu: '0.6'
-    nodeSelector:
-      enabled: false
+    - name: web-db
+      enabled: true
+      schedule: '* * * * *'
+      env:
+        DB_ACTION: backup
+        SET_DATE: true
+        BACKUP_CLOUD_FOLDER: database/web-db
+        BACKUP_CLOUD_FILE: osmseed-api-web-db
+        AWS_S3_BUCKET: secure-bucket
+        RETENTION_DAYS: "30"
+      resources:
+        enabled: false
+        requests:
+          memory: '300Mi'
+          cpu: '0.5'
+        limits:
+          memory: '400Mi'
+          cpu: '0.6'
+      nodeSelector:
+        enabled: false
+    - name: osmcha-db
+      enabled: true
+      schedule: '* * * * *'
+      env:
+        DB_ACTION: backup
+        SET_DATE: true
+        BACKUP_CLOUD_FOLDER: database/osmcha-db
+        BACKUP_CLOUD_FILE: osmseed-osmcha-db
+        AWS_S3_BUCKET: secure-bucket
+        RETENTION_DAYS: "30"
+      resources:
+        enabled: false
+        requests:
+          memory: '300Mi'
+          cpu: '0.5'
+        limits:
+          memory: '400Mi'
+          cpu: '0.6'
+      nodeSelector:
+        enabled: false
 
 # ====================================================================================================
 # Variables for osm-seed for osmosis, this configuration os to get the planet dump files from apidb
@@ -752,7 +754,8 @@ osmchaWeb:
     tag: ''
   env:
     OSM_SERVER_URL: 'https://staging.openstreetmap.org'
-    DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}"
+    DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',\
+      \ 'LOCATION': '127.0.0.1:11211'}}"
     DJANGO_DEBUG: "False"
     DJANGO_SECRET_KEY: abc
     DJANGO_SECURE_BROWSER_XSS_FILTER: "True"