diff --git a/images/backup-restore/start.sh b/images/backup-restore/start.sh index 18ab9cd4..e259976d 100755 --- a/images/backup-restore/start.sh +++ b/images/backup-restore/start.sh @@ -64,6 +64,28 @@ restoreDB() { done } +delete_old_s3_files() { + # Use RETENTION_DAYS from environment variable or default to 30 days + if [ -z "${RETENTION_DAYS}" ]; then + DAYS_AGO=30 + else + DAYS_AGO="${RETENTION_DAYS}" + fi + + echo "Files older than $DAYS_AGO days will be deleted." + echo "Processing s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" + TARGET_DATE=$(date -d "${DAYS_AGO} days ago" +%Y-%m-%d) + aws s3 ls "s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" --recursive | while read -r line; do + FILE_DATE=$(echo "$line" | awk '{print $1}') + FILE_PATH=$(echo "$line" | awk '{print $4}') + if [[ "$FILE_DATE" < "$TARGET_DATE" && ! -z "$FILE_PATH" ]]; then + echo "Deleting ${FILE_PATH} which was modified on ${FILE_DATE}" + aws s3 rm "s3://${AWS_S3_BUCKET}/${FILE_PATH}" + fi + done +} + + # Main logic case "${DB_ACTION}" in backup) @@ -77,3 +99,10 @@ restore) exit 1 ;; esac + +# Check for the CLEAN_BACKUPS var +if [ "$CLEAN_BACKUPS" == "true" ]; then + delete_old_s3_files +else + echo "CLEAN_BACKUPS is not set to true. Skipping deletion." +fi \ No newline at end of file diff --git a/osm-seed/templates/db/db-backup-job.yaml b/osm-seed/templates/db/db-backup-job.yaml index 86c6a191..255d2cc3 100644 --- a/osm-seed/templates/db/db-backup-job.yaml +++ b/osm-seed/templates/db/db-backup-job.yaml @@ -43,6 +43,8 @@ spec: value: {{ $job.env.RESTORE_URL_FILE}} - name: CLEAN_BACKUPS value: {{ quote $job.env.CLEAN_BACKUPS }} + - name: RETENTION_DAYS + value: {{ quote $job.env.RETENTION_DAYS }} {{- end }} # Env vars for osmcha database {{ if eq $job.name "osmcha-db" }} @@ -64,6 +66,8 @@ spec: value: {{ $job.env.RESTORE_URL_FILE}} - name: CLEAN_BACKUPS value: {{ quote $job.env.CLEAN_BACKUPS }} + - name: RETENTION_DAYS + value: {{ quote $job.env.RETENTION_DAYS }} {{- end }} # Cloud provider - name: CLOUDPROVIDER diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 3e4559cb..d89a1360 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -200,47 +200,49 @@ fullHistory: # ==================================================================================================== dbBackupRestore: image: - name: '' - tag: '' + name: 'developmentseed/osmseed-backup-restore' + tag: '0.1.0-0.dev.git.942.h994af82' cronjobs: - - name: web-db - enabled: true - schedule: '* * * * *' - env: - DB_ACTION: backup - SET_DATE: true - BACKUP_CLOUD_FOLDER: database/web-db - BACKUP_CLOUD_FILE: osmseed-api-web-db - AWS_S3_BUCKET: secure-bucket - resources: - enabled: false - requests: - memory: '300Mi' - cpu: '0.5' - limits: - memory: '400Mi' - cpu: '0.6' - nodeSelector: - enabled: false - - name: osmcha-db - enabled: true - schedule: '* * * * *' - env: - DB_ACTION: backup - SET_DATE: true - BACKUP_CLOUD_FOLDER: database/osmcha-db - BACKUP_CLOUD_FILE: osmseed-osmcha-db - AWS_S3_BUCKET: secure-bucket - resources: - enabled: false - requests: - memory: '300Mi' - cpu: '0.5' - limits: - memory: '400Mi' - cpu: '0.6' - nodeSelector: - enabled: false + - name: web-db + enabled: true + schedule: '* * * * *' + env: + DB_ACTION: backup + SET_DATE: true + BACKUP_CLOUD_FOLDER: database/web-db + BACKUP_CLOUD_FILE: osmseed-api-web-db + AWS_S3_BUCKET: secure-bucket + RETENTION_DAYS: "30" + resources: + enabled: false + requests: + memory: '300Mi' + cpu: '0.5' + limits: + memory: '400Mi' + cpu: '0.6' + nodeSelector: + enabled: false + - name: osmcha-db + enabled: true + schedule: '* * * * *' + env: + DB_ACTION: backup + SET_DATE: true + BACKUP_CLOUD_FOLDER: database/osmcha-db + BACKUP_CLOUD_FILE: osmseed-osmcha-db + AWS_S3_BUCKET: secure-bucket + RETENTION_DAYS: "30" + resources: + enabled: false + requests: + memory: '300Mi' + cpu: '0.5' + limits: + memory: '400Mi' + cpu: '0.6' + nodeSelector: + enabled: false # ==================================================================================================== # Variables for osm-seed for osmosis, this configuration os to get the planet dump files from apidb @@ -752,7 +754,8 @@ osmchaWeb: tag: '' env: OSM_SERVER_URL: 'https://staging.openstreetmap.org' - DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',\ + \ 'LOCATION': '127.0.0.1:11211'}}" DJANGO_DEBUG: "False" DJANGO_SECRET_KEY: abc DJANGO_SECURE_BROWSER_XSS_FILTER: "True"