Merge pull request #144 from reef-technologies/streaming_b2_backup

stream db backups directly to B2

{{cookiecutter.repostory_name}}/bin/backup-db.sh

@@ -1,46 +1,29 @@
 #!/bin/bash -eu
-
 set -o pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source "${SCRIPT_DIR}/common.sh"
 
-# Update PATH in case docker-compose is installed via PIP
-# and this script was invoked from e.g. cron
-PATH=/usr/local/sbin:/usr/local/bin:$PATH
-
-if [ "$(basename "$0")" == 'bin' ]; then
-  cd ..
-fi
-
-. .env
+check_env_vars DATABASE_URL
 
-if [ -n "${SENTRY_DSN}" ]; then
-  export SENTRY_DSN
-  eval "$(sentry-cli bash-hook)"
-fi
-
-mkdir -p .backups
-TARGET=".backups/db_dump_$(date +%Y-%m-%d_%H%M%S).sql.gz"
+TARGET_FILENAME="db_dump_$(date +%Y-%m-%d_%H%M%S).sql.gz"
+DOCKER_NETWORK=$(get_db_docker_network)
 
-if [[ "$DATABASE_URL" =~ "@db:" ]]; then
-  DOCKER_NETWORK={{cookiecutter.repostory_name}}_default
-else
-  DOCKER_NETWORK=host
-fi
-
-docker run --rm --network $DOCKER_NETWORK postgres:14.0-alpine pg_dump -Z 9 -c --if-exists "$DATABASE_URL" > "$TARGET"
-echo "${TARGET}"
+DUMP_DB_TO_STDOUT="docker run --rm --network $DOCKER_NETWORK postgres:14.0-alpine pg_dump -Z 9 -c --if-exists $DATABASE_URL"
 
 if [ -n "${BACKUP_B2_BUCKET}" ]; then
-  bin/backup-db-to-b2.sh "${TARGET}"
-fi
-
-if [ -n "${EMAIL_HOST:-}" ] && [ -n "${EMAIL_TARGET:-}" ]; then
-  bin/backup-db-to-email.sh "${TARGET}"
+  $DUMP_DB_TO_STDOUT | bin/backup-file-to-b2.sh - "${TARGET_FILENAME}"
+else
+  LOCAL_BACKUP_DIR=".backups"
+  mkdir -p "$LOCAL_BACKUP_DIR"
+  TARGET="$LOCAL_BACKUP_DIR/$TARGET_FILENAME"
+  $DUMP_DB_TO_STDOUT > "$TARGET"
+
+  if [ -n "${EMAIL_HOST:-}" ] && [ -n "${EMAIL_TARGET:-}" ]; then
+    ${SCRIPT_DIR}/backup-db-to-email.sh "${TARGET}"
+  fi
 fi
 
-
 if [ -n "${BACKUP_LOCAL_ROTATE_KEEP_LAST:-}" ]; then
   echo "Rotating backup files - keeping ${BACKUP_LOCAL_ROTATE_KEEP_LAST} last ones"
   bin/rotate-local-backups.py "${BACKUP_LOCAL_ROTATE_KEEP_LAST}"
 fi
-
-

{{cookiecutter.repostory_name}}/bin/backup-file-to-b2.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -eu
+set -o pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source "${SCRIPT_DIR}/common.sh"
+
+check_env_vars BACKUP_B2_KEY_ID BACKUP_B2_KEY_SECRET BACKUP_B2_BUCKET
+
+if [ "$1" == "-" ]; then
+  BACKUP_B2_FILENAME="$2"
+  [ -n "$BACKUP_B2_FILENAME" ] || (echo "Pass backup file name as the second argument if stdin was provided as data source">&2; exit 2)
+elif [ ! -f "$1" ]; then
+  echo "Pass existing backup file name (with .backups/ directory name) as the first argument"
+  exit 2
+else
+  BACKUP_B2_FILENAME"$(basename "$1")"
+fi
+
+docker run --rm -i -e B2_APPLICATION_KEY="$BACKUP_B2_KEY_SECRET" -e B2_APPLICATION_KEY_ID="$BACKUP_B2_KEY_ID" \
+  backblazeit/b2:3.13.0 upload-file "$BACKUP_B2_BUCKET" "$1" "$BACKUP_B2_FILENAME"

{{cookiecutter.repostory_name}}/bin/common.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -eu
+set -o pipefail
+
+if [ -z "${_COMMON_SH_LOADED:-}" ]; then
+  # Update PATH in case docker-compose is installed via PIP
+  # and this script was invoked from e.g. cron
+  PATH=/usr/local/sbin:/usr/local/bin:$PATH
+
+  check_env_vars() {
+      local required_vars=("$@")
+      local missing_vars=""
+      for var in "${required_vars[@]}"; do
+          if [ -z "${!var}" ]; then
+              missing_vars+="$var "
+          fi
+      done
+
+      if [ -n "$missing_vars" ]; then
+          echo "Error: The following required environment variables are missing: $missing_vars" >&2
+          exit 2
+      fi
+  }
+
+
+  load_project_env() {
+      if [ "$(basename "$0")" == 'bin' ]; then
+        cd ..
+      fi
+
+      . .env
+  }
+
+  get_db_docker_network() {
+    if [[ "$DATABASE_URL" =~ "@db:" ]]; then
+      echo {{cookiecutter.repostory_name}}_default
+    else
+      echo host
+    fi
+  }
+
+  load_project_env
+
+  if [ -n "${SENTRY_DSN}" ]; then
+    export SENTRY_DSN
+    eval "$(sentry-cli bash-hook)"
+  fi
+
+  _COMMON_SH_LOADED=true
+fi

{{cookiecutter.repostory_name}}/bin/restore-db-from-b2.sh

@@ -1,17 +1,15 @@
 #!/bin/bash -eu
+set -o pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source "${SCRIPT_DIR}/common.sh"
 
-if [ "$(basename "$0")" == 'bin' ]; then
-  cd ..
-fi
-
-if [[ $# -ne 3 ]]; then
-    echo "Usage: bin/restore-db-from-b2.sh <B2_KEY_ID> <B2_KEY_SECRET> <B2_FILE_ID>"
+if [[ $# -ne 1 ]]; then
+    echo "Usage: bin/restore-db-from-b2.sh <B2_FILE_ID>"
     echo "All arguments are required"
     exit 2
 fi
 
-FILENAME=.backups/restore_from_b2.sql.gz
+B2_FILE_ID="$1"
 
-b2 authorize-account "$1" "$2"
-b2 download-file-by-id "$3" "$FILENAME"
-bin/restore-db.sh "$FILENAME"
+docker run --rm -i -e B2_APPLICATION_KEY="$BACKUP_B2_KEY_SECRET" -e B2_APPLICATION_KEY_ID="$BACKUP_B2_KEY_ID" \
+  backblazeit/b2:3.13.0 cat "b2id://$1" | ${SCRIPT_DIR}/restore-db.sh -

{{cookiecutter.repostory_name}}/bin/restore-db.sh

@@ -1,20 +1,11 @@
-#!/bin/bash -eux
-
+#!/bin/bash -eu
 set -o pipefail
-
-if [ "$(basename "$0")" == 'bin' ]; then
-  cd ..
-fi
-
-. .env
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source "${SCRIPT_DIR}/common.sh"
 
 target="$1"
 
-if [[ "$DATABASE_URL" =~ "@db:" ]]; then
-  DOCKER_NETWORK={{cookiecutter.repostory_name}}_default
-else
-  DOCKER_NETWORK=host
-fi
+DOCKER_NETWORK=$(get_db_docker_network)
 
 zcat "$target" | docker run -i --rm --network $DOCKER_NETWORK postgres:14.0-alpine psql "$DATABASE_URL"