From 198f878ed9b9562630a4f0a737f20f83a13cf43a Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 08:18:28 +0300 Subject: [PATCH 1/7] backup: only stop the stack for backup Using docker-compose down/up will recreate the containers, possibly using different images if a manual "docker-compose pull" had been executed. --- scripts/backup_restore/post_backup_complete.sh | 2 +- scripts/backup_restore/pre_backup_complete.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/backup_restore/post_backup_complete.sh b/scripts/backup_restore/post_backup_complete.sh index b6a867bb..5bd33796 100755 --- a/scripts/backup_restore/post_backup_complete.sh +++ b/scripts/backup_restore/post_backup_complete.sh @@ -7,4 +7,4 @@ if [ -f "./post_backup.sh" ]; then bash ./post_backup.sh fi -docker-compose up -d \ No newline at end of file +docker-compose start diff --git a/scripts/backup_restore/pre_backup_complete.sh b/scripts/backup_restore/pre_backup_complete.sh index a29c5f96..632ab313 100755 --- a/scripts/backup_restore/pre_backup_complete.sh +++ b/scripts/backup_restore/pre_backup_complete.sh @@ -2,7 +2,7 @@ # This script runs any prebackup commands you may need -docker-compose down +docker-compose stop if [ -f "./pre_backup.sh" ]; then echo "./pre_backup.sh file found, executing:" From de2c030f97dced459194310d5dfa984fdded481d Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 08:15:01 +0300 Subject: [PATCH 2/7] backup: refactor: inline for clarity --- scripts/backup.sh | 22 ++++++++++++++----- .../backup_restore/post_backup_complete.sh | 10 --------- .../backup_restore/post_restore_complete.sh | 8 ------- scripts/backup_restore/pre_backup_complete.sh | 10 --------- scripts/restore.sh | 9 +++++--- 5 files changed, 22 insertions(+), 37 deletions(-) delete mode 100755 scripts/backup_restore/post_backup_complete.sh delete mode 100755 scripts/backup_restore/post_restore_complete.sh delete mode 100755 scripts/backup_restore/pre_backup_complete.sh diff --git a/scripts/backup.sh b/scripts/backup.sh index 6856c1d7..7931b398 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -83,9 +83,14 @@ fi echo "" >> $BACKUPLIST -echo "" >> $LOGFILE -echo "Executing prebackup scripts" >> $LOGFILE -bash ./scripts/backup_restore/pre_backup_complete.sh >> $LOGFILE 2>&1 +echo "Stopping stack to get consistent database backups" >> $LOGFILE +docker-compose stop >> $LOGFILE + +if [ -f "./pre_backup.sh" ]; then + echo "" >> $LOGFILE + echo "./pre_backup.sh file found, executing:" >> $LOGFILE + bash ./pre_backup.sh >> $LOGFILE 2>&1 +fi echo "./services/" >> $BACKUPLIST echo "./volumes/" >> $BACKUPLIST @@ -121,9 +126,14 @@ fi echo "Backup Size (bytes): $(stat --printf="%s" $TMPBACKUPFILE)" >> $LOGFILE echo "" >> $LOGFILE -echo "Executing postbackup scripts" >> $LOGFILE -bash ./scripts/backup_restore/post_backup_complete.sh >> $LOGFILE 2>&1 -echo "" >> $LOGFILE +if [ -f "./post_backup.sh" ]; then + echo "./post_backup.sh file found, executing:" >> $LOGFILE + bash ./post_backup.sh 2>&1 >> $LOGFILE + echo "" > $LOGFILE +fi + +echo "Starting stack back up" >> $LOGFILE +docker-compose start echo "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE echo "" >> $LOGFILE diff --git a/scripts/backup_restore/post_backup_complete.sh b/scripts/backup_restore/post_backup_complete.sh deleted file mode 100755 index 5bd33796..00000000 --- a/scripts/backup_restore/post_backup_complete.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# This script runs any postbackup commands you may need - -if [ -f "./post_backup.sh" ]; then - echo "./post_backup.sh file found, executing:" - bash ./post_backup.sh -fi - -docker-compose start diff --git a/scripts/backup_restore/post_restore_complete.sh b/scripts/backup_restore/post_restore_complete.sh deleted file mode 100755 index a4dd815a..00000000 --- a/scripts/backup_restore/post_restore_complete.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# This script runs any post restore commands you may need - -if [ -f "./post_restore.sh" ]; then - echo "./post_restore.sh file found, executing:" - bash ./post_restore.sh -fi diff --git a/scripts/backup_restore/pre_backup_complete.sh b/scripts/backup_restore/pre_backup_complete.sh deleted file mode 100755 index 632ab313..00000000 --- a/scripts/backup_restore/pre_backup_complete.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# This script runs any prebackup commands you may need - -docker-compose stop - -if [ -f "./pre_backup.sh" ]; then - echo "./pre_backup.sh file found, executing:" - bash ./pre_backup.sh -fi diff --git a/scripts/restore.sh b/scripts/restore.sh index 02c9e8a9..a3b1e090 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -77,9 +77,12 @@ sudo tar -zxvf \ echo "" >> $LOGFILE -echo "Executing post restore scripts" >> $LOGFILE -bash ./scripts/backup_restore/post_restore_complete.sh >> $LOGFILE 2>&1 -echo "" > $LOGFILE + +if [ -f "./post_restore.sh" ]; then + echo "./post_restore.sh file found, executing:" >> $LOGFILE + bash ./post_restore.sh 2>&1 >> $LOGFILE + echo "" > $LOGFILE +fi echo "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE echo "" >> $LOGFILE From 5badc2f23c220edc1ed3deb8fd18cd8dc3027540 Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 08:55:29 +0300 Subject: [PATCH 3/7] backup: fix to chown to correct group Previous version only worked for "pi:pi" or similar, where user and group names are the same. --- scripts/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/backup.sh b/scripts/backup.sh index 7931b398..202313a1 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -106,7 +106,7 @@ sudo tar -czf $TMPBACKUPFILE -T $BACKUPLIST >> $LOGFILE 2>&1 [ -f "$ROLLING" ] && ROLLINGOVERWRITTEN=1 && rm -rf $ROLLING -sudo chown -R $USER:$USER $TMPDIR/backup* >> $LOGFILE 2>&1 +sudo chown -R $USER:$(id -g $USER) $TMPDIR/backup* >> $LOGFILE 2>&1 if [[ "$BACKUPTYPE" -eq "1" || "$BACKUPTYPE" -eq "3" ]]; then cp $TMPBACKUPFILE $BACKUPFILE From 7de8016faf16ab10a671feb40c6627b726c86f4a Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 22:26:39 +0300 Subject: [PATCH 4/7] backup: improve reliability and documentation * allow starting backup from any folder * automatically run using sudo (to never fail due to ./backup owned by root) * run post_backup.sh after 'docker-compose start', to minimize downtime when e.g. uploading backups to a remote server * chown all of ./backup to the appropriate user Fixes #566 --- docs/Backups.md.old | 113 --------------- docs/Basic_setup/Backup-and-Restore.md | 188 +++++++++++++++++++------ scripts/backup.sh | 47 ++++--- 3 files changed, 170 insertions(+), 178 deletions(-) delete mode 100644 docs/Backups.md.old diff --git a/docs/Backups.md.old b/docs/Backups.md.old deleted file mode 100644 index c220ed2e..00000000 --- a/docs/Backups.md.old +++ /dev/null @@ -1,113 +0,0 @@ -# Backups -Because containers can easily be rebuilt from docker hub we only have to back up the data in the "volumes" directory. - -## Cloud Backups -### Dropbox-Uploader -This a great utility to easily upload data from your Pi to the cloud. https://magpi.raspberrypi.org/articles/dropbox-raspberry-pi. It can be installed from the Menu under Backups. -### rclone (Google Drive) -This is a service to upload to Google Drive. The config is described [here]( https://medium.com/@artur.klauser/mounting-google-drive-on-raspberry-pi-f5002c7095c2). Install it from the menu then follow the link for these sections: -* Getting a Google Drive Client ID -* Setting up the Rclone Configuration - -When naming the service in `rclone config` ensure to call it "gdrive" - -**The Auto-mounting instructions for the drive in the link don't work on Rasbian**. Auto-mounting of the drive isn't necessary for the backup script. - -If you want your Google Drive to mount on every boot then follow the instructions at the bottom of the wiki page - - -## Influxdb -`~/IOTstack/scripts/backup_influxdb.sh` does a database snapshot and stores it in ~/IOTstack/backups/influxdb/db . This can be restored with the help a script (that I still need to write) - -## Docker backups -The script `~/IOTstack/scripts/docker_backup.sh` performs the master backup for the stack. - -This script can be placed in a cron job to backup on a schedule. -Edit the crontab with`crontab -e` -Then add `0 23 * * * ~/IOTstack/scripts/docker_backup.sh >/dev/null 2>&1` to have a backup every night at 23:00. - -This script cheats by copying the volume folder live. The correct way would be to stop the stack first then copy the volumes and restart. The cheating method shouldn't be a problem unless you have fast changing data like in influxdb. This is why the script makes a database export of influxdb and ignores its volume. - -### Cloud integration -The docker_backup.sh script now no longer requires modification to enable cloud backups. It now tests for the presence of and enable file in the backups folder -#### Drobox-Uploader -The backup tests for a file called `~/IOTstack/backups/dropbox`, if it is present it will upload to dropbox. To disable dropbox upload delete the file. To enable run `sudo touch ~/IOTstack/backups/dropbox` -#### rclone -The backup tests for a file called `~/IOTstack/backups/rclone`, if it is present it will upload to google drive. To disable rclone upload delete the file. To enable run `sudo touch ~/IOTstack/backups/rclone` - -#### Pruning online backups -@877dev has added functionality to prune both local and cloud backups. For dropbox make sure you dont have any files that contain spaces in your backup directory as the script cannot handle it at this time. - -### Restoring a backup -The "volumes" directory contains all the persistent data necessary to recreate the container. The docker-compose.yml and the environment files are optional as they can be regenerated with the menu. Simply copy the volumes directory into the IOTstack directory, Rebuild the stack and start. - -## Added your Dropbox token incorrectly or aborted the install at the token screen - -Make sure you are running the latest version of the project [link](https://sensorsiot.github.io/IOTstack/Updating-the-Project/). - -Run `~/Dropbox-Uploader/dropbox_uploader.sh unlink` and if you have added it key then it will prompt you to confirm its removal. If no key was found it will ask you for a new key. - -Confirm by running `~/Dropbox-Uploader/dropbox_uploader.sh` it should ask you for your key if you removed it or show you the following prompt if it has the key: - -``` console - $ ~/Dropbox-Uploader/dropbox_uploader.sh -Dropbox Uploader v1.0 -Andrea Fabrizi - andrea.fabrizi@gmail.com - -Usage: /home/pi/Dropbox-Uploader/dropbox_uploader.sh [PARAMETERS] COMMAND... - -Commands: - upload - download [LOCAL_FILE/DIR] - delete - move - copy - mkdir -.... - -``` - -Ensure you **are not** running as sudo as this will store your api in the /root directory as `/root/.dropbox_uploader` - -If you ran the command with sudo the remove the old token file if it exists with either `sudo rm /root/.dropbox_uploader` or `sudo ~/Dropbox-Uploader/dropbox_uploader.sh unlink` - -## Auto-mount Gdrive with rclone - -To enable rclone to mount on boot you will need to make a user service. Run the following commands - -``` console -$ mkdir -p ~/.config/systemd/user -$ nano ~/.config/systemd/user/gdrive.service -``` -Copy the following code into the editor, save and exit - -``` -[Unit] -Description=rclone: Remote FUSE filesystem for cloud storage -Documentation=man:rclone(1) - -[Service] -Type=notify -ExecStartPre=/bin/mkdir -p %h/mnt/gdrive -ExecStart= \ - /usr/bin/rclone mount \ - --fast-list \ - --vfs-cache-mode writes \ - gdrive: %h/mnt/gdrive - -[Install] -WantedBy=default.target -``` -enable it to start on boot with: (no sudo) -``` console -$ systemctl --user enable gdrive.service -``` -start with -``` console -$ systemctl --user start gdrive.service -``` -if you no longer want it to start on boot then type: -``` console -$ systemctl --user disable gdrive.service -``` - diff --git a/docs/Basic_setup/Backup-and-Restore.md b/docs/Basic_setup/Backup-and-Restore.md index 48de688a..c7bcf8da 100644 --- a/docs/Basic_setup/Backup-and-Restore.md +++ b/docs/Basic_setup/Backup-and-Restore.md @@ -4,55 +4,64 @@ This page explains how to use the backup and restore functionality of IOTstack. ## Backup The backup command can be executed from IOTstack's menu, or from a cronjob. -### Running backup -To ensure that all your data is saved correctly, the stack should be brought down. This is mainly due to databases potentially being in a state that could cause data loss. +To ensure that all your data is saved correctly, the stack has to be stopped. +This is mainly due to databases potentially being in a state that could cause +data loss. There are 2 ways to run backups: * From the menu: `Backup and Restore` > `Run backup` * Running the following command: `bash ./scripts/backup.sh` -The command that's run from the command line can also be executed from a cronjob: +The command that's run from the command line can also be executed from crontab: -```0 2 * * * cd /home/pi/IOTstack && /bin/bash ./scripts/backup.sh``` +``` +0 2 * * * cd /home/pi/IOTstack && /bin/bash ./scripts/backup.sh +``` The current directory of bash must be in IOTstack's directory, to ensure that it can find the relative paths of the files it's meant to back up. In the example above, it's assume that it's inside the `pi` user's home directory. -### Arguments -``` -./scripts/backup.sh {TYPE=3} {USER=$(whoami)} +**Usage:** +``` console +$ ~/IOTstack/scripts/backup.sh {TYPE} ``` -* Types: - * 1 = Backup with Date - * A tarball file will be created that contains the date and time the backup was started, in the filename. - * 2 = Rolling Date - * A tarball file will be created that contains the day of the week (0-6) the backup was started, in the filename. - * If a tarball already exists with the same name, it will be overwritten. - * 3 = Both -* User: - This parameter only becomes active if run as root. This script will default to the current logged in user - If this parameter is not supplied when run as root, the script will ask for the username as input +**Types:** + +1. Backup with Date. Backup to filename with the date and time the backup was + started. +2. Rolling Date. Filename will based on the day of the week (0-6) the backup + was started. New backups will overwrite old ones. +3. Both + +You can find the backups in the `~/IOTstack/backups/` folder. With rolling +being in `~/IOTstack/backups/rolling/` and date backups in +`~/IOTstack/backups/backup/`. Log files can also be found in the logs/ +directory. -Backups: +**Examples:** - * You can find the backups in the ./backups/ folder. With rolling being in ./backups/rolling/ and date backups in ./backups/backup/ - * Log files can also be found in the ./backups/logs/ directory. +- Regular date & timestamped backup into ~/IOTstack/backup/backups: -### Examples: + `~/IOTstack/scripts/backup.sh 1` - * `./scripts/backup.sh` - * `./scripts/backup.sh 3` +- Either these commands will produce backups of both types: -Either of these will run both backups. + `~/IOTstack/scripts/backup.sh` or
+ `~/IOTstack/scripts/backup.sh 3` - * `./scripts/backup.sh 2` -This will only produce a backup in the rollowing folder. It will be called 'backup_XX.tar.gz' where XX is the current day of the week (as an int) +- Produce a backup into ~/IOTstack/backup/rolling/. It will be called + 'backup_XX.tar.gz' where XX is the current day of the week (as an int): - * `sudo bash ./scripts/backup.sh 2 pi` + `~/IOTstack/scripts/backup.sh 2` -This will only produce a backup in the rollowing folder and change all the permissions to the 'pi' user. +- (expert use) Usually the backup should be executed without sudo using your + regular user, as this will automatically produce backups with correct + permissions. This will only produce a backup in the rolling folder and + change all the permissions to the 'pi' user: + + `sudo bash ~/IOTstack/scripts/backup.sh 2 pi` ## Restore There are 2 ways to run a restore: @@ -64,7 +73,7 @@ There are 2 ways to run a restore: *Note*: It is suggested that you test that your backups can be restored after initially setting up, and anytime you add or remove a service. Major updates to services can also break backups. -### Arguments +**Usage:** ``` ./scripts/restore.sh {FILENAME=backup.tar.gz} {noask} ``` @@ -76,15 +85,24 @@ The restore script takes 2 arguments: ## Pre and post script hooks The script checks if there are any pre and post back up hooks to execute commands. Both of these files will be included in the backup, and have also been added to the `.gitignore` file, so that they will not be touched when IOTstack updates. -### Prebackup script hook -The prebackup hook script is executed before any compression happens and before anything is written to the temporary backup manifest file (`./.tmp/backup-list_{{NAME}}.txt`). It can be used to prepare any services (such as databases that IOTstack isn't aware of) for backing up. +Both of these scripts will be run as root. + +### Pre-backup script hook +The pre-backup hook script is executed before any compression happens and before anything is written to the temporary backup manifest file (`./.tmp/backup-list_{{NAME}}.txt`). It can be used to prepare any services (such as databases that IOTstack isn't aware of) for backing up. -To use it, simple create a `./pre_backup.sh` file in IOTstack's main directory. It will be executed next time a backup runs. +To use it, simple create the `~/IOTstack/pre_backup.sh` file. It will be executed next time a backup runs. -### Postbackup script hook -The postbackup hook script is executed after the tarball file has been written to disk, and before the final backup log information is written to disk. +### Post-backup script hook { #post-backup } +The post-backup hook script is executed after the backup tarball file has been +written to disk and the stack has been started back up. Any output will be +included in the backup log file. -To use it, simple create a `./post_backup.sh` file in IOTstack's main directory. It will be executed after the next time a backup runs. +To use it, simple create the `~/IOTstack/post_backup.sh` file. It will be +executed after the next time a backup runs. It will be provided the backup +.tar.gz as its first argument. + +This is useful for triggering transfer of the backup to a cloud or another +server, see below for possible third party integrations as to what is possible. ### Post restore script hook The post restore hook script is executed after all files have been extracted and written to disk. It can be used to apply permissions that your custom services may require. @@ -92,19 +110,101 @@ The post restore hook script is executed after all files have been extracted and To use it, simple create a `./post_restore.sh` file in IOTstack's main directory. It will be executed after a restore happens. ## Third party integration -This section explains how to backup your files with 3rd party software. +This section explains how you could backup your files to different +integrations. Actual initiation of the transfer of the backup has to be done +using the [post backup](#post-backup) hook described above. + +### IOTstackBackup + +[IOTstackBackup](https://github.com/Paraphraser/IOTstackBackup) is a project +aiming to provide a sophisticated all-in-on solution: on-line backups of +databases and configurable transfers to remotes without writing your own +scripts. ### Dropbox -Coming soon. +Dropbox-Uploader is a great utility to easily upload data from your Pi to the cloud. https://magpi.raspberrypi.org/articles/dropbox-raspberry-pi. It can be installed from the Menu under Backups. + +#### Troubleshoot: Token added incorrectly or install aborted at the token screen + +Run `~/Dropbox-Uploader/dropbox_uploader.sh unlink` and if you have added it key then it will prompt you to confirm its removal. If no key was found it will ask you for a new key. + +Confirm by running `~/Dropbox-Uploader/dropbox_uploader.sh` it should ask you for your key if you removed it or show you the following prompt if it has the key: + +``` console + $ ~/Dropbox-Uploader/dropbox_uploader.sh +Dropbox Uploader v1.0 +Andrea Fabrizi - andrea.fabrizi@gmail.com + +Usage: /home/pi/Dropbox-Uploader/dropbox_uploader.sh [PARAMETERS] COMMAND... + +Commands: + upload + download [LOCAL_FILE/DIR] + delete + move + copy + mkdir +.... + +``` + +Ensure you **are not** running as sudo as this will store your api in the /root directory as `/root/.dropbox_uploader` + +If you ran the command with sudo the remove the old token file if it exists with either `sudo rm /root/.dropbox_uploader` or `sudo ~/Dropbox-Uploader/dropbox_uploader.sh unlink` + + ### Google Drive -Coming soon. +rclone is a program uploading to Google Drive. Install it from the menu then +see [here]( +https://medium.com/@artur.klauser/mounting-google-drive-on-raspberry-pi-f5002c7095c2) +for these sections: + +* Getting a Google Drive Client ID +* Setting up the Rclone Configuration + +Note: When naming the service in `rclone config` ensure to call it "gdrive" + +**The Auto-mounting instructions for the drive in the link don't work on Rasbian**. Auto-mounting of the drive isn't necessary for the backup script. -### rsync -Coming soon. +If you want your Google Drive to mount on every boot then follow the instructions at the bottom of the wiki page -### Duplicati -Coming soon. +#### Auto-mounting Google Drive -### SFTP -Coming soon. +To enable rclone to mount on boot you will need to make a user service. Run the following commands + +``` console +$ mkdir -p ~/.config/systemd/user +$ nano ~/.config/systemd/user/gdrive.service +``` +Copy the following code into the editor, save and exit + +``` +[Unit] +Description=rclone: Remote FUSE filesystem for cloud storage +Documentation=man:rclone(1) + +[Service] +Type=notify +ExecStartPre=/bin/mkdir -p %h/mnt/gdrive +ExecStart= \ + /usr/bin/rclone mount \ + --fast-list \ + --vfs-cache-mode writes \ + gdrive: %h/mnt/gdrive + +[Install] +WantedBy=default.target +``` +enable it to start on boot with: (no sudo) +``` console +$ systemctl --user enable gdrive.service +``` +start with +``` console +$ systemctl --user start gdrive.service +``` +if you no longer want it to start on boot then type: +``` console +$ systemctl --user disable gdrive.service +``` diff --git a/scripts/backup.sh b/scripts/backup.sh index 202313a1..56bb6528 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -1,34 +1,35 @@ #!/bin/bash +# vim: sw=2 # Usage: -# ./scripts/backup.sh {TYPE=3} {USER=$(whoami)} +# ./scripts/backup.sh {TYPE} # Types: # 1 = Backup with Date # 2 = Rolling Date # 3 = Both -# User: -# This parameter only becomes active if run as root. This script will default to the current logged in user -# If this parameter is not supplied when run as root, the script will ask for the username as input # # Backups: -# You can find the backups in the ./backups/ folder. With rolling being in ./backups/rolling/ and date backups in ./backups/backup/ -# Log files can also be found in the ./backups/logs/ directory. +# You can find the backups in the ./backups/ folder. With rolling being in +# ./backups/rolling/ and date backups in ./backups/backup/ Log files can +# also be found in the ./backups/logs/ directory. # # Examples: # ./scripts/backup.sh # ./scripts/backup.sh 3 -# Either of these will run both backups. +# Both of these will run both backups. # # ./scripts/backup.sh 2 -# This will only produce a backup in the rollowing folder. It will be called 'backup_XX.tar.gz' where XX is the current day of the week (as an int) +# This will only produce a backup into ~/IOTstack/backup/rolling/. It will +# be called 'backup_XX.tar.gz' where XX is the current day of the week (as +# an int) # # sudo bash ./scripts/backup.sh 2 pi -# This will only produce a backup in the rollowing folder and change all the permissions to the 'pi' user. +# This will only produce a backup in the rolling folder and change all +# the permissions to the 'pi' user. This is for expert use, usually this +# script should be executed without sudo using your regular user. -if [ -d "./menu.sh" ]; then - echo "./menu.sh file was not found. Ensure that you are running this from IOTstack's directory." - exit 1 -fi +# Allow running from everywhere, but change folder to script's IOTstack +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." BACKUPTYPE=${1:-"3"} @@ -39,13 +40,15 @@ fi if [[ "$EUID" -eq 0 ]]; then if [ -z ${2+x} ]; then - echo "Enter username to chown (change ownership) files to" - read USER; + echo "Error: this script shouldn't be run as root" + exit 1 else USER=$2 fi else - USER=$(whoami) + # re-run script using sudo, to avoid any permission problems + sudo ${BASH_SOURCE[0]} $BACKUPTYPE ${2:-$(whoami)} + exit $? fi BASEDIR=./backups @@ -126,15 +129,15 @@ fi echo "Backup Size (bytes): $(stat --printf="%s" $TMPBACKUPFILE)" >> $LOGFILE echo "" >> $LOGFILE +echo "Starting stack back up" >> $LOGFILE +docker-compose start + if [ -f "./post_backup.sh" ]; then - echo "./post_backup.sh file found, executing:" >> $LOGFILE - bash ./post_backup.sh 2>&1 >> $LOGFILE + echo "./post_backup.sh file found, executing it at $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE + bash ./post_backup.sh $TMPBACKUPFILE 2>&1 >> $LOGFILE echo "" > $LOGFILE fi -echo "Starting stack back up" >> $LOGFILE -docker-compose start - echo "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE echo "" >> $LOGFILE @@ -152,6 +155,8 @@ else echo " $BACKUPLIST" fi +sudo chown -R $USER:$(id -g $USER) "$BASEDIR" >> $LOGFILE 2>&1 + echo "" >> $LOGFILE echo "### End of log ###" >> $LOGFILE echo "" >> $LOGFILE From 89d7cf0db158be6a3a8509f8901478ebb8870919 Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 23:30:07 +0300 Subject: [PATCH 5/7] restore.sh: refactor: cleanup & simplify --- scripts/restore.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/restore.sh b/scripts/restore.sh index a3b1e090..78208446 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -27,17 +27,12 @@ if [[ ! $PROCEED_WITH_RESTORE =~ ^[Yy]$ ]]; then exit 0 fi -RESTOREFILENAME="backup.tar.gz" -RESTOREFILENAME=${1:-$RESTOREFILENAME} +RESTOREFILENAME=${1:-"backup.tar.gz"} BASEDIR=./backups -TMPDIR=./.tmp -BASERESTOREFILE="$(date +"%Y-%m-%d_%H%M")" RESTOREFILE="$BASEDIR/$RESTOREFILENAME" -LOGFILE="$BASEDIR/logs/restore_$BASERESTOREFILE.log" -BACKUPFILE="$BASEDIR/backup/backup_$BASERESTOREFILE.tar.gz" +LOGFILE="$BASEDIR/logs/restore_$(date +"%Y-%m-%d_%H%M").log" -[ -d ./backups ] || mkdir -p ./backups [ -d ./backups/logs ] || mkdir -p ./backups/logs [ -d ./.tmp ] || sudo rm -rf ./.tmp @@ -70,7 +65,6 @@ sudo rm -rf ./postbuild.sh >> $LOGFILE 2>&1 sudo rm -rf ./pre_backup.sh >> $LOGFILE 2>&1 sudo rm -rf ./post_backup.sh >> $LOGFILE 2>&1 sudo rm -rf ./post_restore.sh >> $LOGFILE 2>&1 -sudo rm -rf ./post_restore.sh >> $LOGFILE 2>&1 sudo tar -zxvf \ $RESTOREFILE >> $LOGFILE 2>&1 From eb1dcbe4bc2b13b7c0eace5805fe856dd4522562 Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Mon, 9 May 2022 23:41:30 +0300 Subject: [PATCH 6/7] restore.sh: ensure run as root * allow start from any current folder * chmod logfile to allow all to read it --- scripts/restore.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/restore.sh b/scripts/restore.sh index 78208446..8bedc807 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -1,4 +1,5 @@ #!/bin/bash +# vim: sw=2 # Usage: # ./scripts/restore.sh [FILENAME=backup.tar.gz] {noask} @@ -14,11 +15,15 @@ # Will restore from the backup file "./backups/some_other_backup.tar.gz" and will not warn that data will be deleted. # -if [ -d "./menu.sh" ]; then - echo "./menu.sh file was not found. Ensure that you are running this from IOTstack's directory." - exit 1 +if [[ "$EUID" != 0 ]]; then + # re-run as root in order to be able to restore owners correctly + sudo ${BASH_SOURCE[0]} "$@" + exit $? fi +# Allow running from everywhere, but change folder to script's IOTstack +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." + echo "Restoring from a backup will erase all existing data." read -p "Continue [y/N]? " -n 1 -r PROCEED_WITH_RESTORE echo "" @@ -85,4 +90,6 @@ echo "" >> $LOGFILE echo "### End of log ###" >> $LOGFILE echo "" >> $LOGFILE +chmod a+r $LOGFILE + cat $LOGFILE From 922464c0d373c5b845826b6b01ce38e809033bcd Mon Sep 17 00:00:00 2001 From: ukkopahis <95980324+ukkopahis@users.noreply.github.com> Date: Sat, 25 Jun 2022 00:31:19 +0300 Subject: [PATCH 7/7] backup: include .env and docker-compose.override.yml --- scripts/backup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/backup.sh b/scripts/backup.sh index 56bb6528..3fa053c0 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -98,6 +98,8 @@ fi echo "./services/" >> $BACKUPLIST echo "./volumes/" >> $BACKUPLIST [ -f "./docker-compose.yml" ] && echo "./docker-compose.yml" >> $BACKUPLIST +[ -f "./docker-compose.override.yml" ] && echo "./docker-compose.override.yml" >> $BACKUPLIST +[ -f "./.env" ] && echo "./.env" >> $BACKUPLIST [ -f "./compose-override.yml" ] && echo "./compose-override.yml" >> $BACKUPLIST [ -f "./extra" ] && echo "./extra" >> $BACKUPLIST [ -f "./.tmp/databases_backup" ] && echo "./.tmp/databases_backup" >> $BACKUPLIST