diff --git a/Dockerfile b/Dockerfile index 822c9e2..b00d037 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,9 @@ -FROM debian:9.12 +FROM neo4j:4.0.4-enterprise -# Install core packages RUN apt-get update RUN apt-get install -y bash curl wget gnupg apt-transport-https apt-utils lsb-release unzip -# Install Neo4j -RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add - -RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee -a /etc/apt/sources.list.d/neo4j.list - -RUN echo "neo4j-enterprise neo4j/question select I ACCEPT" | debconf-set-selections -RUN echo "neo4j-enterprise neo4j/license note" | debconf-set-selections - -RUN apt-get update -RUN apt-get install -y neo4j-enterprise=1:3.5.8 +WORKDIR / # Install AWS CLI @@ -20,7 +11,7 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2 RUN unzip awscliv2.zip RUN ./aws/install -RUN mkdir /data +RUN mkdir /backup # Adding backup script ADD ./scripts/backup.sh /scripts/backup.sh diff --git a/README.md b/README.md index c34f2d7..67b255c 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,4 @@ Environment variables: - PAGE_CACHE (optional) = 1G - BACKUP_NAME (optional) = neo4j-backup -[Plural](https://plural.com) ❤️ Open source \ No newline at end of file +[Plural](https://plural.com) ❤️ Open source diff --git a/package.json b/package.json index 1c6ef20..ed3f844 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@pluralcom/neo4j-backup", - "version": "0.0.1", + "version": "0.1.0", "private": true, "description": "🐳 Docker container files that are used to backup Neo4j on EKS' Kubernetes", "repository": { diff --git a/scripts/backup.sh b/scripts/backup.sh index fb72b70..41db1d9 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -23,40 +23,40 @@ if [ -z $BACKUP_NAME ]; then fi BACKUP_SET="$BACKUP_NAME-$(date "+%Y-%m-%d")" +mkdir /backup/$BACKUP_SET echo "=============== Neo4j Backup ===============================" -echo "Beginning backup from $NEO4J_ADDR to /data/$BACKUP_SET" +echo "Beginning backup from $NEO4J_ADDR to /backup/$BACKUP_SET" echo "Using heap size $HEAP_SIZE and page cache $PAGE_CACHE" echo "To S3 bucket $S3_BUCKET_PATH" echo "============================================================" neo4j-admin backup \ --from="$NEO4J_ADDR" \ - --backup-dir=/data \ - --name="$BACKUP_SET" \ + --backup-dir=/backup/$BACKUP_SET \ --pagecache=$PAGE_CACHE -echo "ls /data" -ls -al /data +echo "ls /backup" +ls -al /backup -if [ -d "/data/$BACKUP_SET" ] ; then +if [ -d "/backup/$BACKUP_SET" ] ; then echo "Backup size:" - du -hs "/data/$BACKUP_SET" + du -hs "/backup/$BACKUP_SET" else echo "Backup file was not found. Terminating job." exit 1 fi -echo "Tarring -> /data/$BACKUP_SET.tar" -tar -cvf "/data/$BACKUP_SET.tar" "/data/$BACKUP_SET" --remove-files +echo "Tarring -> /backup/$BACKUP_SET.tar" +tar -cvf "/backup/$BACKUP_SET.tar" "/backup/$BACKUP_SET" --remove-files -echo "Zipping -> /data/$BACKUP_SET.tar.gz" -gzip -9 "/data/$BACKUP_SET.tar" +echo "Zipping -> /backup/$BACKUP_SET.tar.gz" +gzip -9 "/backup/$BACKUP_SET.tar" echo "Zipped backup size:" -du -hs "/data/$BACKUP_SET.tar.gz" +du -hs "/backup/$BACKUP_SET.tar.gz" -echo "Pushing /data/$BACKUP_SET.tar.gz -> $S3_BUCKET_PATH/$BACKUP_SET.tar.gz" -aws s3 cp "/data/$BACKUP_SET.tar.gz" "$S3_BUCKET_PATH/$BACKUP_SET.tar.gz" +echo "Pushing /backup/$BACKUP_SET.tar.gz -> $S3_BUCKET_PATH/$BACKUP_SET.tar.gz" +aws s3 cp "/backup/$BACKUP_SET.tar.gz" "$S3_BUCKET_PATH/$BACKUP_SET.tar.gz" exit $? \ No newline at end of file diff --git a/scripts/restore.sh b/scripts/restore.sh index 5642332..90e11ea 100644 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -31,8 +31,8 @@ echo "FORCE_OVERWRITE=$FORCE_OVERWRITE" ls /data/databases echo "============================================================" -if [ -d "/data/databases/graph.db" ] ; then - echo "You have an existing graph database at /data/databases/graph.db" +if [ -d "/data/databases/neo4j" ] ; then + echo "You have an existing neo4j database at /data/databases/neo4j" if [ "$FORCE_OVERWRITE" != "true" ] ; then echo "And you have not specified FORCE_OVERWRITE=true, so we will not restore because" @@ -40,7 +40,7 @@ if [ -d "/data/databases/graph.db" ] ; then exit 0; fi else - echo "No existing graph database found at /data/databases/graph.db" + echo "No existing neo4j database found at /data/databases/neo4j" fi # Pass the force flag to the restore operation, which will overwrite @@ -87,7 +87,7 @@ if [[ $BACKUP_FILENAME =~ \.tar\.gz$ ]] ; then if [ -z $BACKUP_SET_DIR ] ; then echo "BACKUP_SET_DIR was not specified, so I am assuming this backup set was formatted by my backup utility" - RESTORE_FROM="$RESTORE_ROOT/data/$UNTARRED_BACKUP_DIR" + RESTORE_FROM="$RESTORE_ROOT/backup/$UNTARRED_BACKUP_DIR" else RESTORE_FROM="$RESTORE_ROOT/$BACKUP_SET_DIR" fi @@ -105,7 +105,7 @@ elif [[ $BACKUP_FILENAME =~ \.zip$ ]] ; then if [ -z $BACKUP_SET_DIR ] ; then echo "BACKUP_SET_DIR was not specified, so I am assuming this backup set was formatted by my backup utility" - RESTORE_FROM="$RESTORE_ROOT/data/$UNZIPPED_BACKUP_DIR" + RESTORE_FROM="$RESTORE_ROOT/backup/$UNZIPPED_BACKUP_DIR" else RESTORE_FROM="$RESTORE_ROOT/$BACKUP_SET_DIR" fi @@ -125,11 +125,14 @@ echo "Post uncompress backup size:" ls -al "$RESTORE_ROOT" du -hs "$RESTORE_FROM" +echo "Unbinding from cluster" +neo4j-admin unbind + cd /data && \ echo "Dry-run command" echo neo4j-admin restore \ - --from="$RESTORE_FROM" \ - --database=graph.db $FORCE_FLAG + --from="$RESTORE_FROM/neo4j" \ + --database=neo4j $FORCE_FLAG # This data is output because of the way neo4j-admin works. It writes the restored set to # /var/lib/neo4j by default. This can fail if volumes aren't sized appropriately, so this @@ -139,47 +142,28 @@ df -h echo "Now restoring" neo4j-admin restore \ - --from="$RESTORE_FROM" \ - --database=graph.db $FORCE_FLAG + --from="$RESTORE_FROM/neo4j" \ + --database=neo4j $FORCE_FLAG RESTORE_EXIT_CODE=$? if [ "$RESTORE_EXIT_CODE" -ne 0 ]; then echo "Restore process failed; will not continue" exit $RESTORE_EXIT_CODE +else + echo "Restoration succeeded" fi -echo "Rehoming database" -echo "Restored to:" -ls -l /var/lib/neo4j/data/databases - -# neo4j-admin restore puts the DB in the wrong place, it needs to be re-homed -# for docker. -mkdir /data/databases - -# Danger: here we are destroying previous data. -# Optional: you can move the database out of the way to preserve the data just in case, -# but we don't do it this way because for large DBs this will just rapidly fill the disk -# and cause out of disk errors. -if [ -d "/data/databases/graph.db" ] ; then - if [ "$FORCE_OVERWRITE" = "true" ] ; then - echo "Removing previous database because FORCE_OVERWRITE=true" - rm -rf /data/databases/graph.db - fi -fi - -mv /var/lib/neo4j/data/databases/graph.db /data/databases/ - # Modify permissions/group, because we're running as root. # Neo4j user id is 101 on the official docker image chown -R 101 /data/databases -chgrp -R 101 /data/databases +chgrp -R 101 /data/transactions echo "Final permissions" -ls -al /data/databases/graph.db +ls -al /data/databases/neo4j echo "Final size" -du -hs /data/databases/graph.db +du -hs /data/databases/neo4j if [ "$PURGE_ON_COMPLETE" = true ] ; then echo "Purging backupset from disk"