⬆️ Upgrade to Neo4j 4.0.4 (#1)

* Initial commit

* Initial commit

* Modified S3 upload naming schema

* Modified the script print outs to match the actual data

* Migrated neo4j version to 4.0.4

* Modified the dockerfile

* Modified the docker file #2

* Fixed a bug in the backup script

* Cleaned up the Dockerfile

* Corrected some merge errors

* Modified restore file

* Modifying the restore command

* Fixing restore script

* Finalized restore script

* Bumped version number to 0.1.0

* Removed commented legacy code

Dockerfile

@@ -1,26 +1,17 @@
-FROM debian:9.12
+FROM neo4j:4.0.4-enterprise
 
-# Install core packages
 RUN apt-get update
 RUN apt-get install -y bash curl wget gnupg apt-transport-https apt-utils lsb-release unzip
 
-# Install Neo4j
-RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add -
-RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee -a /etc/apt/sources.list.d/neo4j.list
-
-RUN echo "neo4j-enterprise neo4j/question select I ACCEPT" | debconf-set-selections
-RUN echo "neo4j-enterprise neo4j/license note" | debconf-set-selections
-
-RUN apt-get update
-RUN apt-get install -y neo4j-enterprise=1:3.5.8
+WORKDIR /
 
 # Install AWS CLI
 
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
 RUN unzip awscliv2.zip
 RUN ./aws/install
 
-RUN mkdir /data
+RUN mkdir /backup
 
 # Adding backup script
 ADD ./scripts/backup.sh /scripts/backup.sh

README.md

@@ -13,4 +13,4 @@ Environment variables:
 - PAGE_CACHE (optional) = 1G
 - BACKUP_NAME (optional) = neo4j-backup
 
-[Plural](https://plural.com) ❤️ Open source
+[Plural](https://plural.com) ❤️ Open source

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pluralcom/neo4j-backup",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "private": true,
   "description": "🐳 Docker container files that are used to backup Neo4j on EKS' Kubernetes",
   "repository": {

scripts/backup.sh

@@ -23,40 +23,40 @@ if [ -z $BACKUP_NAME ]; then
 fi
 
 BACKUP_SET="$BACKUP_NAME-$(date "+%Y-%m-%d")"
+mkdir /backup/$BACKUP_SET
 
 echo "=============== Neo4j Backup ==============================="
-echo "Beginning backup from $NEO4J_ADDR to /data/$BACKUP_SET"
+echo "Beginning backup from $NEO4J_ADDR to /backup/$BACKUP_SET"
 echo "Using heap size $HEAP_SIZE and page cache $PAGE_CACHE"
 echo "To S3 bucket $S3_BUCKET_PATH"
 echo "============================================================"
 
 neo4j-admin backup \
     --from="$NEO4J_ADDR" \
-    --backup-dir=/data \
-    --name="$BACKUP_SET" \
+    --backup-dir=/backup/$BACKUP_SET \
     --pagecache=$PAGE_CACHE
 
-echo "ls /data"
-ls -al /data
+echo "ls /backup"
+ls -al /backup
 
-if [ -d "/data/$BACKUP_SET" ] ; then
+if [ -d "/backup/$BACKUP_SET" ] ; then
     echo "Backup size:"
-    du -hs "/data/$BACKUP_SET"
+    du -hs "/backup/$BACKUP_SET"
 else 
     echo "Backup file was not found. Terminating job."
     exit 1
 fi
 
-echo "Tarring -> /data/$BACKUP_SET.tar"
-tar -cvf "/data/$BACKUP_SET.tar" "/data/$BACKUP_SET" --remove-files
+echo "Tarring -> /backup/$BACKUP_SET.tar"
+tar -cvf "/backup/$BACKUP_SET.tar" "/backup/$BACKUP_SET" --remove-files
 
-echo "Zipping -> /data/$BACKUP_SET.tar.gz"
-gzip -9 "/data/$BACKUP_SET.tar"
+echo "Zipping -> /backup/$BACKUP_SET.tar.gz"
+gzip -9 "/backup/$BACKUP_SET.tar"
 
 echo "Zipped backup size:"
-du -hs "/data/$BACKUP_SET.tar.gz"
+du -hs "/backup/$BACKUP_SET.tar.gz"
 
-echo "Pushing /data/$BACKUP_SET.tar.gz -> $S3_BUCKET_PATH/$BACKUP_SET.tar.gz"
-aws s3 cp "/data/$BACKUP_SET.tar.gz" "$S3_BUCKET_PATH/$BACKUP_SET.tar.gz"
+echo "Pushing /backup/$BACKUP_SET.tar.gz -> $S3_BUCKET_PATH/$BACKUP_SET.tar.gz"
+aws s3 cp "/backup/$BACKUP_SET.tar.gz" "$S3_BUCKET_PATH/$BACKUP_SET.tar.gz"
 
 exit $?

scripts/restore.sh

@@ -31,16 +31,16 @@ echo "FORCE_OVERWRITE=$FORCE_OVERWRITE"
 ls /data/databases
 echo "============================================================"
 
-if [ -d "/data/databases/graph.db" ] ; then
-    echo "You have an existing graph database at /data/databases/graph.db"
+if [ -d "/data/databases/neo4j" ] ; then
+    echo "You have an existing neo4j database at /data/databases/neo4j"
 
     if [ "$FORCE_OVERWRITE" != "true" ] ; then
         echo "And you have not specified FORCE_OVERWRITE=true, so we will not restore because"
         echo "that would overwrite your existing data.   Exiting.".
         exit 0;
     fi
 else 
-    echo "No existing graph database found at /data/databases/graph.db"
+    echo "No existing neo4j database found at /data/databases/neo4j"
 fi
 
 # Pass the force flag to the restore operation, which will overwrite
@@ -87,7 +87,7 @@ if [[ $BACKUP_FILENAME =~ \.tar\.gz$ ]] ; then
 
     if [ -z $BACKUP_SET_DIR ] ; then
         echo "BACKUP_SET_DIR was not specified, so I am assuming this backup set was formatted by my backup utility"
-        RESTORE_FROM="$RESTORE_ROOT/data/$UNTARRED_BACKUP_DIR"
+        RESTORE_FROM="$RESTORE_ROOT/backup/$UNTARRED_BACKUP_DIR"
     else 
         RESTORE_FROM="$RESTORE_ROOT/$BACKUP_SET_DIR"
     fi
@@ -105,7 +105,7 @@ elif [[ $BACKUP_FILENAME =~ \.zip$ ]] ; then
 
     if [ -z $BACKUP_SET_DIR ] ; then
         echo "BACKUP_SET_DIR was not specified, so I am assuming this backup set was formatted by my backup utility"
-        RESTORE_FROM="$RESTORE_ROOT/data/$UNZIPPED_BACKUP_DIR"
+        RESTORE_FROM="$RESTORE_ROOT/backup/$UNZIPPED_BACKUP_DIR"
     else
         RESTORE_FROM="$RESTORE_ROOT/$BACKUP_SET_DIR"
     fi
@@ -125,11 +125,14 @@ echo "Post uncompress backup size:"
 ls -al "$RESTORE_ROOT"
 du -hs "$RESTORE_FROM"
 
+echo "Unbinding from cluster"
+neo4j-admin unbind
+
 cd /data && \
 echo "Dry-run command"
 echo neo4j-admin restore \
-    --from="$RESTORE_FROM" \
-    --database=graph.db $FORCE_FLAG
+    --from="$RESTORE_FROM/neo4j" \
+    --database=neo4j $FORCE_FLAG
 
 # This data is output because of the way neo4j-admin works.  It writes the restored set to
 # /var/lib/neo4j by default.  This can fail if volumes aren't sized appropriately, so this 
@@ -139,47 +142,28 @@ df -h
 
 echo "Now restoring"
 neo4j-admin restore \
-    --from="$RESTORE_FROM" \
-    --database=graph.db $FORCE_FLAG
+    --from="$RESTORE_FROM/neo4j" \
+    --database=neo4j $FORCE_FLAG
 
 RESTORE_EXIT_CODE=$?
 
 if [ "$RESTORE_EXIT_CODE" -ne 0 ]; then 
     echo "Restore process failed; will not continue"
     exit $RESTORE_EXIT_CODE
+else
+    echo "Restoration succeeded"
 fi
 
-echo "Rehoming database"
-echo "Restored to:"
-ls -l /var/lib/neo4j/data/databases
-
-# neo4j-admin restore puts the DB in the wrong place, it needs to be re-homed
-# for docker.
-mkdir /data/databases
-
-# Danger: here we are destroying previous data.
-# Optional: you can move the database out of the way to preserve the data just in case,
-# but we don't do it this way because for large DBs this will just rapidly fill the disk
-# and cause out of disk errors.
-if [ -d "/data/databases/graph.db" ] ; then
-   if [ "$FORCE_OVERWRITE" = "true" ] ; then
-      echo "Removing previous database because FORCE_OVERWRITE=true"
-      rm -rf /data/databases/graph.db
-   fi
-fi
-
-mv /var/lib/neo4j/data/databases/graph.db /data/databases/
-
 # Modify permissions/group, because we're running as root.
 # Neo4j user id is 101 on the official docker image
 chown -R 101 /data/databases
-chgrp -R 101 /data/databases
+chgrp -R 101 /data/transactions
 
 echo "Final permissions"
-ls -al /data/databases/graph.db
+ls -al /data/databases/neo4j
 
 echo "Final size"
-du -hs /data/databases/graph.db
+du -hs /data/databases/neo4j
 
 if [ "$PURGE_ON_COMPLETE" = true ] ; then
     echo "Purging backupset from disk"