Merge pull request #21 from match4everyone/easier-deploy

Optimize docker deployment and reduce error surfaces

.env.example

@@ -0,0 +1,3 @@
+# Use backend_user to specify a user to run the backend container as.
+# If none is specified root will be used
+BACKEND_USER=user:group

.gitignore

@@ -1,6 +1,7 @@
 # VS Code
 .vscode
 .venv
+*.code-workspace
 
 # Production environment variables
 .env
@@ -193,9 +194,6 @@ psd
 thumb
 sketch
 
-# Ignore local docker-compose file, in case you make a copy for local changes
-docker-compose.yml
-
 # End of https://www.gitignore.io/api/react,django,python
 
 # pycharm settings

.travis.yml

@@ -3,18 +3,16 @@ services:
 
 before_script:
  - ./scripts/write_envs_to_file.sh
- - ./deploy.sh
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --build -d
  - ./scripts/wait_for_backend.sh
- - docker logs $(docker ps --format '{{.Names}}' | grep backend)
- - docker logs $(docker ps --format '{{.Names}}' | grep database)
-
 
 script:
  - bash scripts/check_website_availability.sh
- - docker exec backend python3 manage.py test
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml exec backend python3 manage.py test
 
 after_script:
- - docker-compose down
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml logs
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml down
 
 env:
  - POSTGRES_DB=match4everyone POSTGRES_PASSWORD="BuZK@HPB_FDpGx3gvnnB9@eypozpC8PeesGP7PUC*DDgbbj-Zpv3kRwCkmU6FQoL" POSTGRES_USER=match4everyone SECRET_KEY="QP=14DkrovY9qrqXmZk3sjgakmy@x#6Log7EiW-9K4g5W&xY)_yN3a*rsinLYP|"

README.md

@@ -7,51 +7,56 @@ Originally developed as [match4healthcare](https://github.com/match4everyone/mat
 
 ## Quick install
 - Copy `backend.prod.env.example` and `database.prod.env.example` to `backend.prod.env` and `database.prod.env` and fill in appropriate values
-- Run `./deploy.sh` (uses Docker) and visit `http://localhost:8000/`
+- Run `docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build` (uses Docker) and visit `http://localhost:8000/`
 
+## Environment variables
+
+TODO
 
 ## Install by hand
 ### Development
 - Build images and run containers
-`docker-compose -f docker-compose.dev.yml up --build`
+`docker-compose up --build`
 - Start previously built containers in background
 `docker-compose start`
-- Apply migrations
-`docker exec backend python3 manage.py migrate`
-- Collect static files
-`docker exec backend python3 manage.py collectstatic`
-- Load test data:
-`docker exec backend python3 manage.py loaddata fixture.json`
+- Load test data (file needs to exist inside backend folder):
+`docker-compose exec backend django-admin loaddata fixture.json`
 
 File changes in python files trigger an auto-reload of the server.
-Migrations have to be executed with `docker exec backend python3 manage.py migrate`.
+Migrations are automatically executed when the container starts.
 
-After changes to the Docker configuration, you have to restart and build the containers with `docker-compose -f docker-compose.dev.yml up --build`.
+After changes to the Docker configuration, you have to restart and build the containers with `docker-compose up --build`.
 
 ### Production
-Set `SECRET_KEY`, `SENDGRID_API_KEY` and `MAPBOX_TOKEN`in `backend.prod.env` for Django
-`POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`  inside `database.prod.env` for postgres on your host machine.
-Also add a `SLACK_LOG_WEBHOOK` to enable slack logging.
 
-To run a container in production and in a new environment execute the `deploy.sh` script which builds the containers, runs all configurations and starts the web service.
+## Reverse Proxy
+
+We recommend running the gunicorn server behind a reverse proxy to provide ssl and possibly run multiple services on one server.
+The default configuration will make the docker container reachable on port 8000 only on 127.0.0.1.
+
+A sample nginx configuration can be found at ./tools/nginx-sample-site.
+
+## Setup
+Copy `backend.prod.env.example` to `backend.prod.env` and set variables as documented in the example file for Django
+Copy `database.prod.env.example` to `database.prod.env` and set variables as documented in the example file for postgres on your host machine.
+
+To run a container in production and in a new environment execute the `docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build` script which builds the containers, runs all configurations and starts the web service.
 
 If you want to deploy manually follow these steps closly:
 
-1. Build the containers
-(Run `export CURRENT_UID=$(id -u):$(id -g)` if you want to run the backend as non-root)
-`docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml up -d --build`
-2. Make messages
-`docker exec --env PYTHONPATH="/match4everyone-backend:$PYTHONPATH" backend django-admin makemessages --no-location`
-3. Compile messages
-`docker exec --env PYTHONPATH="/match4everyone-backend:$PYTHONPATH" backend django-admin compilemessages`
-4. Collect static
-`docker exec backend python3 manage.py collectstatic --no-input`
-5. Migrate
-`docker exec backend python3 manage.py migrate`
-5. Check if all the variables are correct
-`docker exec backend python3 manage.py check`
-6. Restart the backend container (important, whitenoise does not reload static files after it has started)
-`docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml down && docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml up -d`
+#### Build the containers
+(Copy `.env.example` to `.env` and adjust variables if you want to run the backend as non-root)
+`docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build`
+
+Building containers will run a number of django tasks automatically:
+- make messages (`django-admin makemessages --no-location`)
+- compile messages (`django-admin compilemessages`)
+- collect static files (`django-admin collectstatic --no-input`)
+
+Starting the containers will run the following django tasks on every backend startup:
+- Perform migrations (`django-admin migrate`)
+- Perform system check (`django-admin check`)
+
 
 ## Helpful management commands
 
@@ -86,7 +91,7 @@ In order to run pre-commit checks every time, please run `pre-commit install` on
 - Add translatable strings in python with `_("Welcome to my site.")` and import `from django.utils.translation import gettext as _` ([Documentation](https://docs.djangoproject.com/en/3.0/topics/i18n/translation/#internationalization-in-python-code))
 - Add translatable strings in templates with `{% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %}` or alternatively with the `trans` block and include `{% load i18n %}` at the top ([Documentation](https://docs.djangoproject.com/en/3.0/topics/i18n/translation/#internationalization-in-template-code))
 - Update the translation file
-`django-admin makemessages -l de --no-location --ignore 00_old_m4h_matching_code` (line numbers omitted to allow nicer merges)
+`django-admin makemessages -l de --no-location` (line numbers omitted to allow nicer merges)
 - Edit translations in `backend/locale/de/LC_MESSAGES/django.po`
 
 ### Testing

backend/.dockerignore

@@ -0,0 +1,5 @@
+**/__pycache__
+**/.gitkeep
+**/.gitignore
+run/*
+!run/log

backend/Dockerfile

@@ -1,11 +1,23 @@
 FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y python3 python3-pip libpq-dev gettext
-WORKDIR /match4everyone-backend
-COPY requirements.txt /match4everyone-backend/requirements.txt
+
+WORKDIR /backend
+COPY requirements.txt /backend/requirements.txt
 RUN pip3 install -r requirements.txt
-COPY requirements.prod.txt /match4everyone-backend/requirements.prod.txt
+COPY requirements.prod.txt /backend/requirements.prod.txt
 RUN pip3 install -r requirements.prod.txt
 
+COPY . .
+
+RUN python3 manage.py makemessages --no-location
+RUN python3 manage.py compilemessages
+RUN python3 manage.py collectstatic --no-input
+# Change permissions on run/ in case app is later run by non-root user
+# and delete log files as these are created during the above commands when django loads the configuration
+# and will be created non writeable by others than root
+RUN rm -v run/log/* && chmod a+rwX run/log && chmod a+rwX backups
+
 EXPOSE 8000
+ENTRYPOINT ["./entrypoint.sh"]
 CMD ["python3", "manage.py", "runserver", "0.0.0.0:8000"]

backend/apps/matching/management/commands/makemessages.py

@@ -0,0 +1,23 @@
+import fileinput
+import re
+
+from django.core.management.commands.makemessages import Command as MakeMessageCommand
+
+
+class Command(MakeMessageCommand):
+
+    help = (  # noqa
+        MakeMessageCommand.help
+        + "Modified behaviour: Will reset creation date in all po files to 1900-01-01 00:00+0000"
+        "to prevent diffs from being created when the only change would be a rerun of makemessages"
+    )
+
+    def build_potfiles(self):
+        potfiles = super().build_potfiles()
+        print("Using constant POT-Creation-Date: 1900-01-01 00:00+0000")
+        pattern = re.compile(r"POT-Creation-Date: .*\\n")
+        for line in fileinput.input(files=potfiles, inplace=True):
+            line = pattern.sub(r"POT-Creation-Date: 1900-01-01 00:00+0000\\n", line)
+            print(line, end="")
+
+        return potfiles

backend/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+if ! [[ -z "${PRODUCTION}" ]]; then
+  echo Running in production mode, waiting for database to come up
+  # Moved from docker-compose, as this is needed for migrate
+  echo -n "Waiting for database"
+  while ! (< /dev/tcp/database/5432) &> /dev/null; do
+    echo -n .
+    sleep .2
+  done
+  echo " OK"
+fi
+
+echo PYTHONPATH: $PYTHONPATH
+django-admin migrate
+django-admin check
+
+exec "$@"

backend/gunicorn.conf.py

@@ -8,5 +8,5 @@
 worker_class = "gevent"
 worker_connections = 1000
 
-accesslog = os.path.join(settings.RUN_DIR, "gunicorn-access.log")
+accesslog = os.path.join(settings.LOG_DIR, "gunicorn-access.log")
 access_log_format = '%(t)s|"%(r)s"|%(s)s|%(T)s'

backend/locale/de/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-06-30 11:18+0200\n"
+"POT-Creation-Date: 1900-01-01 00:00+0000\n"
 "PO-Revision-Date: 2020-06-21 09:42+0200\n"
 "Last-Translator: \n"
 "Language-Team: \n"

backend/match4everyone/settings/common.py

@@ -21,7 +21,7 @@
 # add paths here and import: from django.col import settings and use settings.XXX_DIR
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 RUN_DIR = os.path.join(BASE_DIR, "run")
-
+LOG_DIR = os.path.join(RUN_DIR, "log")
 
 # Application definition
 
@@ -57,6 +57,7 @@
 ]
 
 MIDDLEWARE = [
+    "whitenoise.middleware.WhiteNoiseMiddleware",
     # 'cms.middleware.utils.ApphookReloadMiddleware' TODO: Not working right now "ModuleNotFoundError: No module named 'cms.middleware.utils.ApphookReloadMiddlewaredjango'; 'cms.middleware.utils' is not a package"
     "django.middleware.security.SecurityMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
@@ -159,16 +160,15 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 PROJECT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-STATIC_URL = "/static/"
 
 MEDIA_ROOT = os.path.join(RUN_DIR, "media")
-
 MEDIA_URL = "/media/"
 # TODO: Serve media files properly (http://docs.django-cms.org/en/latest/how_to/install.html#media-and-static-file-handling)
 
+STATIC_URL = "/static/"
 STATIC_ROOT = os.path.join(RUN_DIR, "static")
-
 STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"),)
+STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 LEAFLET_TILESERVER = os.getenv(
     "LEAFLET_TILESERVER"
@@ -214,15 +214,15 @@
             "class": "logging.handlers.RotatingFileHandler",
             "formatter": "json",
             "level": "ERROR",
-            "filename": path.join(RUN_DIR, "match4everyone.json.error.log"),
+            "filename": path.join(LOG_DIR, "match4everyone.json.error.log"),
             "maxBytes": 1024 * 1024 * 15,  # 15MB
             "backupCount": 10,
         },
         "auditlogfile": {
             "class": "logging.handlers.RotatingFileHandler",
             "formatter": "json",
             "level": "INFO",
-            "filename": path.join(RUN_DIR, "match4everyone.json.audit.log"),
+            "filename": path.join(LOG_DIR, "match4everyone.json.audit.log"),
             "maxBytes": 1024 * 1024 * 15,  # 15MB
             "backupCount": 10,
         },

backend/match4everyone/settings/production.py

@@ -5,7 +5,7 @@
 
 from match4everyone.constants.enum import Environment
 from match4everyone.settings.common import *  # noqa
-from match4everyone.settings.common import IS_FORK, MIDDLEWARE, RUN_DIR
+from match4everyone.settings.common import IS_FORK, RUN_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -41,9 +41,6 @@
     }
 }
 
-MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware")
-
-STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 # =============== MAIL RELAY SERVER CONFIGURATION ===============
 # TODO: add environment variable based detection whether we are on prod or staging # noqa: T003

backend/requirements.prod.txt

@@ -3,5 +3,3 @@ gunicorn==20.0.4
 psutil==5.7.0
 psycopg2==2.8.5
 psycopg2==2.8.5
-whitenoise==5.0.1
-whitenoise[brotli]

backend/requirements.txt

@@ -31,3 +31,4 @@ sendgrid==6.2.0
 six==1.15.0
 tqdm==4.46.1
 whitenoise==5.0.1
+whitenoise[brotli]

backend/run/.gitignore

@@ -7,11 +7,16 @@
 
 # Except for the static folder
 !static
-!static/.gitkeep
 # But its content
 static/*
+!static/.gitkeep
+
 # Except for the media folder
 !media
-!media/.gitkeep
 # But its content
 media/*
+!media/.gitkeep
+
+!log
+log/*
+!log/.gitkeep

backup.sh

@@ -1,4 +1,21 @@
 #!/usr/bin/env bash
-source database.prod.env
-docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml exec backend sh -c 'python3 manage.py dumpdata > /match4everyone-backend/backups/fixture-$(date +%F_%H%M%S).json'
-docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml exec database sh -c "pg_dumpall -U $POSTGRES_USER> /backups/pg_backup-$(date +%F_%H%M%S).sql"
+# This backup script will create a database dump from the postgres container
+# the created backup will then be moved to the current directory
+set -o errexit
+
+WORKINGDIR=$(pwd)
+
+echo "Creating PostgreSQL Dump"
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml exec database sh -c 'pg_dumpall -U $POSTGRES_USER> /backups/pg_backup-$(date +%F_%H%M%S).sql'
+
+# Create a throwaway container (--rm removes after running the command) to move the backups from the DB-container volume to the local directory for further processing
+# (Mounts the current working dir as /backup-bind-mount inside the container and moves the backups there)
+echo "Moving the backups to $WORKINGDIR"
+DB_CONTAINER="$(docker-compose -f docker-compose.yml -f docker-compose.prod.yml ps -q database)"
+docker run --rm --volumes-from "$DB_CONTAINER" -v "${WORKINGDIR}:/host" alpine sh -c "mv -v backups/* /host/database/backups"
+
+echo -e "\nCreating Django fixtures"
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml exec backend  sh -c 'django-admin dumpdata > /backend/backups/fixture-$(date +%F_%H%M%S).json'
+echo "Moving the backups to $WORKINGDIR"
+BACKEND_CONTAINER="$(docker-compose -f docker-compose.yml -f docker-compose.prod.yml ps -q backend)"
+docker run --rm --volumes-from "$BACKEND_CONTAINER" -v "${WORKINGDIR}:/host" alpine sh -c "mv -v /backend/backups/*.json /host/backend/backups"

database/.gitignore

@@ -3,4 +3,5 @@
 # Ignore database files
 data/*
 # Ignore backups
-backups/
+backups/*
+!backups/.gitkeep

database/Dockerfile

@@ -1,3 +1,4 @@
-FROM postgres
+FROM postgres:12
 
 COPY init-db.sh /docker-entrypoint-initdb.d/
+COPY postgresql.conf /etc/postgresql.conf

database/postgresql.conf

@@ -312,7 +312,7 @@ min_wal_size = 80MB
 #primary_slot_name = ''			# replication slot on sending server
 					# (change requires restart)
 #promote_trigger_file = ''		# file name whose presence ends recovery
-#hot_standby = on			# "off" disallows queries during recovery
+hot_standby = off			# "off" disallows queries during recovery
 					# (change requires restart)
 #max_standby_archive_delay = 30s	# max delay before canceling queries
 					# when reading WAL from archive;