Easier deployment

.env.example

@@ -0,0 +1,3 @@
+# Use backend_user to specify a user to run the backend container as.
+# If none is specified root will be used
+BACKEND_USER=user:group

.gitignore

@@ -193,9 +193,6 @@ psd
 thumb
 sketch
 
-# Ignore local docker-compose file, in case you make a copy for local changes
-docker-compose.yml
-
 # End of https://www.gitignore.io/api/react,django,python
 
 # pycharm settings

.travis.yml

@@ -3,18 +3,17 @@ services:
 
 before_script:
  - ./scripts/write_envs_to_file.sh
- - ./deploy.sh
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --build -d
  - ./scripts/wait_for_backend.sh
  - docker logs $(docker ps --format '{{.Names}}' | grep backend)
  - docker logs $(docker ps --format '{{.Names}}' | grep database)
 
-
 script:
  - bash scripts/check_website_availability.sh
- - docker exec backend python3 manage.py test
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml exec backend python3 manage.py test
 
 after_script:
- - docker-compose down
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml down
 
 env:
  - POSTGRES_DB=match4everyone POSTGRES_PASSWORD="BuZK@HPB_FDpGx3gvnnB9@eypozpC8PeesGP7PUC*DDgbbj-Zpv3kRwCkmU6FQoL" POSTGRES_USER=match4everyone SECRET_KEY="QP=14DkrovY9qrqXmZk3sjgakmy@x#6Log7EiW-9K4g5W&xY)_yN3a*rsinLYP|"

README.md

@@ -7,51 +7,47 @@ Originally developed as [match4healthcare](https://github.com/match4everyone/mat
 
 ## Quick install
 - Copy `backend.prod.env.example` and `database.prod.env.example` to `backend.prod.env` and `database.prod.env` and fill in appropriate values
-- Run `./deploy.sh` (uses Docker) and visit `http://localhost:8000/`
+- Run `docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build` (uses Docker) and visit `http://localhost:8000/`
 
+## Environment variables
+
+TODO
 
 ## Install by hand
 ### Development
 - Build images and run containers
-`docker-compose -f docker-compose.dev.yml up --build`
+`docker-compose up --build`
 - Start previously built containers in background
 `docker-compose start`
-- Apply migrations
-`docker exec backend python3 manage.py migrate`
-- Collect static files
-`docker exec backend python3 manage.py collectstatic`
-- Load test data:
-`docker exec backend python3 manage.py loaddata fixture.json`
+- Load test data (file needs to exist inside backend folder):
+`docker-compose exec backend django-admin loaddata fixture.json`
 
 File changes in python files trigger an auto-reload of the server.
-Migrations have to be executed with `docker exec backend python3 manage.py migrate`.
+Migrations are automatically executed when the container starts.
 
-After changes to the Docker configuration, you have to restart and build the containers with `docker-compose -f docker-compose.dev.yml up --build`.
+After changes to the Docker configuration, you have to restart and build the containers with `docker-compose up --build`.
 
 ### Production
-Set `SECRET_KEY`, `SENDGRID_API_KEY` and `MAPBOX_TOKEN`in `backend.prod.env` for Django
-`POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`  inside `database.prod.env` for postgres on your host machine.
-Also add a `SLACK_LOG_WEBHOOK` to enable slack logging.
+Copy `backend.prod.env.example` to `backend.prod.env` and set variables as documented in the example file for Django
+Copy `database.prod.env.example` to `database.prod.env` and set variables as documented in the example file for postgres on your host machine.
 
 To run a container in production and in a new environment execute the `deploy.sh` script which builds the containers, runs all configurations and starts the web service.
 
 If you want to deploy manually follow these steps closly:
 
-1. Build the containers
-(Run `export CURRENT_UID=$(id -u):$(id -g)` if you want to run the backend as non-root)
-`docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml up -d --build`
-2. Make messages
-`docker exec --env PYTHONPATH="/match4everyone-backend:$PYTHONPATH" backend django-admin makemessages --no-location`
-3. Compile messages
-`docker exec --env PYTHONPATH="/match4everyone-backend:$PYTHONPATH" backend django-admin compilemessages`
-4. Collect static
-`docker exec backend python3 manage.py collectstatic --no-input`
-5. Migrate
-`docker exec backend python3 manage.py migrate`
-5. Check if all the variables are correct
-`docker exec backend python3 manage.py check`
-6. Restart the backend container (important, whitenoise does not reload static files after it has started)
-`docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml down && docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml up -d`
+#### Build the containers
+(Copy `.env.example` to `.env` and adjust variables if you want to run the backend as non-root)
+`docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build`
+
+Building containers will run a number of django tasks automatically:
+- make messages (`django-admin makemessages --no-location`)
+- compile messages (`django-admin compilemessages`)
+- collect static files (`django-admin collectstatic --no-input`)
+
+Starting the containers will run the following django tasks on every backend startup:
+- Perform migrations (`django-admin migrate`)
+- Perform system check (`django-admin check`)
+
 
 ## Helpful management commands
 
@@ -86,7 +82,7 @@ In order to run pre-commit checks every time, please run `pre-commit install` on
 - Add translatable strings in python with `_("Welcome to my site.")` and import `from django.utils.translation import gettext as _` ([Documentation](https://docs.djangoproject.com/en/3.0/topics/i18n/translation/#internationalization-in-python-code))
 - Add translatable strings in templates with `{% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %}` or alternatively with the `trans` block and include `{% load i18n %}` at the top ([Documentation](https://docs.djangoproject.com/en/3.0/topics/i18n/translation/#internationalization-in-template-code))
 - Update the translation file
-`django-admin makemessages -l de --no-location --ignore 00_old_m4h_matching_code` (line numbers omitted to allow nicer merges)
+`django-admin makemessages -l de --no-location` (line numbers omitted to allow nicer merges)
 - Edit translations in `backend/locale/de/LC_MESSAGES/django.po`
 
 ### Testing

backend/.dockerignore

@@ -0,0 +1,2 @@
+**/__pycache__
+run/*

backend/Dockerfile

@@ -1,11 +1,23 @@
 FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y python3 python3-pip libpq-dev gettext
-WORKDIR /match4everyone-backend
-COPY requirements.txt /match4everyone-backend/requirements.txt
+
+WORKDIR /backend
+COPY requirements.txt /backend/requirements.txt
 RUN pip3 install -r requirements.txt
-COPY requirements.prod.txt /match4everyone-backend/requirements.prod.txt
+COPY requirements.prod.txt /backend/requirements.prod.txt
 RUN pip3 install -r requirements.prod.txt
 
+COPY . .
+
+RUN python3 manage.py makemessages --no-location --ignore 00_old_m4h_matching_code
+RUN python3 manage.py compilemessages
+
+# Change permissions on run/ in case app is later run by non-root user
+# and delete log files as these are created during the above commands when django loads the configuration
+# and will be created non writeable by others than root
+RUN chmod -Rc a+rwX run && rm -v run/*log
+
 EXPOSE 8000
+ENTRYPOINT ["./entrypoint.sh"]
 CMD ["python3", "manage.py", "runserver", "0.0.0.0:8000"]

backend/apps/matching/management/commands/makemessages.py

@@ -0,0 +1,23 @@
+import fileinput
+import re
+
+from django.core.management.commands.makemessages import Command as MakeMessageCommand
+
+
+class Command(MakeMessageCommand):
+
+    help = (  # noqa
+        MakeMessageCommand.help
+        + "Modified behaviour: Will reset creation date in all po files to 1900-01-01 00:00+0000"
+        "to prevent diffs from being created when the only change would be a rerun of makemessages"
+    )
+
+    def build_potfiles(self):
+        potfiles = super().build_potfiles()
+        print("Using constant POT-Creation-Date: 1900-01-01 00:00+0000")
+        pattern = re.compile(r"POT-Creation-Date: .*\\n")
+        for line in fileinput.input(files=potfiles, inplace=True):
+            line = pattern.sub(r"POT-Creation-Date: 1900-01-01 00:00+0000\\n", line)
+            print(line, end="")
+
+        return potfiles

backend/entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+if ! [[ -z "${PRODUCTION}" ]]; then
+  echo Running in production mode, waiting for database to come up
+  # Moved from docker-compose, as this is needed for migrate
+  while ! (< /dev/tcp/database/5432) &> /dev/null; do
+    echo "Waiting for database..."
+    sleep 1
+  done
+fi
+
+# While collectstatic should rather run in the Dockerfile build
+# it depends on the right DJANGO_SETTINGS_MODULE as it is subject to the
+# STATICFILES_STORAGE setting. Whitenoise is only configured in production as of now
+# so we would need to build the image in production mode, this would require setting a DJANGO_SECRET key
+echo PYTHONPATH: $PYTHONPATH
+django-admin collectstatic --no-input
+django-admin migrate
+django-admin check
+
+exec "$@"

backend/locale/de/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-06-21 07:40+0000\n"
+"POT-Creation-Date: 1900-01-01 00:00+0000\n"
 "PO-Revision-Date: 2020-06-21 09:42+0200\n"
 "Last-Translator: \n"
 "Language-Team: \n"
@@ -351,7 +351,7 @@ msgid ""
 "\n"
 "    Approve\n"
 "    "
-msgstr  ""
+msgstr ""
 "\n"
 "    Freischalten\n"
 "    "
@@ -361,7 +361,8 @@ msgstr "Diese Teilnehmer müssen noch freigeschaltet werden."
 
 msgid "These participants are approved, but you can disapprove them any time."
 msgstr ""
-"Diese Teilnehmer sind freigeschaltet, aber Sie können die Freischaltung jederzeit wieder rückgängig machen."
+"Diese Teilnehmer sind freigeschaltet, aber Sie können die Freischaltung "
+"jederzeit wieder rückgängig machen."
 
 msgid "Staff"
 msgstr "Administrator"
@@ -393,7 +394,9 @@ msgstr "Sie haben den Teilnehmer mit der E-Mail-Adresse '{name}' gelöscht."
 
 #, python-brace-format
 msgid "You changed the approval of the participant with email '{name}'."
-msgstr "Sie haben für den Teilnehmer mit der E-Mail-Adresse '{name}' den Status geändert."
+msgstr ""
+"Sie haben für den Teilnehmer mit der E-Mail-Adresse '{name}' den Status "
+"geändert."
 
 msgid "Access stats"
 msgstr "Website Zugriffe"

backup.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 source database.prod.env
-docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml exec backend sh -c 'python3 manage.py dumpdata > /match4everyone-backend/backups/fixture-$(date +%F_%H%M%S).json'
+docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml exec backend sh -c 'python3 manage.py dumpdata > /backend/backups/fixture-$(date +%F_%H%M%S).json'
 docker-compose -f docker-compose.dev.yml -f docker-compose.prod.yml exec database sh -c "pg_dumpall -U $POSTGRES_USER> /backups/pg_backup-$(date +%F_%H%M%S).sql"

docker-compose.override.yml

@@ -0,0 +1,13 @@
+version: '3.7'
+
+# This file is automatically merged with docker-compose.yml when docker-compose is called without -f parameter
+# when invoked using docker-compose -f ... -f ... it is not automatically merged into the configuration
+#
+# Options declared here only apply for development mode or when explicitly added to the docker-compose call
+
+services:
+  backend:
+    volumes:
+      - './backend:/backend' # bind mount the code directory
+    env_file:
+      - backend.dev.env

docker-compose.prod.yml

@@ -4,7 +4,7 @@ services:
 
   backend:
     volumes:
-      - './backend:/match4everyone-backend'
+      - backend-run:/backend/run
     ports:
       - '80:8000'
     networks:
@@ -14,8 +14,8 @@ services:
       - database.prod.env
     environment:
      - DJANGO_SETTINGS_MODULE=match4everyone.settings.production
-    user: ${CURRENT_UID:-0}
-    command: bash -c 'while ! (< /dev/tcp/database/5432) &> /dev/null; do echo "Waiting for database..."; sleep 1; done; gunicorn -c /match4everyone-backend/gunicorn.conf.py match4everyone.wsgi'
+     - PRODUCTION=True
+    command: gunicorn -c /backend/gunicorn.conf.py match4everyone.wsgi
     depends_on:
       - database
 
@@ -39,5 +39,5 @@ services:
 networks:
  network:
 
-#volumes:
-#  postgres-data:
+volumes:
+  backend-run:

docker-compose.dev.yml → docker-compose.yml

@@ -3,15 +3,16 @@ version: '3.7'
 services:
 
   backend:
-    container_name: backend
     build:
       context: ./backend
       dockerfile: Dockerfile
-    volumes:
-      - './backend:/match4everyone-backend'
     ports:
       - '8000:8000'
+    tty: true
     environment:
+      - DJANGO_SETTINGS_MODULE=match4everyone.settings.development
       - CI
       - TRAVIS
       - TRAVIS_PULL_REQUEST_SLUG
+      - PYTHONPATH=/backend # needed for django-admin to find the module
+    user: ${BACKEND_USER:-0}

scripts/check_website_availability.sh

@@ -26,7 +26,10 @@ failed=False
 URL=localhost:8000
 
 # Log file location
-ERR_LOG_PATH=backend/run/match4everyone.json.error.log
+ERR_LOG_PATH=./run/match4everyone.json.error.log
+
+# docker-compose to run command in backend
+DC_EXEC="docker-compose -f docker-compose.yml -f docker-compose.prod.yml exec backend bash -c "
 
 # Execute command and print status
 function test() {
@@ -61,8 +64,8 @@ function check_website_up() {
 function check_error_log_empty() {
 	# Check if log file exists
 	for i in $(seq 1 10); do
-		if [ -f "$ERR_LOG_PATH" ]; then
-    		break
+		if $DC_EXEC "[ -f \"$ERR_LOG_PATH\" ]"; then
+			break
 		else
 			if [[ i -eq 10 ]]; then
 				printf "Log file not found\t"
@@ -74,9 +77,9 @@ function check_error_log_empty() {
 
 	curl --silent --output /dev/null localhost:8000
 	sleep 5
-	if [  -s "$ERR_LOG_PATH" ]; then
+	if $DC_EXEC "[  -s \"$ERR_LOG_PATH\" ]"; then
 		printf "\n"
-		cat "$ERR_LOG_PATH"
+		$DC_EXEC "cat \"$ERR_LOG_PATH\""
 		printf "\n"
 		return 1
 	fi

scripts/wait_for_backend.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
-while ! [ $(docker ps | grep backend | wc -l) -gt 0 ]; do
+while [ -z "$(docker-compose -f docker-compose.yml -f docker-compose.prod.yml ps --services --filter "status=running"|grep backend)" ]; do
     sleep 1
 done

scripts/write_envs_to_file.sh

@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
-echo "POSTGRES_DB=${POSTGRES_DB}" > database.prod.env
-echo "POSTGRES_USER=${POSTGRES_USER}" >> database.prod.env
-echo "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" >> database.prod.env
-
-echo "SECRET_KEY=${SECRET_KEY}" > backend.prod.env
-echo "SENDGRID_API_KEY=${SENDGRID_API_KEY}" >> backend.prod.env
-echo "RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}" >> backend.prod.env
-echo "RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}" >> backend.prod.env
+DATABASE_ENV_VARS="POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD"
+BACKEND_ENV_VARS="SECRET_KEY SENDGRID_API_KEY SLACK_LOG_WEBHOOK LEAFLET_TILESERVER MAPBOX_TOKEN"
+
+# Write env variables into env file
+# This way you can set them in the travis configuration and they
+# will be written into the appropriate env files by  this script
+
+for ENVVAR in ${DATABASE_ENV_VARS}; do
+    echo "${ENVVAR}=${!ENVVAR}" >> database.prod.env
+done
+
+for ENVVAR in ${BACKEND_ENV_VARS}; do
+    echo "${ENVVAR}=${!ENVVAR}" >> backend.prod.env
+done