Merge project setup with appropriate local dev environment settings

.dockerignore

@@ -1,7 +1,5 @@
 # Django project
 /media/
-/static/
-*.sqlite3
 
 # Python and others
 __pycache__
@@ -37,3 +35,15 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+
+# Github workflows
+.github
+
+# Git
+.git
+
+# Docker compose
+docker-compose*
+
+# Makefile
+Makefile

.env.example

@@ -1,10 +1,18 @@
-SECRET_KEY = ""
+ENV=${ENV:-dev}
+SECRET_KEY=${SECRET_KEY:-somesuperdupersecretkeyfortesting}
+
 # SECURITY WARNING: define the correct hosts in production!
-ALLOWED_HOSTS = ["*"]
+ALLOWED_HOSTS=${ALLOWED_HOSTS:-["*"]}
+
+# Superuser creation (override in production)
+DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin}
+DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:[email protected]}
+DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-adminpassword}
 
 # Postgres DB settings
-DB_NAME = ""
-DB_USER = ""
-DB_PASSWORD = ""
-DB_HOST = ""
-DB_PORT = 5432
+# port, name, host, allowed hosts
+DB_NAME=${DB_NAME:-postgres}
+DB_USER=${DB_USER:-postgres}
+DB_PASSWORD=${DB_PASSWORD:-postgres}
+DB_HOST=${DB_HOST:-db}
+DB_PORT=${DB_PORT:-5432}

.github/workflows/docker_image_build.yml

@@ -0,0 +1,18 @@
+# Use hotosm workflow immage build fors pushes to main or release branches
+name: Image Build
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release/*'
+      - 'dev'
+
+jobs:
+  pytest:
+    uses: hotosm/gh-workflows/.github/workflows/test_pytest.yml@main
+    secrets: inherit
+    with:
+      image_name: ghcr.io/${{ github.repository }}
+      build_target: prod
+      environment: dev

.github/workflows/pytest.yml

@@ -0,0 +1,22 @@
+# Use hotosm workflow test_compose for pull requests to main or release branches
+name: PyTest
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - 'release/*'
+      - 'dev'
+jobs:
+  pytest:
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@main
+    secrets: inherit
+    with:
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_dockerfile: Dockerfile
+      build_target: test
+      compose_service: web
+      compose_command: "pytest"
+      compose_file: "docker-compose.dev.yml"
+      cache_extra_imgs: |
+        "docker.io/postgres:16-alpine3.18"

.gitignore

@@ -1 +1,6 @@
+.env.dev
+.env.prod
 .env
+**/__pycache__/
+**/.pytest_cache/
+**/.venv/

Dockerfile

@@ -1,58 +1,145 @@
-# Use an official Python runtime based on Debian 10 "buster" as a parent image.
-FROM python:3.10-slim-buster
+ARG PYTHON_IMG_TAG=3.11
 
-# Add user that will be used in the container.
-RUN useradd wagtail
 
-# Port used by this container to serve HTTP.
-EXPOSE 8000
+# Define the base stage
+FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base
+ARG APP_VERSION
+ARG COMMIT_REF
+ARG PYTHON_IMG_TAG
+LABEL org.hotosm.fmtm.app-name="backend" \
+      org.hotosm.fmtm.app-version="${APP_VERSION}" \
+      org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \
+      org.hotosm.fmtm.python-img-tag="${PYTHON_IMG_TAG}" \
+      org.hotosm.fmtm.maintainer="[email protected]" \
+      org.hotosm.fmtm.api-port="8000"
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends "locales" "ca-certificates" \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && update-ca-certificates
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
-# Set environment variables.
-# 1. Force Python stdout and stderr streams to be unbuffered.
-# 2. Set PORT variable that is used by Gunicorn. This should match "EXPOSE"
-#    command.
-ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1\
-    PORT=8000
 
-# Install system packages required by Wagtail and Django.
-RUN apt-get update --yes --quiet && apt-get install --yes --quiet --no-install-recommends \
-    build-essential \
-    libpq-dev \
-    libmariadbclient-dev \
-    libjpeg62-turbo-dev \
-    zlib1g-dev \
-    libwebp-dev \
- && rm -rf /var/lib/apt/lists/*
+# Extract dependencies using poetry (to requirements.txt)
+FROM base as extract-deps
+WORKDIR /opt/python
+COPY pyproject.toml poetry.lock* /opt/python/
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir \
+    poetry==1.7.1 poetry-plugin-export==1.6.0 \
+    && poetry config warnings.export false
+RUN poetry export --without dev --output requirements.txt
+RUN poetry export --only dev --output requirements-dev.txt
 
-# Install pipenv
-RUN pip install pipenv
 
-# Install the application server.
-RUN pip install "gunicorn==20.0.4"
+# Define build stage (install deps)
+FROM base as build
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "build-essential" \
+        "gcc" \
+        "libpq-dev" \
+        "libmariadb-dev" \
+        "libjpeg62-turbo-dev" \
+        "zlib1g-dev" \
+        "libwebp-dev" \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=extract-deps \
+    /opt/python/requirements.txt /opt/python/
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir -r /opt/python/requirements.txt
 
-# Copy Pipfile and Pipfile.lock
-COPY Pipfile Pipfile.lock ./
-
-# Install the project requirements.
-RUN pipenv install --system --deploy
 
+# Define run stage
+FROM base as runtime
+ARG PYTHON_IMG_TAG
+ENV PORT=8000 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/home/wagtail/.local/bin:$PATH" \
+    PYTHONPATH="/app" \
+    PYTHON_LIB="/home/wagtail/.local/lib/python$PYTHON_IMG_TAG/site-packages" \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
+    CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+# Install non-dev versions of packages (smaller)
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "postgresql-client" \
+        "libmariadb3" \
+        "libjpeg62-turbo" \
+        "zlib1g" \
+        "libwebp-dev" \
+    && rm -rf /var/lib/apt/lists/*
+# Copy the entrypoint script into the Docker image
+COPY --chown=wagtail:wagtail entrypoint.dev.sh /app/entrypoint.dev.sh
+# Copy pip dependencies from build stage
+COPY --from=build \
+    /root/.local \
+    /home/wagtail/.local
 # Use /app folder as a directory where the source code is stored.
 WORKDIR /app
+# Copy project
+COPY . /app/
+# Add non-root user, permissions
+RUN useradd -u 1001 -m -c "hotosm account" -d /home/wagtail -s /bin/false wagtail \
+    && chown -R wagtail:wagtail /app /home/wagtail \
+    && chmod +x /app/entrypoint.dev.sh
+# Change to non-root user
+USER wagtail
 
-# Set this directory to be owned by the "wagtail" user. This Wagtail project
-# uses SQLite, the folder needs to be owned by the user that
-# will be writing to the database file.
-RUN chown wagtail:wagtail /app
 
-# Copy the source code of the project into the container.
-COPY --chown=wagtail:wagtail . .
+# Define test (ci) stage
+FROM runtime as test
+USER root
+ARG PYTHON_IMG_TAG
+COPY --from=extract-deps \
+    /opt/python/requirements-dev.txt /opt/python/
+# Copy packages from user to root dirs (run ci as root)
+# && install dev dependencies (pytest)
+RUN mv /home/wagtail/.local/bin/* /usr/local/bin/ \
+    && mv /home/wagtail/.local/lib/python${PYTHON_IMG_TAG}/site-packages/* \
+    /usr/local/lib/python${PYTHON_IMG_TAG}/site-packages/ \
+    && pip install --upgrade --no-warn-script-location \
+    --no-cache-dir -r \
+    /opt/python/requirements-dev.txt \
+    && rm -r /opt/python \
+    # Pre-compile packages to .pyc (init speed gains)
+    && python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+CMD [ "pytest" ]
 
-# Use user "wagtail" to run the build commands below and the server itself.
-USER wagtail
 
-# Collect static files.
-RUN python manage.py collectstatic --noinput --clear
+# Define debug (development) stage
+FROM runtime as debug
+# Add Healthcheck
+HEALTHCHECK --start-period=10s --interval=5s --retries=20 --timeout=5s \
+    CMD curl --fail http://localhost:8000 || exit 1
+# Use the entrypoint script as the Docker entrypoint
+ENTRYPOINT ["/app/entrypoint.dev.sh"]
+CMD ["./wait-for-it.sh", "db:5432", "--", \
+    "python", "manage.py", "runserver", "0.0.0.0:8000"]
+
 
+# Define prod stage
+FROM runtime as prod
+# Add Healthcheck
+HEALTHCHECK --start-period=10s --interval=5s --retries=20 --timeout=5s \
+    CMD curl --fail http://localhost:8000 || exit 1
+# Pre-compile packages to .pyc (init speed gains)
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)" \
+    # Collect static files
+    && python manage.py collectstatic --noinput --clear
 # Runtime command that executes when "docker run" is called, it does the
 # following:
 #   1. Migrate the database.

LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2023 Humanitarian OpenStreetMap Team
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

Makefile

@@ -0,0 +1,29 @@
+.PHONY: build-dev build-prod test up-dev up-prod down-dev down-prod
+
+build-dev:
+	@docker compose -f docker-compose.dev.yml build
+	@docker compose -f docker-compose.dev.yml up -d
+
+build-prod:
+	@docker compose -f docker-compose.prod.yml build
+	@docker compose -f docker-compose.prod.yml up -d
+
+test:
+	@docker build --target test -t myproject:test -f Dockerfile .
+	@docker run --rm myproject:test
+	@docker rmi myproject:test
+
+up-dev:
+	@docker compose -f docker-compose.dev.yml up -d
+
+up-prod:
+	@docker compose -f docker-compose.prod.yml up
+
+down-dev:
+	@docker compose -f docker-compose.dev.yml down
+
+down-prod:
+	@docker compose -f docker-compose.prod.yml down
+
+refresh-db:
+	@if [ -n "$(shell docker volume ls -q)" ]; then docker volume rm $(shell docker volume ls -q); fi