diff --git a/.gitignore b/.gitignore index 71362317..3eb9fb54 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,9 @@ typings/ # Amplify configs for the frontend app taui/src/aws-exports.js + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +src/backend/static \ No newline at end of file diff --git a/README.md b/README.md index 2ff0dbc8..9088dac3 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,21 @@ versions are not installed). Navigate to http://localhost:9966 to view the development environment. +### STRTA + +This project uses [`scripts-to-rule-them-all`](https://github.com/azavea/architecture/blob/master/doc/arch/adr-0000-scripts-to-rule-them-all.md) to bootstrap, test, and maintain projects consistently across all teams. Below is a quick explanation for the specific usage of each script. + +| Script | Use | +| ----------- | ---------------------------------------------------------- | +| `bootstrap` | Pull down secrets from S3 | +| `infra` | Execute Terraform subcommands with remote state management | +| `manage` | Issue Django management commands | +| `server` | Start the frontend and backend services | +| `setup` | Setup the project development environment | +| `test` | Run linters and tests | +| `update` | Update project, assemble, run migrations | + + ### Logging In Once it is running, log in using staging credentials. From there, you can make a Client ID by diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 52cb8b39..12a97572 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,9 +1,10 @@ -version: '3' +version: '2.1' services: taui: environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + terraform: image: quay.io/azavea/terraform:0.11.11 volumes: @@ -17,6 +18,7 @@ services: - ECHOLOCATOR_SITE_BUCKET=${ECHOLOCATOR_SITE_BUCKET:-echo-locator-staging-site-us-east-1} working_dir: /usr/local/src entrypoint: bash + amplify: build: ./deployment/amplify volumes: @@ -24,3 +26,6 @@ services: - $HOME/.aws:/root/.aws:ro working_dir: /usr/local/src entrypoint: bash + + django: + image: "echolocator:${GIT_COMMIT:-latest}" diff --git a/docker-compose.yml b/docker-compose.yml index 1356d4fe..0ae7afaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.1' services: taui: build: ./taui @@ -10,3 +10,52 @@ services: - "9966:9966" entrypoint: yarn command: start + + django: + image: echolocator + environment: + - POSTGRES_HOST=database + - POSTGRES_PORT=5432 + - POSTGRES_USER=echolocator + - POSTGRES_PASSWORD=echolocator + - POSTGRES_DB=echolocator + - DJANGO_ENV=Development + - DJANGO_SECRET_KEY=secret + - DJANGO_LOG_LEVEL=INFO + build: + context: ./src/backend + dockerfile: Dockerfile + volumes: + - ./src/backend:/usr/local/src/backend + - $HOME/.aws:/root/.aws:ro + working_dir: /usr/local/src/backend + depends_on: + database: + condition: service_healthy + command: + - "-b :8085" + - "--reload" + - "--timeout=90" + - "--access-logfile=-" + - "--error-logfile=-" + - "--log-level=debug" + - "echo.wsgi" + ports: + - "8085:8085" + cpus: 2 + + database: + image: postgis/postgis:13-3.1 + expose: + - "5432" + environment: + - POSTGRES_USER=echolocator + - POSTGRES_PASSWORD=echolocator + - POSTGRES_DB=echolocator + healthcheck: + test: [ "CMD", "pg_isready", "-U", "echolocator" ] + interval: 3s + timeout: 3s + retries: 3 + start_period: 5s + command: postgres -c log_statement=all diff --git a/scripts/bootstrap b/scripts/bootstrap new file mode 100755 index 00000000..ec12c99f --- /dev/null +++ b/scripts/bootstrap @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECHOLOCATOR_DEBUG}" ]]; then + set -x +fi + +DIR="$(dirname "${0}")/../" + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Pull down secrets from S3. +" +} + +# TODO: potentially as a part of this issue: https://github.com/azavea/echo-locator/issues/376 +# after switching to a different bundler, we may want to copy and populate some env vars +function pull_env() { + pushd "${DIR}" + + echo "Pulling .env from ${1}" + # aws s3 cp "s3://${1}/.env" ".env" + + popd +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + pull_env + fi +fi diff --git a/scripts/manage b/scripts/manage new file mode 100755 index 00000000..6419c83b --- /dev/null +++ b/scripts/manage @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECHOLOCATOR_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Run a Django management command +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + docker-compose \ + run --rm --entrypoint python \ + django \ + manage.py "$@" + fi +fi \ No newline at end of file diff --git a/scripts/server b/scripts/server index da24ecc2..ef718d22 100755 --- a/scripts/server +++ b/scripts/server @@ -9,8 +9,7 @@ fi function usage() { echo -n \ "Usage: $(basename "$0") - -Run a development server. +Starts servers using docker-compose. " } @@ -20,6 +19,6 @@ then then usage else - docker-compose up --build + docker-compose up fi fi diff --git a/scripts/setup b/scripts/setup new file mode 100755 index 00000000..8b2f67bc --- /dev/null +++ b/scripts/setup @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECHOLOCATOR_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Attempts to setup the project's development environment. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + ./scripts/bootstrap + ./scripts/update + fi +fi diff --git a/scripts/update b/scripts/update new file mode 100755 index 00000000..bbe46159 --- /dev/null +++ b/scripts/update @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECHOLOCATOR_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Build container images and execute database migrations. +" +} + +function cleanup() { + docker-compose stop +} + +trap cleanup ERR + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + # Ensure container images are current + docker-compose build + + # Bring up PostgreSQL and Django in a way that respects + # configured service health checks. + docker-compose up -d database django + + # Apply any outstanding Django migrations + ./scripts/manage migrate + + # Collect Django static files + ./scripts/manage collectstatic --no-input + + docker-compose stop + fi +fi diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile new file mode 100644 index 00000000..1136510b --- /dev/null +++ b/src/backend/Dockerfile @@ -0,0 +1,28 @@ +FROM quay.io/azavea/django:3.2-python3.9-slim + +RUN mkdir -p /usr/local/src/backend +WORKDIR /usr/local/src/backend + +COPY requirements.txt /usr/local/src/backend/ +RUN set -ex \ + && buildDeps=" \ + build-essential \ + " \ + && deps=" \ + postgresql-client-13 \ + " \ + && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \ + && pip install --no-cache-dir -r requirements.txt \ + && apt-get purge -y --auto-remove $buildDeps + +COPY . /usr/local/src/backend + +CMD ["-b :8085", \ + "--workers=1", \ + "--timeout=60", \ + "--access-logfile=-", \ + "--access-logformat=%({X-Forwarded-For}i)s %(h)s %(l)s %(u)s %(t)s \"%(r)s\" %(s)s %(b)s \"%(f)s\" \"%(a)s\"", \ + "--error-logfile=-", \ + "--log-level=info", \ + "--capture-output", \ + "echo.wsgi"] diff --git a/src/backend/api/__init__.py b/src/backend/api/__init__.py new file mode 100644 index 00000000..e8662783 --- /dev/null +++ b/src/backend/api/__init__.py @@ -0,0 +1 @@ +default_app_config = 'api.apps.ApiConfig' \ No newline at end of file diff --git a/src/backend/api/admin.py b/src/backend/api/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/src/backend/api/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/src/backend/api/apps.py b/src/backend/api/apps.py new file mode 100644 index 00000000..66656fd2 --- /dev/null +++ b/src/backend/api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'api' diff --git a/src/backend/api/migrations/__init__.py b/src/backend/api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/api/models.py b/src/backend/api/models.py new file mode 100644 index 00000000..71a83623 --- /dev/null +++ b/src/backend/api/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/src/backend/api/tests.py b/src/backend/api/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/backend/api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/backend/api/views.py b/src/backend/api/views.py new file mode 100644 index 00000000..0b6e719d --- /dev/null +++ b/src/backend/api/views.py @@ -0,0 +1,3 @@ +from rest_framework.response import Response +from rest_framework import status +from rest_framework.decorators import api_view \ No newline at end of file diff --git a/src/backend/echo/__init__.py b/src/backend/echo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/echo/asgi.py b/src/backend/echo/asgi.py new file mode 100644 index 00000000..f0679667 --- /dev/null +++ b/src/backend/echo/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for echo project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'echo.settings') + +application = get_asgi_application() diff --git a/src/backend/echo/settings.py b/src/backend/echo/settings.py new file mode 100644 index 00000000..c8862fff --- /dev/null +++ b/src/backend/echo/settings.py @@ -0,0 +1,162 @@ +""" + +Generated by 'django-admin startproject' using Django 3.2. + +For more information on this file, see +Django settings for echo project. + +https://docs.djangoproject.com/en/3.2/topics/settings/ +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +import os + +from django.core.exceptions import ImproperlyConfigured +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'secret') + +# Set environment +ENVIRONMENT = os.getenv('DJANGO_ENV', 'Development') +VALID_ENVIRONMENTS = ('Production', 'Staging', 'Development') +if ENVIRONMENT not in VALID_ENVIRONMENTS: + raise ImproperlyConfigured( + 'Invalid ENVIRONMENT provided, must be one of {}' + .format(VALID_ENVIRONMENTS)) + +LOGLEVEL = os.getenv('DJANGO_LOG_LEVEL', 'INFO') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = (ENVIRONMENT == 'Development') + +ALLOWED_HOSTS = [] + +if ENVIRONMENT == 'Development': + ALLOWED_HOSTS.append('localhost') + ALLOWED_HOSTS.append('django') + +# Application definition + +INSTALLED_APPS = [ + 'whitenoise.runserver_nostatic', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.gis', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django_extensions', + 'rest_framework', + 'rest_framework_gis', + 'api', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', + 'spa.middleware.SPAMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'echo.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'echo.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.getenv('POSTGRES_DB'), + 'USER': os.getenv('POSTGRES_USER'), + 'PASSWORD': os.getenv('POSTGRES_PASSWORD'), + 'HOST': os.getenv('POSTGRES_HOST'), + 'PORT': os.getenv('POSTGRES_PORT') + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'static') + +# Set the django-spa static file storage: +STATICFILES_STORAGE = 'spa.storage.SPAStaticFilesStorage' + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +WATCHMAN_ERROR_CODE = 503 +WATCHMAN_CHECKS = ( + 'watchman.checks.databases', + 'api.checks.gazetteercache' +) \ No newline at end of file diff --git a/src/backend/echo/urls.py b/src/backend/echo/urls.py new file mode 100644 index 00000000..76624b63 --- /dev/null +++ b/src/backend/echo/urls.py @@ -0,0 +1,22 @@ +"""echo URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path +from django.conf.urls.static import static + +from echo import settings + +urlpatterns = static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/src/backend/echo/wsgi.py b/src/backend/echo/wsgi.py new file mode 100644 index 00000000..35ee16ac --- /dev/null +++ b/src/backend/echo/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for echo project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'echo.settings') + +application = get_wsgi_application() diff --git a/src/backend/manage.py b/src/backend/manage.py new file mode 100755 index 00000000..5e67a341 --- /dev/null +++ b/src/backend/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'echo.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/src/backend/requirements.txt b/src/backend/requirements.txt new file mode 100644 index 00000000..aaa53fdf --- /dev/null +++ b/src/backend/requirements.txt @@ -0,0 +1,5 @@ +django-extensions==3.1.5 +django-watchman==1.3.0 +djangorestframework-gis==0.18 +djangorestframework==3.13.1 +django-spa==0.3.6 \ No newline at end of file diff --git a/taui/package.json b/taui/package.json index 57b27d40..6c76c5f5 100644 --- a/taui/package.json +++ b/taui/package.json @@ -14,7 +14,7 @@ "prestart": "yarn", "pretest": "yarn", "semantic-release": "semantic-release", - "start": "mastarm build --serve --watch", + "start": "mastarm build --serve --watch --proxy http://django:8085", "test": "mastarm lint-js src --ignore-path src/.eslintignore && mastarm lint-styles --config .stylelintrc && mastarm lint-messages && mastarm build" }, "repository": { @@ -87,4 +87,4 @@ "standard": { "parser": "babel-eslint" } -} +} \ No newline at end of file