From dd9abcd38b5ec5b65085611115d6b806a2a7a6df Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 16 Feb 2024 15:07:35 -0500 Subject: [PATCH 01/40] adding prod-deployment infrastructure --- bats_ai/settings.py | 1 + client/.env.production | 7 +++--- dev/client.Dockerfile | 29 ++++++++++++++++++++++ docker-compose.prod.yml | 53 +++++++++++++++++++++++++++++++++++++++++ nginx/nginx.conf | 39 ++++++++++++++++++++++++++++++ setup.py | 1 + 6 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 dev/client.Dockerfile create mode 100644 docker-compose.prod.yml create mode 100644 nginx/nginx.conf diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 9764a92..58cd007 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -14,6 +14,7 @@ CORS_ALLOWED_ORIGINS = [ 'http://localhost:3000', + 'http://localhost', ] diff --git a/client/.env.production b/client/.env.production index edf6415..bee52e4 100644 --- a/client/.env.production +++ b/client/.env.production @@ -1,4 +1,3 @@ -VUE_APP_API_ROOT=https://CHANGEME/api/v1 -VUE_APP_OAUTH_API_ROOT=https://CHANGEME/oauth/ -VUE_APP_OAUTH_CLIENT_ID=CHANGEME -VUE_APP_SENTRY_DSN=CHANGEME +VUE_APP_API_ROOT=http://localhost:8000/api/v1 +VUE_APP_OAUTH_API_ROOT=http://localhost:8000/oauth/ +VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP diff --git a/dev/client.Dockerfile b/dev/client.Dockerfile new file mode 100644 index 0000000..e741064 --- /dev/null +++ b/dev/client.Dockerfile @@ -0,0 +1,29 @@ +# Use official Node.js image as the base image +FROM node:16 as build-stage + +# Set working directory +WORKDIR /app + +# Copy package.json and package-lock.json +COPY client/package*.json ./ + +# Install dependencies +RUN npm install + +# Copy the rest of the application +COPY client . + +# Build the Vue.js application +RUN npm run build + +# Production stage +FROM nginx:alpine + +# Copy built app from build stage to nginx +COPY --from=build-stage /app/dist /usr/share/nginx/html + +# Expose port 80 to the outer world +EXPOSE 80 + +# Command to run nginx +CMD ["nginx", "-g", "daemon off;"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..4c4cf94 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,53 @@ +version: '3.8' + +services: + django: + build: + context: . + dockerfile: ./dev/django.Dockerfile + command: 'gunicorn bats_ai.wsgi:application --bind 0.0.0.0:8000' + # ./manage.py runserver 0.0.0.0:8000 --noreload + # entrypoint: ["/bin/bash"] + # command: "" + # Log printing via Rich is enhanced by a TTY + tty: true + env_file: ./dev/.env.docker-compose + volumes: + - .:/opt/django-project + ports: + - 8000:8000 + depends_on: + - postgres + - rabbitmq + - minio + celery: + build: + context: . + dockerfile: ./dev/django.Dockerfile + command: [ + "celery", + "--app", "bats_ai.celery", + "worker", + "--loglevel", "INFO", + "--without-heartbeat" + ] + # Docker Compose does not set the TTY width, which causes Celery errors + tty: false + env_file: ./dev/.env.docker-compose + volumes: + - .:/opt/django-project + depends_on: + - postgres + - rabbitmq + - minio + client: + build: + context: . + dockerfile: ./dev/client.Dockerfile + ports: + - 80:80 + depends_on: + - django +networks: + backend: + driver: bridge \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..0a908aa --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,39 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + upstream django { + server django:8000; + } + upstream vue_app { + server vue_app:80; + } + + server { + listen 80; + server_name localhost; + + location /api { + proxy_pass http://django; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + proxy_pass http://vue_app; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + try_files $uri $uri/ /index.html; + } + } +} diff --git a/setup.py b/setup.py index a8deb60..6589de7 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ include_package_data=True, install_requires=[ 'celery', + 'gunicorn', 'django-ninja', 'django>=4.1, <4.2', 'django-allauth', From f489ce1481671aae0629c70f4c1fee2de08bbc41 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 08:51:00 -0500 Subject: [PATCH 02/40] adding environment variables --- dev/.env.docker-compose | 2 ++ docker-compose.prod.yml | 2 +- docker-compose.yml | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dev/.env.docker-compose b/dev/.env.docker-compose index be89c27..3aadfbe 100644 --- a/dev/.env.docker-compose +++ b/dev/.env.docker-compose @@ -1,4 +1,6 @@ DJANGO_CONFIGURATION=DevelopmentConfiguration +DJANGO_DATABASE_NAME=django +DJANGO_DATABASE_PASSWORD=postgres DJANGO_DATABASE_URL=postgres://postgres:postgres@postgres:5432/django DJANGO_CELERY_BROKER_URL=amqp://rabbitmq:5672/ DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4c4cf94..96a2567 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -50,4 +50,4 @@ services: - django networks: backend: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 52df5c6..bab8bd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,8 @@ services: postgres: image: postgis/postgis:latest environment: - POSTGRES_DB: django - POSTGRES_PASSWORD: postgres + - POSTGRES_DB=${DJANGO_DATABASE_NAME:-django} + - POSTGRES_PASSWORD=${DJANGO_MINIO_STORAGE_SECRET_KEY:-postgres} ports: - ${DOCKER_POSTGRES_PORT-5432}:5432 volumes: @@ -24,8 +24,8 @@ services: tty: true command: ["server", "/data", "--console-address", ":${DOCKER_MINIO_CONSOLE_PORT-9001}"] environment: - MINIO_ROOT_USER: minioAccessKey - MINIO_ROOT_PASSWORD: minioSecretKey + - MINIO_ROOT_USER=${DJANGO_MINIO_STORAGE_ACCESS_KEY:-minioAccessKey} + - MINIO_ROOT_PASSWORD=${DJANGO_DATABASE_PASSWORD:-minioSecretKey} ports: - ${DOCKER_MINIO_PORT-9000}:9000 - ${DOCKER_MINIO_CONSOLE_PORT-9001}:9001 From ebeaa0e549830c95c27e80990e44ff9a5e5976c6 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 10:56:59 -0500 Subject: [PATCH 03/40] reconfigure items --- bats_ai/settings.py | 18 ++++++++- client/.env.production | 4 +- dev/.env.prod.docker-compose | 11 ++++++ dev/client.Dockerfile | 17 ++++++--- docker-compose.prod.yml | 74 +++++++++++++++++++++++++++++++++++- nginx/nginx.conf | 52 +++++++++++++++++++------ 6 files changed, 155 insertions(+), 21 deletions(-) create mode 100644 dev/.env.prod.docker-compose diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 58cd007..88527a8 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -82,7 +82,23 @@ class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): - pass + SECRET_KEY = 'secretkey' # Dummy value for local development configuration + + DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' + MINIO_STORAGE_ENDPOINT = values.Value( + 'minio:9000', + ) + MINIO_STORAGE_USE_HTTPS = values.BooleanValue(False) + MINIO_STORAGE_ACCESS_KEY = values.SecretValue() + MINIO_STORAGE_SECRET_KEY = values.SecretValue() + MINIO_STORAGE_MEDIA_BUCKET_NAME = values.Value( + environ_name='STORAGE_BUCKET_NAME', + environ_required=True, + ) + MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET = True + MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' + MINIO_STORAGE_MEDIA_USE_PRESIGNED = True + MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' class HerokuProductionConfiguration(BatsAiMixin, HerokuProductionBaseConfiguration): diff --git a/client/.env.production b/client/.env.production index bee52e4..080cf16 100644 --- a/client/.env.production +++ b/client/.env.production @@ -1,3 +1,3 @@ -VUE_APP_API_ROOT=http://localhost:8000/api/v1 -VUE_APP_OAUTH_API_ROOT=http://localhost:8000/oauth/ +VUE_APP_API_ROOT=http://localhost/api/v1 +VUE_APP_OAUTH_API_ROOT=http://localhost/oauth/ VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose new file mode 100644 index 0000000..239641d --- /dev/null +++ b/dev/.env.prod.docker-compose @@ -0,0 +1,11 @@ +DJANGO_CONFIGURATION=DProductionConfiguration +DJANGO_DATABASE_NAME=django +DJANGO_DATABASE_PASSWORD=postgres +DJANGO_DATABASE_URL=postgres://postgres:postgres@postgres:5432/django +DJANGO_CELERY_BROKER_URL=amqp://rabbitmq:5672/ +DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 +DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey +DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey +DJANGO_STORAGE_BUCKET_NAME=django-storage +DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 +DJANGO_CORS_ORIGIN_WHITELIST=http://localhost:3000 diff --git a/dev/client.Dockerfile b/dev/client.Dockerfile index e741064..908f5c1 100644 --- a/dev/client.Dockerfile +++ b/dev/client.Dockerfile @@ -1,4 +1,4 @@ -# Use official Node.js image as the base image +# Use official Node.js image as the base image for building Vue.js app FROM node:16 as build-stage # Set working directory @@ -16,14 +16,21 @@ COPY client . # Build the Vue.js application RUN npm run build -# Production stage +# Use NGINX as the final base image FROM nginx:alpine -# Copy built app from build stage to nginx +# Remove default NGINX website +RUN rm -rf /usr/share/nginx/html/* + +# Copy built Vue.js app to NGINX HTML directory COPY --from=build-stage /app/dist /usr/share/nginx/html -# Expose port 80 to the outer world +RUN ls +# Copy custom NGINX configuration +COPY nginx/nginx.conf /etc/nginx/nginx.conf + +# Expose port 80 EXPOSE 80 -# Command to run nginx +# Start NGINX CMD ["nginx", "-g", "daemon off;"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 96a2567..b484436 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -12,6 +12,8 @@ services: # Log printing via Rich is enhanced by a TTY tty: true env_file: ./dev/.env.docker-compose + networks: + - django-nginx volumes: - .:/opt/django-project ports: @@ -44,10 +46,80 @@ services: build: context: . dockerfile: ./dev/client.Dockerfile + networks: + - django-nginx ports: - 80:80 depends_on: - django + + postgres: + image: postgis/postgis:latest + environment: + - POSTGRES_DB=${DJANGO_DATABASE_NAME:-django} + - POSTGRES_PASSWORD=${DJANGO_MINIO_STORAGE_SECRET_KEY:-postgres} + networks: + - django-nginx + ports: + - ${DOCKER_POSTGRES_PORT-5432}:5432 + volumes: + - postgres:/var/lib/postgresql/data + + rabbitmq: + image: rabbitmq:management + networks: + - django-nginx + ports: + - ${DOCKER_RABBITMQ_PORT-5672}:5672 + - ${DOCKER_RABBITMQ_CONSOLE_PORT-15672}:15672 + volumes: + - rabbitmq:/var/lib/rabbitmq/mnesia + + minio: + image: minio/minio:latest + # When run with a TTY, minio prints credentials on startup + tty: true + command: ["server", "/data", "--console-address", ":${DOCKER_MINIO_CONSOLE_PORT-9001}"] + environment: + - MINIO_ROOT_USER=${DJANGO_MINIO_STORAGE_ACCESS_KEY:-minioAccessKey} + - MINIO_ROOT_PASSWORD=${DJANGO_DATABASE_PASSWORD:-minioSecretKey} + networks: + - django-nginx + ports: + - ${DOCKER_MINIO_PORT-9000}:9000 + - ${DOCKER_MINIO_CONSOLE_PORT-9001}:9001 + volumes: + - minio:/data + + flower: + build: + context: . + dockerfile: ./dev/django.Dockerfile + command: [ + "celery", + "--app", "bats_ai.celery", + "flower" + ] + tty: false + env_file: ./dev/.env.docker-compose + volumes: + - .:/opt/django-project + networks: + - django-nginx + ports: + - ${DOCKER_FLOWER_PORT-5555}:5555 + depends_on: + - postgres + - rabbitmq + - minio + - celery + +volumes: + postgres: + sourcedb: + minio: + rabbitmq: + networks: - backend: + django-nginx: driver: bridge diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 0a908aa..c3b51b6 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -7,33 +7,61 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; - - upstream django { - server django:8000; - } - upstream vue_app { - server vue_app:80; - } + client_max_body_size 100m; server { listen 80; - server_name localhost; + + # Serve Vue.js static files + location / { + root /usr/share/nginx/html; + index index.html; + try_files $uri $uri/ /index.html; + } + # Proxy API requests to Django location /api { - proxy_pass http://django; + proxy_pass http://django:8000/api; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + location /oauth { + proxy_pass http://django:8000/oauth; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + location /static { + proxy_pass http://django:8000/static; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; } - location / { - proxy_pass http://vue_app; + location /admin { + proxy_pass http://django:8000/admin; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - try_files $uri $uri/ /index.html; + proxy_redirect off; } + location /accounts { + proxy_pass http://django:8000/accounts; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + + } } From d7bbe8b892df112e764b64005d6f10140f342a84 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:10:03 -0500 Subject: [PATCH 04/40] updating prod config --- client/.env.production | 4 ++-- client/.env.production.test | 3 +++ docker-compose.prod.yml | 44 +++++++++++++++++++++++++++++++++---- nginx/nginx.conf | 8 +++++++ traefik/dynamic.xml | 0 5 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 client/.env.production.test create mode 100644 traefik/dynamic.xml diff --git a/client/.env.production b/client/.env.production index 080cf16..84eebba 100644 --- a/client/.env.production +++ b/client/.env.production @@ -1,3 +1,3 @@ -VUE_APP_API_ROOT=http://localhost/api/v1 -VUE_APP_OAUTH_API_ROOT=http://localhost/oauth/ +VUE_APP_API_ROOT=http://batai.kitware.com/api/v1 +VUE_APP_OAUTH_API_ROOT=http://batai.kitware.com/oauth/ VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP diff --git a/client/.env.production.test b/client/.env.production.test new file mode 100644 index 0000000..080cf16 --- /dev/null +++ b/client/.env.production.test @@ -0,0 +1,3 @@ +VUE_APP_API_ROOT=http://localhost/api/v1 +VUE_APP_OAUTH_API_ROOT=http://localhost/oauth/ +VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b484436..da15856 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,6 +1,36 @@ version: '3.8' services: + # COMMENTED OUT UNTIL READY TO TEST + # traefik: + # restart: always + # command: > + # --log.level=${LOG_LEVEL:-DEBUG} + # --providers.docker=true + # --providers.docker.exposedByDefault=false + # --providers.file.filename=/var/traefik/dynamic.yml + # --entrypoints.web.address=:80 + # --entrypoints.websecure.address=:443 + # --entrypoints.websecure.http.tls.certresolver=myresolver + # --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} + # --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json + # --certificatesresolvers.myresolver.acme.httpchallenge=true + # --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web + # --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} + # labels: + # # Traefik HTTPS Redirect + # - "traefik.enable=true" + # - "traefik.http.routers.http-catchall.entrypoints=web" + # - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + # - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" + # - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" + # volumes: + # - "./traefik/letsencrypt:/letsencrypt" + # - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" + # ports: + # - "80:80" + # - "443:443" + django: build: context: . @@ -11,7 +41,7 @@ services: # command: "" # Log printing via Rich is enhanced by a TTY tty: true - env_file: ./dev/.env.docker-compose + env_file: ./dev/.env.prod.docker-compose networks: - django-nginx volumes: @@ -35,7 +65,7 @@ services: ] # Docker Compose does not set the TTY width, which causes Celery errors tty: false - env_file: ./dev/.env.docker-compose + env_file: ./dev/.env.prod.docker-compose volumes: - .:/opt/django-project depends_on: @@ -49,10 +79,16 @@ services: networks: - django-nginx ports: - - 80:80 + - 90:80 depends_on: - django - + labels: + - "traefik.http.routers.client-rtr.entrypoints=websecure" + - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" + - "traefik.enable=true" + - "traefik.http.services.client-svc.loadbalancer.server.port=8080" + - "traefik.http.routers.client-rtr.entrypoints=web" + - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: image: postgis/postgis:latest environment: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c3b51b6..274fa8a 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,6 +62,14 @@ http { proxy_redirect off; } + location /django-storage { + proxy_pass http://minio:9000/django-storage; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } } } diff --git a/traefik/dynamic.xml b/traefik/dynamic.xml new file mode 100644 index 0000000..e69de29 From 0f66d766ed715d471265f5409a6882c25fad7828 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:10:17 -0500 Subject: [PATCH 05/40] updating prod config --- dev/.env.prod.docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose index 239641d..aadd912 100644 --- a/dev/.env.prod.docker-compose +++ b/dev/.env.prod.docker-compose @@ -1,4 +1,4 @@ -DJANGO_CONFIGURATION=DProductionConfiguration +DJANGO_CONFIGURATION=ProductionConfiguration DJANGO_DATABASE_NAME=django DJANGO_DATABASE_PASSWORD=postgres DJANGO_DATABASE_URL=postgres://postgres:postgres@postgres:5432/django @@ -8,4 +8,4 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 -DJANGO_CORS_ORIGIN_WHITELIST=http://localhost:3000 +DJANGO_CORS_ORIGIN_WHITELIST=http://batai.kitware.com From a3d330891840d4dbc14fc29c393b720fdf8ba76c Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:14:30 -0500 Subject: [PATCH 06/40] allowed hosts --- bats_ai/settings.py | 1 + traefik/dynamic.xml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 88527a8..e93292f 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -99,6 +99,7 @@ class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' + ALLOWED_HOSTS = ['batai.kitware.com'] class HerokuProductionConfiguration(BatsAiMixin, HerokuProductionBaseConfiguration): diff --git a/traefik/dynamic.xml b/traefik/dynamic.xml index e69de29..ca6764f 100644 --- a/traefik/dynamic.xml +++ b/traefik/dynamic.xml @@ -0,0 +1,5 @@ +tls: + options: + default: + minVersion: VersionTLS12 + sniStrict: true From 8cb91ea25dd53551c9e1330b1b0190276523c523 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:19:58 -0500 Subject: [PATCH 07/40] adding fake email url --- dev/.env.prod.docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose index aadd912..8cd0454 100644 --- a/dev/.env.prod.docker-compose +++ b/dev/.env.prod.docker-compose @@ -9,3 +9,4 @@ DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 DJANGO_CORS_ORIGIN_WHITELIST=http://batai.kitware.com +DJANGO_EMAIL_URL=submission://USER:PASSWORD@smtp.sendgrid.com From f2265ca7d8029b316a86b5f26df7364a499bfe54 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:22:12 -0500 Subject: [PATCH 08/40] new configuration --- bats_ai/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index e93292f..a6ea799 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -80,8 +80,7 @@ class DevelopmentConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): pass - -class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): +class KitwareConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): SECRET_KEY = 'secretkey' # Dummy value for local development configuration DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' @@ -102,5 +101,8 @@ class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): ALLOWED_HOSTS = ['batai.kitware.com'] +class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): + + class HerokuProductionConfiguration(BatsAiMixin, HerokuProductionBaseConfiguration): pass From 3dba307d6b23942770858bafd8a2075dd7c399e0 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:23:50 -0500 Subject: [PATCH 09/40] new configuration --- bats_ai/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index a6ea799..0c260cb 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -102,7 +102,7 @@ class KitwareConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): - + pass class HerokuProductionConfiguration(BatsAiMixin, HerokuProductionBaseConfiguration): pass From 57e6ca38b9f1c6f9643deb9cc6e77b35d2854163 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:25:46 -0500 Subject: [PATCH 10/40] allowed hosts' --- dev/.env.prod.docker-compose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose index 8cd0454..9fe1865 100644 --- a/dev/.env.prod.docker-compose +++ b/dev/.env.prod.docker-compose @@ -1,4 +1,4 @@ -DJANGO_CONFIGURATION=ProductionConfiguration +DJANGO_CONFIGURATION=KitwareConfiguration DJANGO_DATABASE_NAME=django DJANGO_DATABASE_PASSWORD=postgres DJANGO_DATABASE_URL=postgres://postgres:postgres@postgres:5432/django @@ -10,3 +10,4 @@ DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 DJANGO_CORS_ORIGIN_WHITELIST=http://batai.kitware.com DJANGO_EMAIL_URL=submission://USER:PASSWORD@smtp.sendgrid.com +DJANGO_ALLOWED_HOSTS=['batai.kitware.com'] From ac11a797f28ce264e5cdcad759537cfa86265641 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:26:56 -0500 Subject: [PATCH 11/40] prod config --- bats_ai/settings.py | 1 - docker-compose.prod.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 0c260cb..94788e2 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -98,7 +98,6 @@ class KitwareConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' - ALLOWED_HOSTS = ['batai.kitware.com'] class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index da15856..c4a0480 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -137,7 +137,7 @@ services: "flower" ] tty: false - env_file: ./dev/.env.docker-compose + env_file: ./dev/.env.prod.docker-compose volumes: - .:/opt/django-project networks: From 52f100e07c71116e5ff6437ff6551867c54ae540 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:30:04 -0500 Subject: [PATCH 12/40] add network to celery --- docker-compose.prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c4a0480..e902f9b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -66,6 +66,8 @@ services: # Docker Compose does not set the TTY width, which causes Celery errors tty: false env_file: ./dev/.env.prod.docker-compose + networks: + - django-nginx volumes: - .:/opt/django-project depends_on: From b9fe3197f55462bd3ed98589591dc9b9c52e9527 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:33:02 -0500 Subject: [PATCH 13/40] prod --- bats_ai/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 94788e2..510b9ef 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -80,7 +80,7 @@ class DevelopmentConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): pass -class KitwareConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): +class KitwareConfiguration(BatsAiMixin, ProductionBaseConfiguration): SECRET_KEY = 'secretkey' # Dummy value for local development configuration DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' @@ -98,6 +98,7 @@ class KitwareConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' + ALLOWED_HOSTS = ['batai.kitware.com'] class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): From d37b41e3b7e54c854ee0850d7d5bef160da90bfe Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:39:33 -0500 Subject: [PATCH 14/40] base configuration --- bats_ai/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 510b9ef..65d107e 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -10,6 +10,7 @@ ProductionBaseConfiguration, TestingBaseConfiguration, ) +from composed_configuration._configuration import _BaseConfiguration from configurations import values CORS_ALLOWED_ORIGINS = [ @@ -80,9 +81,10 @@ class DevelopmentConfiguration(BatsAiMixin, DevelopmentBaseConfiguration): class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): pass -class KitwareConfiguration(BatsAiMixin, ProductionBaseConfiguration): - SECRET_KEY = 'secretkey' # Dummy value for local development configuration +class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): + SECRET_KEY = 'secretkey' # Dummy value for local development configuration + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' MINIO_STORAGE_ENDPOINT = values.Value( 'minio:9000', From b3054659e0b10a6d9b1028dd305ecd78b8b0fe7e Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:44:18 -0500 Subject: [PATCH 15/40] adding traefik --- bats_ai/settings.py | 5 ---- docker-compose.prod.yml | 56 ++++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 65d107e..d83fdbc 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -13,11 +13,6 @@ from composed_configuration._configuration import _BaseConfiguration from configurations import values -CORS_ALLOWED_ORIGINS = [ - 'http://localhost:3000', - 'http://localhost', -] - class BatsAiMixin(ConfigMixin): WSGI_APPLICATION = 'bats_ai.wsgi.application' diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e902f9b..eedaaf0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,34 +2,34 @@ version: '3.8' services: # COMMENTED OUT UNTIL READY TO TEST - # traefik: - # restart: always - # command: > - # --log.level=${LOG_LEVEL:-DEBUG} - # --providers.docker=true - # --providers.docker.exposedByDefault=false - # --providers.file.filename=/var/traefik/dynamic.yml - # --entrypoints.web.address=:80 - # --entrypoints.websecure.address=:443 - # --entrypoints.websecure.http.tls.certresolver=myresolver - # --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} - # --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json - # --certificatesresolvers.myresolver.acme.httpchallenge=true - # --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web - # --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} - # labels: - # # Traefik HTTPS Redirect - # - "traefik.enable=true" - # - "traefik.http.routers.http-catchall.entrypoints=web" - # - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" - # - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - # - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" - # volumes: - # - "./traefik/letsencrypt:/letsencrypt" - # - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" - # ports: - # - "80:80" - # - "443:443" + traefik: + restart: always + command: > + --log.level=${LOG_LEVEL:-DEBUG} + --providers.docker=true + --providers.docker.exposedByDefault=false + --providers.file.filename=/var/traefik/dynamic.yml + --entrypoints.web.address=:80 + --entrypoints.websecure.address=:443 + --entrypoints.websecure.http.tls.certresolver=myresolver + --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} + --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json + --certificatesresolvers.myresolver.acme.httpchallenge=true + --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web + --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} + labels: + # Traefik HTTPS Redirect + - "traefik.enable=true" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" + - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" + volumes: + - "./traefik/letsencrypt:/letsencrypt" + - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" + ports: + - "80:80" + - "443:443" django: build: From 6f01c8adc9ac46f230ebe22379fbdeecc0f921b2 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:46:01 -0500 Subject: [PATCH 16/40] update traefik --- docker-compose.prod.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index eedaaf0..031be8c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -4,7 +4,12 @@ services: # COMMENTED OUT UNTIL READY TO TEST traefik: restart: always + image: traefik:v2.4 + container_name: traefik command: > + --providers.docker=true + --providers.docker.exposedByDefault=false + --entrypoints.web.address=:80 --log.level=${LOG_LEVEL:-DEBUG} --providers.docker=true --providers.docker.exposedByDefault=false @@ -25,6 +30,7 @@ services: - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: + - "${SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock" - "./traefik/letsencrypt:/letsencrypt" - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" ports: From c0344f47a78a68c05118efd1e6c011ab8b7fd607 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 11:49:11 -0500 Subject: [PATCH 17/40] port fix --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 031be8c..544d256 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -94,7 +94,7 @@ services: - "traefik.http.routers.client-rtr.entrypoints=websecure" - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - - "traefik.http.services.client-svc.loadbalancer.server.port=8080" + - "traefik.http.services.client-svc.loadbalancer.server.port=80" - "traefik.http.routers.client-rtr.entrypoints=web" - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: From ff04240994513bd389247f33401ef6b47559772d Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 12:04:13 -0500 Subject: [PATCH 18/40] traefik config --- docker-compose.prod.yml | 6 +++--- traefik/{dynamic.xml => dynamic.yml} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename traefik/{dynamic.xml => dynamic.yml} (100%) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 544d256..e8ab4fc 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -25,8 +25,8 @@ services: labels: # Traefik HTTPS Redirect - "traefik.enable=true" - - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + - "traefik.http.routers.girder-rtr.entrypoints=websecure" + - "traefik.http.routers.girder-rtr.rule=Host(`${HOSTNAME}`)" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -94,7 +94,7 @@ services: - "traefik.http.routers.client-rtr.entrypoints=websecure" - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - - "traefik.http.services.client-svc.loadbalancer.server.port=80" + - "traefik.http.services.client-svc.loadbalancer.server.port=90" - "traefik.http.routers.client-rtr.entrypoints=web" - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: diff --git a/traefik/dynamic.xml b/traefik/dynamic.yml similarity index 100% rename from traefik/dynamic.xml rename to traefik/dynamic.yml From fba95a08d1b2fd47d74693531b92bf0de20bcbb9 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 12:05:17 -0500 Subject: [PATCH 19/40] traefik config --- docker-compose.prod.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e8ab4fc..957c5b0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -25,8 +25,8 @@ services: labels: # Traefik HTTPS Redirect - "traefik.enable=true" - - "traefik.http.routers.girder-rtr.entrypoints=websecure" - - "traefik.http.routers.girder-rtr.rule=Host(`${HOSTNAME}`)" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -94,8 +94,7 @@ services: - "traefik.http.routers.client-rtr.entrypoints=websecure" - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - - "traefik.http.services.client-svc.loadbalancer.server.port=90" - - "traefik.http.routers.client-rtr.entrypoints=web" + - "traefik.http.services.client-svc.loadbalancer.server.port=80" - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: image: postgis/postgis:latest From 4fc3236deb2fe5e891027dd28bd53f5a9f6f5916 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 19 Feb 2024 12:12:48 -0500 Subject: [PATCH 20/40] remove traefik --- docker-compose.prod.yml | 70 ++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 957c5b0..7bff558 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,40 +2,40 @@ version: '3.8' services: # COMMENTED OUT UNTIL READY TO TEST - traefik: - restart: always - image: traefik:v2.4 - container_name: traefik - command: > - --providers.docker=true - --providers.docker.exposedByDefault=false - --entrypoints.web.address=:80 - --log.level=${LOG_LEVEL:-DEBUG} - --providers.docker=true - --providers.docker.exposedByDefault=false - --providers.file.filename=/var/traefik/dynamic.yml - --entrypoints.web.address=:80 - --entrypoints.websecure.address=:443 - --entrypoints.websecure.http.tls.certresolver=myresolver - --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} - --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json - --certificatesresolvers.myresolver.acme.httpchallenge=true - --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web - --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} - labels: - # Traefik HTTPS Redirect - - "traefik.enable=true" - - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" - - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" - volumes: - - "${SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock" - - "./traefik/letsencrypt:/letsencrypt" - - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" - ports: - - "80:80" - - "443:443" + # traefik: + # restart: always + # image: traefik:v2.4 + # container_name: traefik + # command: > + # --providers.docker=true + # --providers.docker.exposedByDefault=false + # --entrypoints.web.address=:80 + # --log.level=${LOG_LEVEL:-DEBUG} + # --providers.docker=true + # --providers.docker.exposedByDefault=false + # --providers.file.filename=/var/traefik/dynamic.yml + # --entrypoints.web.address=:80 + # --entrypoints.websecure.address=:443 + # --entrypoints.websecure.http.tls.certresolver=myresolver + # --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} + # --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json + # --certificatesresolvers.myresolver.acme.httpchallenge=true + # --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web + # --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} + # labels: + # # Traefik HTTPS Redirect + # - "traefik.enable=true" + # - "traefik.http.routers.http-catchall.entrypoints=web" + # - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + # - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" + # - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" + # volumes: + # - "${SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock" + # - "./traefik/letsencrypt:/letsencrypt" + # - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" + # ports: + # - "80:80" + # - "443:443" django: build: @@ -94,7 +94,7 @@ services: - "traefik.http.routers.client-rtr.entrypoints=websecure" - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - - "traefik.http.services.client-svc.loadbalancer.server.port=80" + - "traefik.http.services.client-svc.loadbalancer.server.port=90" - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: image: postgis/postgis:latest From ebcebcfd7aec01e7b2938b969c1304d0f9eb79eb Mon Sep 17 00:00:00 2001 From: Bryon Lewis <61746913+BryonLewis@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:44:20 -0500 Subject: [PATCH 21/40] Temporal Annotations (#30) * temporal annotation models and endpoints * migrations for temporal endpoints * updating migrations * basics of rendering temporal annotations * linting * temporal creation/editing/deletion * supporting multiple users with sequence and pulse annotations * client cleanup of minor issues --- bats_ai/core/admin/__init__.py | 4 + bats_ai/core/admin/species.py | 17 ++ bats_ai/core/admin/temporal_annotations.py | 18 ++ .../migrations/0007_temporalannotations.py | 43 ++++ bats_ai/core/models/__init__.py | 2 + bats_ai/core/models/temporal_annotations.py | 15 ++ bats_ai/core/views/__init__.py | 4 + bats_ai/core/views/annotations.py | 34 ++- bats_ai/core/views/recording.py | 192 ++++++++++++++- bats_ai/core/views/temporal_annotations.py | 66 +++++ client/src/App.vue | 8 +- client/src/api/api.ts | 48 +++- client/src/components/AnnotationEditor.vue | 40 +++- client/src/components/AnnotationList.vue | 207 ++++++++++------ client/src/components/MapLocation.vue | 6 +- client/src/components/SpectrogramViewer.vue | 62 +++-- client/src/components/TemporalList.vue | 114 +++++++++ client/src/components/UploadRecording.vue | 2 +- client/src/components/geoJS/LayerManager.vue | 226 ++++++++++++++---- client/src/components/geoJS/geoJSUtils.ts | 104 +++++++- .../geoJS/layers/editAnnotationLayer.ts | 78 +++--- .../components/geoJS/layers/rectangleLayer.ts | 11 - .../geoJS/layers/speciesSequenceLayer.ts | 135 +++++++++++ .../components/geoJS/layers/temporalLayer.ts | 193 +++++++++++++++ .../src/components/geoJS/layers/timeLayer.ts | 56 ++++- client/src/router/index.ts | 14 +- client/src/use/useState.ts | 26 +- client/src/views/Login.vue | 16 ++ client/src/views/Recordings.vue | 2 +- client/src/views/Spectrogram.vue | 41 ++-- 30 files changed, 1517 insertions(+), 267 deletions(-) create mode 100644 bats_ai/core/admin/species.py create mode 100644 bats_ai/core/admin/temporal_annotations.py create mode 100644 bats_ai/core/migrations/0007_temporalannotations.py create mode 100644 bats_ai/core/models/temporal_annotations.py create mode 100644 bats_ai/core/views/temporal_annotations.py create mode 100644 client/src/components/TemporalList.vue create mode 100644 client/src/components/geoJS/layers/speciesSequenceLayer.ts create mode 100644 client/src/components/geoJS/layers/temporalLayer.ts create mode 100644 client/src/views/Login.vue diff --git a/bats_ai/core/admin/__init__.py b/bats_ai/core/admin/__init__.py index 68eb9b8..23510a1 100644 --- a/bats_ai/core/admin/__init__.py +++ b/bats_ai/core/admin/__init__.py @@ -1,11 +1,15 @@ from .annotations import AnnotationsAdmin from .image import ImageAdmin from .recording import RecordingAdmin +from .species import SpeciesAdmin from .spectrogram import SpectrogramAdmin +from .temporal_annotations import TemporalAnnotationsAdmin __all__ = [ 'AnnotationsAdmin', 'ImageAdmin', 'RecordingAdmin', 'SpectrogramAdmin', + 'TemporalAnnotationsAdmin', + 'SpeciesAdmin', ] diff --git a/bats_ai/core/admin/species.py b/bats_ai/core/admin/species.py new file mode 100644 index 0000000..cad6420 --- /dev/null +++ b/bats_ai/core/admin/species.py @@ -0,0 +1,17 @@ +from django.contrib import admin + +from bats_ai.core.models import Species + + +@admin.register(Species) +class SpeciesAdmin(admin.ModelAdmin): + list_display = [ + 'pk', + 'species_code', + 'family', + 'genus', + 'species', + 'common_name', + 'species_code_6', + ] + list_select_related = True diff --git a/bats_ai/core/admin/temporal_annotations.py b/bats_ai/core/admin/temporal_annotations.py new file mode 100644 index 0000000..9415855 --- /dev/null +++ b/bats_ai/core/admin/temporal_annotations.py @@ -0,0 +1,18 @@ +from django.contrib import admin + +from bats_ai.core.models import TemporalAnnotations + + +@admin.register(TemporalAnnotations) +class TemporalAnnotationsAdmin(admin.ModelAdmin): + list_display = [ + 'pk', + 'recording', + 'owner', + 'start_time', + 'end_time', + 'type', + 'comments', + ] + list_select_related = True + autocomplete_fields = ['owner'] diff --git a/bats_ai/core/migrations/0007_temporalannotations.py b/bats_ai/core/migrations/0007_temporalannotations.py new file mode 100644 index 0000000..6549b82 --- /dev/null +++ b/bats_ai/core/migrations/0007_temporalannotations.py @@ -0,0 +1,43 @@ +# Generated by Django 4.1.13 on 2024-02-15 18:08 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('core', '0006_alter_recording_recording_location'), + ] + + operations = [ + migrations.CreateModel( + name='TemporalAnnotations', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('start_time', models.IntegerField(blank=True, null=True)), + ('end_time', models.IntegerField(blank=True, null=True)), + ('type', models.TextField(blank=True, null=True)), + ('comments', models.TextField(blank=True, null=True)), + ( + 'owner', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ( + 'recording', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='core.recording' + ), + ), + ('species', models.ManyToManyField(to='core.species')), + ], + ), + ] diff --git a/bats_ai/core/models/__init__.py b/bats_ai/core/models/__init__.py index 3dfd89d..cc8bf05 100644 --- a/bats_ai/core/models/__init__.py +++ b/bats_ai/core/models/__init__.py @@ -4,6 +4,7 @@ from .recording_annotation_status import RecordingAnnotationStatus from .species import Species from .spectrogram import Spectrogram +from .temporal_annotations import TemporalAnnotations __all__ = [ 'Annotations', @@ -12,4 +13,5 @@ 'RecordingAnnotationStatus', 'Species', 'Spectrogram', + 'TemporalAnnotations', ] diff --git a/bats_ai/core/models/temporal_annotations.py b/bats_ai/core/models/temporal_annotations.py new file mode 100644 index 0000000..501e2ac --- /dev/null +++ b/bats_ai/core/models/temporal_annotations.py @@ -0,0 +1,15 @@ +from django.contrib.auth.models import User +from django.db import models + +from .recording import Recording +from .species import Species + + +class TemporalAnnotations(models.Model): + recording = models.ForeignKey(Recording, on_delete=models.CASCADE) + owner = models.ForeignKey(User, on_delete=models.CASCADE) + start_time = models.IntegerField(blank=True, null=True) + end_time = models.IntegerField(blank=True, null=True) + type = models.TextField(blank=True, null=True) + comments = models.TextField(blank=True, null=True) + species = models.ManyToManyField(Species) diff --git a/bats_ai/core/views/__init__.py b/bats_ai/core/views/__init__.py index 6a0e84a..ddf5efb 100644 --- a/bats_ai/core/views/__init__.py +++ b/bats_ai/core/views/__init__.py @@ -1,7 +1,11 @@ +from .annotations import router as AnnotationRouter from .recording import router as RecordingRouter from .species import router as SpeciesRouter +from .temporal_annotations import router as TemporalAnnotationRouter __all__ = [ 'RecordingRouter', 'SpeciesRouter', + 'AnnotationRouter', + 'TemporalAnnotationRouter', ] diff --git a/bats_ai/core/views/annotations.py b/bats_ai/core/views/annotations.py index 68c9445..e83fd25 100644 --- a/bats_ai/core/views/annotations.py +++ b/bats_ai/core/views/annotations.py @@ -2,9 +2,9 @@ from django.http import HttpRequest from ninja import Schema -from ninja.errors import HttpError from ninja.pagination import RouterPaginated -from oauth2_provider.models import AccessToken + +from bats_ai.core.models import Annotations, Recording logger = logging.getLogger(__name__) @@ -23,10 +23,28 @@ class AnnotationSchema(Schema): comments: str -def get_owner_id(request: HttpRequest): - token = request.headers.get('Authorization').replace('Bearer ', '') - token_found = AccessToken.objects.get(token=token) - if not token_found: - raise HttpError(401, 'Authentication credentials were not provided.') +@router.get('/{id}') +def get_annotation(request: HttpRequest, id: int): + try: + annotation = Annotations.objects.get(pk=id) + recording = annotation.recording + + # Check if the user owns the recording or if the recording is public + if recording.owner == request.user or recording.public: + # Query annotations associated with the recording that are owned by the current user + annotations_qs = Annotations.objects.filter(recording=recording, owner=request.user) + + # Serialize the annotations using AnnotationSchema + annotations_data = [ + AnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict() + for annotation in annotations_qs + ] + + return annotations_data + else: + return { + 'error': 'Permission denied. You do not own this annotation, or the associated recording is not public.' + } - return token_found.user.pk + except Recording.DoesNotExist: + return {'error': 'Recording not found'} diff --git a/bats_ai/core/views/recording.py b/bats_ai/core/views/recording.py index 4fc9323..f9926fa 100644 --- a/bats_ai/core/views/recording.py +++ b/bats_ai/core/views/recording.py @@ -10,8 +10,12 @@ from ninja.files import UploadedFile from ninja.pagination import RouterPaginated -from bats_ai.core.models import Annotations, Recording, Species +from bats_ai.core.models import Annotations, Recording, Species, TemporalAnnotations from bats_ai.core.views.species import SpeciesSchema +from bats_ai.core.views.temporal_annotations import ( + TemporalAnnotationSchema, + UpdateTemporalAnnotationSchema, +) logger = logging.getLogger(__name__) @@ -202,13 +206,22 @@ def get_spectrogram(request: HttpRequest, id: int): spectro_data['currentUser'] = request.user.email annotations_qs = Annotations.objects.filter(recording=recording, owner=request.user) + temporal_annotations_qs = TemporalAnnotations.objects.filter( + recording=recording, owner=request.user + ) # Serialize the annotations using AnnotationSchema annotations_data = [ AnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict() for annotation in annotations_qs ] + temporal_annotations_data = [ + TemporalAnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict() + for annotation in temporal_annotations_qs + ] + spectro_data['annotations'] = annotations_data + spectro_data['temporal'] = temporal_annotations_data return spectro_data @@ -306,6 +319,9 @@ def get_other_user_annotations(request: HttpRequest, id: int): annotations_qs = Annotations.objects.filter(recording=recording).exclude( owner=request.user ) + temporal_qs = TemporalAnnotations.objects.filter(recording=recording).exclude( + owner=request.user + ) # Create a dictionary to store annotations for each user annotations_by_user = {} @@ -315,13 +331,24 @@ def get_other_user_annotations(request: HttpRequest, id: int): user_email = annotation.owner.email # If user_email is not already a key in the dictionary, initialize it with an empty list - annotations_by_user.setdefault(user_email, []) + annotations_by_user.setdefault(user_email, {'annotations': [], 'temporal': []}) # Append the annotation to the list for the corresponding user_email - annotations_by_user[user_email].append( + annotations_by_user[user_email]['annotations'].append( AnnotationSchema.from_orm(annotation, owner_email=user_email).dict() ) + for annotation in temporal_qs: + user_email = annotation.owner.email + + # If user_email is not already a key in the dictionary, initialize it with an empty list + annotations_by_user.setdefault(user_email, {'annotations': [], 'temporal': []}) + + # Append the annotation to the list for the corresponding user_email + annotations_by_user[user_email]['temporal'].append( + TemporalAnnotationSchema.from_orm(annotation, owner_email=user_email).dict() + ) + return annotations_by_user else: return { @@ -403,7 +430,7 @@ def patch_annotation( recording_id: int, id: int, annotation: UpdateAnnotationsSchema, - species_ids: list[int], + species_ids: list[int] | None, ): try: recording = Recording.objects.get(pk=recording_id) @@ -428,16 +455,68 @@ def patch_annotation( annotation_instance.save() # Clear existing species associations - annotation_instance.species.clear() + if species_ids: + annotation_instance.species.clear() + # Add species to the annotation based on the provided species_ids + for species_id in species_ids: + try: + species_obj = Species.objects.get(pk=species_id) + annotation_instance.species.add(species_obj) + except Species.DoesNotExist: + # Handle the case where the species with the given ID doesn't exist + return {'error': f'Species with ID {species_id} not found'} - # Add species to the annotation based on the provided species_ids - for species_id in species_ids: - try: - species_obj = Species.objects.get(pk=species_id) - annotation_instance.species.add(species_obj) - except Species.DoesNotExist: - # Handle the case where the species with the given ID doesn't exist - return {'error': f'Species with ID {species_id} not found'} + return {'message': 'Annotation updated successfully', 'id': annotation_instance.pk} + else: + return { + 'error': 'Permission denied. You do not own this recording, and it is not public.' + } + + except Recording.DoesNotExist: + return {'error': 'Recording not found'} + except Annotations.DoesNotExist: + return {'error': 'Annotation not found'} + + +@router.patch('/{recording_id}/temporal-annotations/{id}') +def patch_temporal_annotation( + request, + recording_id: int, + id: int, + annotation: UpdateTemporalAnnotationSchema, + species_ids: list[int] | None, +): + try: + recording = Recording.objects.get(pk=recording_id) + + # Check if the user owns the recording or if the recording is public + if recording.owner == request.user or recording.public: + annotation_instance = TemporalAnnotations.objects.get( + pk=id, recording=recording, owner=request.user + ) + + # Update annotation details + if annotation.start_time: + annotation_instance.start_time = annotation.start_time + if annotation.end_time: + annotation_instance.end_time = annotation.end_time + if annotation.comments: + annotation_instance.comments = annotation.comments + if annotation.type: + annotation_instance.type = annotation.type + annotation_instance.save() + + # Clear existing species associations + if species_ids: + annotation_instance.species.clear() + # Add species to the annotation based on the provided species_ids + for species_id in species_ids: + try: + species_obj = Species.objects.get(pk=species_id) + annotation_instance.species.add(species_obj) + except Species.DoesNotExist: + # Handle the case where the species with the given ID doesn't exist + return {'error': f'Species with ID {species_id} not found'} return {'message': 'Annotation updated successfully', 'id': annotation_instance.pk} else: @@ -475,3 +554,90 @@ def delete_annotation(request, recording_id: int, id: int): return {'error': 'Recording not found'} except Annotations.DoesNotExist: return {'error': 'Annotation not found'} + + +# TEMPORAL ANNOTATIONS + + +@router.get('/{id}/temporal-annotations') +def get_temporal_annotations(request: HttpRequest, id: int): + try: + recording = Recording.objects.get(pk=id) + + # Check if the user owns the recording or if the recording is public + if recording.owner == request.user or recording.public: + # Query annotations associated with the recording that are owned by the current user + annotations_qs = TemporalAnnotations.objects.filter( + recording=recording, owner=request.user + ) + + # Serialize the annotations using AnnotationSchema + annotations_data = [ + TemporalAnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict() + for annotation in annotations_qs + ] + + return annotations_data + else: + return { + 'error': 'Permission denied. You do not own this recording, and it is not public.' + } + + except Recording.DoesNotExist: + return {'error': 'Recording not found'} + + +@router.put('/{id}/temporal-annotations') +def put_temporal_annotation( + request, + id: int, + annotation: TemporalAnnotationSchema, + species_ids: list[int] | None, +): + try: + recording = Recording.objects.get(pk=id) + if recording.owner == request.user or recording.public: + # Create a new annotation + new_annotation = TemporalAnnotations.objects.create( + recording=recording, + owner=request.user, + start_time=annotation.start_time, + end_time=annotation.end_time, + type=annotation.type, + comments=annotation.comments, + ) + + return {'message': 'Annotation added successfully', 'id': new_annotation.pk} + else: + return { + 'error': 'Permission denied. You do not own this recording, and it is not public.' + } + + except Recording.DoesNotExist: + return {'error': 'Recording not found'} + + +@router.delete('/{recording_id}/temporal-annotations/{id}') +def delete_temporal_annotation(request, recording_id: int, id: int): + try: + recording = Recording.objects.get(pk=recording_id) + + # Check if the user owns the recording or if the recording is public + if recording.owner == request.user or recording.public: + annotation_instance = TemporalAnnotations.objects.get( + pk=id, recording=recording, owner=request.user + ) + + # Delete the annotation + annotation_instance.delete() + + return {'message': 'Annotation deleted successfully'} + else: + return { + 'error': 'Permission denied. You do not own this recording, and it is not public.' + } + + except Recording.DoesNotExist: + return {'error': 'Recording not found'} + except Annotations.DoesNotExist: + return {'error': 'Annotation not found'} diff --git a/bats_ai/core/views/temporal_annotations.py b/bats_ai/core/views/temporal_annotations.py new file mode 100644 index 0000000..0495acb --- /dev/null +++ b/bats_ai/core/views/temporal_annotations.py @@ -0,0 +1,66 @@ +from django.http import HttpRequest +from ninja import Schema +from ninja.pagination import RouterPaginated + +from bats_ai.core.models import Annotations, Recording, TemporalAnnotations +from bats_ai.core.views.species import SpeciesSchema + +router = RouterPaginated() + + +class TemporalAnnotationSchema(Schema): + id: int + start_time: int + end_time: int + type: str + comments: str + species: list[SpeciesSchema] | None + owner_email: str = None + + @classmethod + def from_orm(cls, obj, owner_email=None, **kwargs): + return cls( + start_time=obj.start_time, + end_time=obj.end_time, + type=obj.type, + species=[SpeciesSchema.from_orm(species) for species in obj.species.all()], + comments=obj.comments, + id=obj.id, + owner_email=owner_email, # Include owner_email in the schema + ) + + +class UpdateTemporalAnnotationSchema(Schema): + start_time: int = None + end_time: int = None + type: str | None = None + comments: str | None = None + + +@router.get('/{id}') +def get_temporal_annotation(request: HttpRequest, id: int): + try: + annotation = Annotations.objects.get(pk=id) + recording = annotation.recording + + # Check if the user owns the recording or if the recording is public + if recording.owner == request.user or recording.public: + # Query annotations associated with the recording that are owned by the current user + annotations_qs = TemporalAnnotations.objects.filter( + recording=recording, owner=request.user + ) + + # Serialize the annotations using AnnotationSchema + annotations_data = [ + TemporalAnnotationSchema.from_orm(annotation, owner_email=request.user.email).dict() + for annotation in annotations_qs + ] + + return annotations_data + else: + return { + 'error': 'Permission denied. You do not own this annotation, or the associated recording is not public.' + } + + except Recording.DoesNotExist: + return {'error': 'Recording not found'} diff --git a/client/src/App.vue b/client/src/App.vue index c31c389..43b0aa9 100644 --- a/client/src/App.vue +++ b/client/src/App.vue @@ -1,18 +1,21 @@ + + + + diff --git a/client/src/components/UploadRecording.vue b/client/src/components/UploadRecording.vue index 8f71304..75bf020 100644 --- a/client/src/components/UploadRecording.vue +++ b/client/src/components/UploadRecording.vue @@ -291,7 +291,7 @@ export default defineComponent({ diff --git a/client/src/components/geoJS/LayerManager.vue b/client/src/components/geoJS/LayerManager.vue index 8a1ad46..ef0d3f1 100644 --- a/client/src/components/geoJS/LayerManager.vue +++ b/client/src/components/geoJS/LayerManager.vue @@ -1,13 +1,15 @@ + + diff --git a/client/src/views/Recordings.vue b/client/src/views/Recordings.vue index 37bc374..944d9cb 100644 --- a/client/src/views/Recordings.vue +++ b/client/src/views/Recordings.vue @@ -119,7 +119,7 @@ export default defineComponent({ id: item.id, }; if (item.recording_location) { - const [ lat, lon ] = item.recording_location.coordinates; + const [ lon, lat ] = item.recording_location.coordinates; editingRecording.value['location'] = {lat, lon}; } uploadDialog.value = true; diff --git a/client/src/views/Spectrogram.vue b/client/src/views/Spectrogram.vue index 3a7b421..42b96af 100644 --- a/client/src/views/Spectrogram.vue +++ b/client/src/views/Spectrogram.vue @@ -5,10 +5,9 @@ import { getAnnotations, getSpectrogram, Species, - SpectrogramAnnotation, getSpectrogramCompressed, - OtherUserAnnotations, getOtherUserAnnotations, + getTemporalAnnotations, } from "../api/api"; import SpectrogramViewer from "../components/SpectrogramViewer.vue"; import { SpectroInfo } from "../components/geoJS/geoJSUtils"; @@ -39,12 +38,14 @@ export default defineComponent({ createColorScale, currentUser, annotationState, + annotations, + temporalAnnotations, + otherUserAnnotations, + selectedId, + selectedType, } = useState(); const image: Ref = ref(new Image()); const spectroInfo: Ref = ref(); - const annotations: Ref = ref([]); - const otherUserAnnotations: Ref = ref({}); - const selectedId: Ref = ref(null); const selectedUsers: Ref = ref([]); const speciesList: Ref = ref([]); const loadedImage = ref(false); @@ -53,6 +54,8 @@ export default defineComponent({ const getAnnotationsList = async (annotationId?: number) => { const response = await getAnnotations(props.id); annotations.value = response.data.sort((a, b) => a.start_time - b.start_time); + const tempResp = await getTemporalAnnotations(props.id); + temporalAnnotations.value = tempResp.data.sort((a, b) => a.start_time - b.start_time); if (annotationId !== undefined) { selectedId.value = annotationId; } @@ -83,7 +86,8 @@ export default defineComponent({ : await getSpectrogram(props.id); image.value.src = `data:image/png;base64,${response.data["base64_spectrogram"]}`; spectroInfo.value = response.data["spectroInfo"]; - annotations.value = response.data["annotations"]?.sort((a, b) => a.start_time - b.start_time); + annotations.value = response.data["annotations"]?.sort((a, b) => a.start_time - b.start_time) || []; + temporalAnnotations.value = response.data["temporal"]?.sort((a, b) => a.start_time - b.start_time) || []; if (response.data.currentUser) { currentUser.value = response.data.currentUser; } @@ -101,12 +105,18 @@ export default defineComponent({ selectedId.value = annotationId; }; const selectedAnnotation = computed(() => { - if (selectedId.value !== null && annotations.value) { + if (selectedId.value !== null && selectedType.value === 'pulse' && annotations.value) { const found = annotations.value.findIndex((item) => item.id === selectedId.value); if (found !== -1) { return annotations.value[found]; } } + if (selectedId.value !== null && selectedType.value === 'sequence' && temporalAnnotations.value) { + const found = temporalAnnotations.value.findIndex((item) => item.id === selectedId.value); + if (found !== -1) { + return temporalAnnotations.value[found]; + } + } return null; }); watch(gridEnabled, () => { @@ -153,6 +163,11 @@ export default defineComponent({ setSelectedUsers(selectedUsers.value); }); + const processSelection = ({id, annotationType}: { id: number, annotationType: 'pulse' | 'sequence'}) => { + selectedId.value = id; + selectedType.value = annotationType; + }; + return { annotationState, compressed, @@ -161,11 +176,13 @@ export default defineComponent({ spectroInfo, annotations, selectedId, + selectedType, setSelection, getAnnotationsList, setParentGeoViewer, setHoverData, toggleLayerVisibility, + processSelection, speciesList, selectedAnnotation, parentGeoViewerRef, @@ -175,6 +192,7 @@ export default defineComponent({ freqRef, // Other user selection otherUserAnnotations, + temporalAnnotations, otherUsers, selectedUsers, deleteChip, @@ -318,9 +336,7 @@ export default defineComponent({ :image="image" :spectro-info="spectroInfo" :recording-id="id" - :annotations="annotations" :other-user-annotations="otherUserAnnotations" - :selected-id="selectedId" :grid="gridEnabled" @selected="setSelection($event)" @create:annotation="getAnnotationsList($event)" @@ -333,9 +349,6 @@ export default defineComponent({ :image="image" :spectro-info="spectroInfo" :recording-id="id" - :annotations="annotations" - :other-user-annotations="otherUserAnnotations" - :selected-id="selectedId" :parent-geo-viewer-ref="parentGeoViewerRef" @selected="setSelection($event)" /> @@ -343,9 +356,9 @@ export default defineComponent({ Date: Tue, 20 Feb 2024 13:26:18 -0500 Subject: [PATCH 22/40] prod update --- docker-compose.prod.yml | 70 ++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7bff558..957c5b0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,40 +2,40 @@ version: '3.8' services: # COMMENTED OUT UNTIL READY TO TEST - # traefik: - # restart: always - # image: traefik:v2.4 - # container_name: traefik - # command: > - # --providers.docker=true - # --providers.docker.exposedByDefault=false - # --entrypoints.web.address=:80 - # --log.level=${LOG_LEVEL:-DEBUG} - # --providers.docker=true - # --providers.docker.exposedByDefault=false - # --providers.file.filename=/var/traefik/dynamic.yml - # --entrypoints.web.address=:80 - # --entrypoints.websecure.address=:443 - # --entrypoints.websecure.http.tls.certresolver=myresolver - # --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} - # --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json - # --certificatesresolvers.myresolver.acme.httpchallenge=true - # --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web - # --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} - # labels: - # # Traefik HTTPS Redirect - # - "traefik.enable=true" - # - "traefik.http.routers.http-catchall.entrypoints=web" - # - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" - # - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - # - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" - # volumes: - # - "${SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock" - # - "./traefik/letsencrypt:/letsencrypt" - # - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" - # ports: - # - "80:80" - # - "443:443" + traefik: + restart: always + image: traefik:v2.4 + container_name: traefik + command: > + --providers.docker=true + --providers.docker.exposedByDefault=false + --entrypoints.web.address=:80 + --log.level=${LOG_LEVEL:-DEBUG} + --providers.docker=true + --providers.docker.exposedByDefault=false + --providers.file.filename=/var/traefik/dynamic.yml + --entrypoints.web.address=:80 + --entrypoints.websecure.address=:443 + --entrypoints.websecure.http.tls.certresolver=myresolver + --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-Bryon.Lewis@kitware.com} + --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json + --certificatesresolvers.myresolver.acme.httpchallenge=true + --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web + --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} + labels: + # Traefik HTTPS Redirect + - "traefik.enable=true" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" + - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" + volumes: + - "${SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock" + - "./traefik/letsencrypt:/letsencrypt" + - "./traefik/dynamic.yml:/var/traefik/dynamic.yml:ro" + ports: + - "80:80" + - "443:443" django: build: @@ -94,7 +94,7 @@ services: - "traefik.http.routers.client-rtr.entrypoints=websecure" - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - - "traefik.http.services.client-svc.loadbalancer.server.port=90" + - "traefik.http.services.client-svc.loadbalancer.server.port=80" - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: image: postgis/postgis:latest From e0d2d65f01f46f85aeff9d4aef989293fdf7d60e Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 13:45:02 -0500 Subject: [PATCH 23/40] updating hostname --- bats_ai/settings.py | 3 ++- client/.env.production | 4 ++-- dev/.env.prod.docker-compose | 4 ++-- docker-compose.prod.yml | 3 +-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index d83fdbc..88d65df 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -95,11 +95,12 @@ class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' - ALLOWED_HOSTS = ['batai.kitware.com'] + ALLOWED_HOSTS = ['batdetectai.kitware.com'] class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): pass + class HerokuProductionConfiguration(BatsAiMixin, HerokuProductionBaseConfiguration): pass diff --git a/client/.env.production b/client/.env.production index 84eebba..e8484af 100644 --- a/client/.env.production +++ b/client/.env.production @@ -1,3 +1,3 @@ -VUE_APP_API_ROOT=http://batai.kitware.com/api/v1 -VUE_APP_OAUTH_API_ROOT=http://batai.kitware.com/oauth/ +VUE_APP_API_ROOT=http://batdetectai.kitware.com/api/v1 +VUE_APP_OAUTH_API_ROOT=http://batdetectai.kitware.com/oauth/ VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose index 9fe1865..3e265a5 100644 --- a/dev/.env.prod.docker-compose +++ b/dev/.env.prod.docker-compose @@ -8,6 +8,6 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 -DJANGO_CORS_ORIGIN_WHITELIST=http://batai.kitware.com +DJANGO_CORS_ORIGIN_WHITELIST=http://batdetectai.kitware.com DJANGO_EMAIL_URL=submission://USER:PASSWORD@smtp.sendgrid.com -DJANGO_ALLOWED_HOSTS=['batai.kitware.com'] +DJANGO_ALLOWED_HOSTS=['batdetectai.kitware.com'] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 957c5b0..8521731 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,7 +26,7 @@ services: # Traefik HTTPS Redirect - "traefik.enable=true" - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.rule=Host('batai.kitware.com')" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -95,7 +95,6 @@ services: - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" - "traefik.enable=true" - "traefik.http.services.client-svc.loadbalancer.server.port=80" - - "traefik.http.routers.client-rtr.rule=HostRegexp(`{catchall:.*}`)" postgres: image: postgis/postgis:latest environment: From f75d9ae1a7d053811ee8bb63bb169efa976c88be Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 13:49:55 -0500 Subject: [PATCH 24/40] updating hostname --- docker-compose.prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8521731..42c5183 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,7 +26,7 @@ services: # Traefik HTTPS Redirect - "traefik.enable=true" - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=Host('batai.kitware.com')" + - "traefik.http.routers.http-catchall.rule=Host('batdetectai.kitware.com')" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -92,7 +92,7 @@ services: - django labels: - "traefik.http.routers.client-rtr.entrypoints=websecure" - - "traefik.http.routers.client-rtr.rule=Host(`batai.kitware.com`)" + - "traefik.http.routers.client-rtr.rule=Host(`batdetectai.kitware.com`)" - "traefik.enable=true" - "traefik.http.services.client-svc.loadbalancer.server.port=80" postgres: From 3cbeba1e4c619603a226124f37f80d80b2fae15b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:15:28 -0500 Subject: [PATCH 25/40] traefik routing --- docker-compose.prod.yml | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 42c5183..8ca74bf 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -6,12 +6,12 @@ services: restart: always image: traefik:v2.4 container_name: traefik + networks: + - django-nginx command: > --providers.docker=true --providers.docker.exposedByDefault=false - --entrypoints.web.address=:80 --log.level=${LOG_LEVEL:-DEBUG} - --providers.docker=true --providers.docker.exposedByDefault=false --providers.file.filename=/var/traefik/dynamic.yml --entrypoints.web.address=:80 @@ -26,7 +26,7 @@ services: # Traefik HTTPS Redirect - "traefik.enable=true" - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=Host('batdetectai.kitware.com')" + - "traefik.http.routers.http-catchall.rule=Host(`batdetectai.kitware.com`)" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -58,6 +58,33 @@ services: - postgres - rabbitmq - minio + labels: + - "traefik.enable=true" + + # Routing rule for /api + - "traefik.http.routers.django-api.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/api`)" + - "traefik.http.routers.django-api.service=django" + - "traefik.http.routers.django-api.middlewares=django-middleware" + + # Routing rule for /admin + - "traefik.http.routers.django-admin.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/admin`)" + - "traefik.http.routers.django-admin.service=django" + - "traefik.http.routers.django-admin.middlewares=django-middleware" + + # Routing rule for /accounts + - "traefik.http.routers.django-accounts.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/accounts`)" + - "traefik.http.routers.django-accounts.service=django" + - "traefik.http.routers.django-accounts.middlewares=django-middleware" + + # Routing rule for /oauth + - "traefik.http.routers.django-oauth.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/oauth`)" + - "traefik.http.routers.django-oauth.service=django" + - "traefik.http.routers.django-oauth.middlewares=django-middleware" + + # Routing rule for /static + - "traefik.http.routers.django-static.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/static`)" + - "traefik.http.routers.django-static.service=django" + - "traefik.http.routers.django-static.middlewares=django-middleware" celery: build: context: . @@ -97,6 +124,7 @@ services: - "traefik.http.services.client-svc.loadbalancer.server.port=80" postgres: image: postgis/postgis:latest + env_file: ./dev/.env.prod.docker-compose environment: - POSTGRES_DB=${DJANGO_DATABASE_NAME:-django} - POSTGRES_PASSWORD=${DJANGO_MINIO_STORAGE_SECRET_KEY:-postgres} @@ -108,6 +136,7 @@ services: - postgres:/var/lib/postgresql/data rabbitmq: + env_file: ./dev/.env.prod.docker-compose image: rabbitmq:management networks: - django-nginx @@ -122,6 +151,7 @@ services: # When run with a TTY, minio prints credentials on startup tty: true command: ["server", "/data", "--console-address", ":${DOCKER_MINIO_CONSOLE_PORT-9001}"] + env_file: ./dev/.env.prod.docker-compose environment: - MINIO_ROOT_USER=${DJANGO_MINIO_STORAGE_ACCESS_KEY:-minioAccessKey} - MINIO_ROOT_PASSWORD=${DJANGO_DATABASE_PASSWORD:-minioSecretKey} @@ -134,6 +164,7 @@ services: - minio:/data flower: + env_file: ./dev/.env.prod.docker-compose build: context: . dockerfile: ./dev/django.Dockerfile @@ -143,7 +174,6 @@ services: "flower" ] tty: false - env_file: ./dev/.env.prod.docker-compose volumes: - .:/opt/django-project networks: From 4037929151792590947a93cfeb6ed47491decbb4 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:19:35 -0500 Subject: [PATCH 26/40] update nginx --- docker-compose.prod.yml | 5 ++++ nginx/nginx.conf | 54 +---------------------------------------- 2 files changed, 6 insertions(+), 53 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8ca74bf..ac39ad6 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -62,26 +62,31 @@ services: - "traefik.enable=true" # Routing rule for /api + - "traefik.http.routers.django-api.entrypoints=websecure" - "traefik.http.routers.django-api.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/api`)" - "traefik.http.routers.django-api.service=django" - "traefik.http.routers.django-api.middlewares=django-middleware" # Routing rule for /admin + - "traefik.http.routers.django-admin.entrypoints=websecure" - "traefik.http.routers.django-admin.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/admin`)" - "traefik.http.routers.django-admin.service=django" - "traefik.http.routers.django-admin.middlewares=django-middleware" # Routing rule for /accounts + - "traefik.http.routers.django-accounts.entrypoints=websecure" - "traefik.http.routers.django-accounts.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/accounts`)" - "traefik.http.routers.django-accounts.service=django" - "traefik.http.routers.django-accounts.middlewares=django-middleware" # Routing rule for /oauth + - "traefik.http.routers.django-oauth.entrypoints=websecure" - "traefik.http.routers.django-oauth.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/oauth`)" - "traefik.http.routers.django-oauth.service=django" - "traefik.http.routers.django-oauth.middlewares=django-middleware" # Routing rule for /static + - "traefik.http.routers.django-static.entrypoints=websecure" - "traefik.http.routers.django-static.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/static`)" - "traefik.http.routers.django-static.service=django" - "traefik.http.routers.django-static.middlewares=django-middleware" diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 274fa8a..710ba91 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -18,58 +18,6 @@ http { root /usr/share/nginx/html; index index.html; try_files $uri $uri/ /index.html; - } - # Proxy API requests to Django - location /api { - proxy_pass http://django:8000/api; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - location /oauth { - proxy_pass http://django:8000/oauth; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - location /static { - proxy_pass http://django:8000/static; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - - location /admin { - proxy_pass http://django:8000/admin; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - location /accounts { - proxy_pass http://django:8000/accounts; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - - location /django-storage { - proxy_pass http://minio:9000/django-storage; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - + } } } From 46cf8bf49c941486bc2392daf280f353bb59869b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:29:59 -0500 Subject: [PATCH 27/40] updating traefik --- docker-compose.prod.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index ac39ad6..53c1518 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -61,35 +61,33 @@ services: labels: - "traefik.enable=true" + # Middleware for HTTPS redirection + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + # Routing rule for /api - - "traefik.http.routers.django-api.entrypoints=websecure" - "traefik.http.routers.django-api.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/api`)" - "traefik.http.routers.django-api.service=django" - - "traefik.http.routers.django-api.middlewares=django-middleware" + - "traefik.http.routers.django-api.middlewares=redirect-to-https" # Routing rule for /admin - - "traefik.http.routers.django-admin.entrypoints=websecure" - "traefik.http.routers.django-admin.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/admin`)" - "traefik.http.routers.django-admin.service=django" - - "traefik.http.routers.django-admin.middlewares=django-middleware" + - "traefik.http.routers.django-admin.middlewares=redirect-to-https" # Routing rule for /accounts - - "traefik.http.routers.django-accounts.entrypoints=websecure" - "traefik.http.routers.django-accounts.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/accounts`)" - "traefik.http.routers.django-accounts.service=django" - - "traefik.http.routers.django-accounts.middlewares=django-middleware" + - "traefik.http.routers.django-accounts.middlewares=redirect-to-https" # Routing rule for /oauth - - "traefik.http.routers.django-oauth.entrypoints=websecure" - "traefik.http.routers.django-oauth.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/oauth`)" - "traefik.http.routers.django-oauth.service=django" - - "traefik.http.routers.django-oauth.middlewares=django-middleware" + - "traefik.http.routers.django-oauth.middlewares=redirect-to-https" # Routing rule for /static - - "traefik.http.routers.django-static.entrypoints=websecure" - "traefik.http.routers.django-static.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/static`)" - "traefik.http.routers.django-static.service=django" - - "traefik.http.routers.django-static.middlewares=django-middleware" + - "traefik.http.routers.django-static.middlewares=redirect-to-https" celery: build: context: . @@ -118,8 +116,6 @@ services: dockerfile: ./dev/client.Dockerfile networks: - django-nginx - ports: - - 90:80 depends_on: - django labels: From 05853fbea716ac7e3fa72a196858cf2b40745a1e Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:33:13 -0500 Subject: [PATCH 28/40] reverting to nginx proxy --- docker-compose.prod.yml | 30 ----------------------- nginx/nginx.conf | 54 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 53c1518..802c0b7 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -58,36 +58,6 @@ services: - postgres - rabbitmq - minio - labels: - - "traefik.enable=true" - - # Middleware for HTTPS redirection - - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" - - # Routing rule for /api - - "traefik.http.routers.django-api.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/api`)" - - "traefik.http.routers.django-api.service=django" - - "traefik.http.routers.django-api.middlewares=redirect-to-https" - - # Routing rule for /admin - - "traefik.http.routers.django-admin.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/admin`)" - - "traefik.http.routers.django-admin.service=django" - - "traefik.http.routers.django-admin.middlewares=redirect-to-https" - - # Routing rule for /accounts - - "traefik.http.routers.django-accounts.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/accounts`)" - - "traefik.http.routers.django-accounts.service=django" - - "traefik.http.routers.django-accounts.middlewares=redirect-to-https" - - # Routing rule for /oauth - - "traefik.http.routers.django-oauth.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/oauth`)" - - "traefik.http.routers.django-oauth.service=django" - - "traefik.http.routers.django-oauth.middlewares=redirect-to-https" - - # Routing rule for /static - - "traefik.http.routers.django-static.rule=Host(`batdetectai.kitware.com`) && PathPrefix(`/static`)" - - "traefik.http.routers.django-static.service=django" - - "traefik.http.routers.django-static.middlewares=redirect-to-https" celery: build: context: . diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 710ba91..274fa8a 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -18,6 +18,58 @@ http { root /usr/share/nginx/html; index index.html; try_files $uri $uri/ /index.html; - } + } + # Proxy API requests to Django + location /api { + proxy_pass http://django:8000/api; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + location /oauth { + proxy_pass http://django:8000/oauth; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + location /static { + proxy_pass http://django:8000/static; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + + location /admin { + proxy_pass http://django:8000/admin; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + location /accounts { + proxy_pass http://django:8000/accounts; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + + location /django-storage { + proxy_pass http://minio:9000/django-storage; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + } } From beced4da6e3d619b25b1f24b8224ff8e002ee5d8 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:41:43 -0500 Subject: [PATCH 29/40] issuer cert --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 802c0b7..e47dd0b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -21,7 +21,7 @@ services: --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json --certificatesresolvers.myresolver.acme.httpchallenge=true --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web - --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory} + --certificatesresolvers.myresolver.acme.caserver=${ACME_CA_SERVER:-https://acme-v02.api.letsencrypt.org/directory} labels: # Traefik HTTPS Redirect - "traefik.enable=true" From 2904eb19a43fb71fd04e00e7a4e453b46712c3ae Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:45:39 -0500 Subject: [PATCH 30/40] trusted origins --- bats_ai/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 88d65df..6d7a9ec 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -96,6 +96,7 @@ class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' ALLOWED_HOSTS = ['batdetectai.kitware.com'] + CSRF_TRUSTED_ORIGINS = ["https://batdetectai.kitware.com", "https://www.batdetectai.kitware.com"] class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): From 99e08b6908069ddd342fd766a5bd20b0c3d739a4 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:54:29 -0500 Subject: [PATCH 31/40] add https mixin to configuration --- bats_ai/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 6d7a9ec..e8d7e4d 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -10,7 +10,7 @@ ProductionBaseConfiguration, TestingBaseConfiguration, ) -from composed_configuration._configuration import _BaseConfiguration +from composed_configuration._configuration import _BaseConfiguration, HttpsMixin from configurations import values @@ -77,7 +77,7 @@ class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): pass -class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): +class KitwareConfiguration(BatsAiMixin, HttpsMixin, _BaseConfiguration): SECRET_KEY = 'secretkey' # Dummy value for local development configuration EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' From a40fab8f01015caa03f5178ec60b54d683fe52f8 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 15:55:44 -0500 Subject: [PATCH 32/40] update configuration --- client/.env.production | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/.env.production b/client/.env.production index e8484af..0eb0b2c 100644 --- a/client/.env.production +++ b/client/.env.production @@ -1,3 +1,3 @@ -VUE_APP_API_ROOT=http://batdetectai.kitware.com/api/v1 -VUE_APP_OAUTH_API_ROOT=http://batdetectai.kitware.com/oauth/ +VUE_APP_API_ROOT=https://batdetectai.kitware.com/api/v1 +VUE_APP_OAUTH_API_ROOT=https://batdetectai.kitware.com/oauth/ VUE_APP_OAUTH_CLIENT_ID=HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP From bd62f60111b117e5e66cfbd14840d92beadf7302 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 16:29:33 -0500 Subject: [PATCH 33/40] remove https --- bats_ai/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bats_ai/settings.py b/bats_ai/settings.py index e8d7e4d..c780ed7 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -77,7 +77,7 @@ class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): pass -class KitwareConfiguration(BatsAiMixin, HttpsMixin, _BaseConfiguration): +class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): SECRET_KEY = 'secretkey' # Dummy value for local development configuration EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' From 26f116c85ca66c20844200850f884182d31a552f Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 17:03:04 -0500 Subject: [PATCH 34/40] refactoring prod .env files --- DEPLOYMENT.md | 33 +++++++++++++++++++ ...pose => .env.prod.docker-compose.template} | 0 docker-compose.prod.yml | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 DEPLOYMENT.md rename dev/{.env.prod.docker-compose => .env.prod.docker-compose.template} (100%) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..6f5c29a --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,33 @@ +# bats-ai + +## Deployment with Docker (recommended quickstart) + +This was a bit rushed so the deployment utilizes a single docker file `docker-compose.prod.yml` in the root of the directory + +I wanted some simple instructions below to configure the deployment + +Be sure to use the proper hostname (batdetectai.kitware.com) in all locations that require it. + +## Docker Compose Differences + +I created a `client` service which has it's own Dockerfile and builds the vue client app. The `client` service also uses a reverse proxy to route `/api`, `/admin` fields to the django server. +The client will need to be built with a different Client ID for accessing the server. + +### Initial Setup for Deployment + +1. Run `docker compose run --rm django ./manage.py migrate` +2. Run `docker compose run --rm django ./manage.py createsuperuser` + and follow the prompts to create your own user +3. Run `docker compose run --rm django ./manage.py makeclient \ + --username your.super.user@email.address \ + --uri https://batdetectai.kitware.com/` +4. Run `docker compose run --rm django ./manage.py collectstatic` to collect the static files +5. Run `docker compose -f docker-compose.prod.yml up` to start the server add `-d` for a silent version to run in the background +6. Copy over the ./dev/.env.prod.docker-compose.template to `./dev/.env.prod.docker-compose.template` and change the default passwords +7. Change the ID in the `./client/env.production` to a custom ID +8. After creating the basic application log into the django admin `batdetectai.kitware.com/admin` and change the ApplicationId to the ID in the `./client.env.production` +9. Test logging in/out and uploading data to the server. + +### system.d service + +Service that will automatically start and launch the server \ No newline at end of file diff --git a/dev/.env.prod.docker-compose b/dev/.env.prod.docker-compose.template similarity index 100% rename from dev/.env.prod.docker-compose rename to dev/.env.prod.docker-compose.template diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e47dd0b..61dbd63 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -11,7 +11,7 @@ services: command: > --providers.docker=true --providers.docker.exposedByDefault=false - --log.level=${LOG_LEVEL:-DEBUG} + --log.level=${LOG_LEVEL:-WARN} --providers.docker.exposedByDefault=false --providers.file.filename=/var/traefik/dynamic.yml --entrypoints.web.address=:80 From e5d2344a372e2b946d0db3d80b610f7c6ae5fbdc Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 19:56:39 -0500 Subject: [PATCH 35/40] refactoring environment files --- DEPLOYMENT.md | 18 ++++++++++++------ bats_ai/settings.py | 11 +++++++---- client/README.md | 4 ++-- dev/.env.prod.docker-compose.template | 6 +++--- docker-compose.prod.yml | 6 ++++-- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 6f5c29a..ca7483c 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -2,16 +2,22 @@ ## Deployment with Docker (recommended quickstart) -This was a bit rushed so the deployment utilizes a single docker file `docker-compose.prod.yml` in the root of the directory +This was a bit rushed so the deployment utilizes a single +docker file `docker-compose.prod.yml` in the root of the directory I wanted some simple instructions below to configure the deployment -Be sure to use the proper hostname (batdetectai.kitware.com) in all locations that require it. +Be sure to use the proper hostname (batdetectai.kitware.com) in +all locations that require it. ## Docker Compose Differences -I created a `client` service which has it's own Dockerfile and builds the vue client app. The `client` service also uses a reverse proxy to route `/api`, `/admin` fields to the django server. -The client will need to be built with a different Client ID for accessing the server. +I created a `client` service which has it's own Dockerfile and +builds the vue client app. +The `client` service also uses a reverse proxy to route +`/api`, `/admin` fields to the django server. +The client will need to be built with a different Client ID +for accessing the server. ### Initial Setup for Deployment @@ -21,7 +27,7 @@ The client will need to be built with a different Client ID for accessing the se 3. Run `docker compose run --rm django ./manage.py makeclient \ --username your.super.user@email.address \ --uri https://batdetectai.kitware.com/` -4. Run `docker compose run --rm django ./manage.py collectstatic` to collect the static files +4. Run `docker compose run --rm django ./manage.py collectstatic`to collect the static files 5. Run `docker compose -f docker-compose.prod.yml up` to start the server add `-d` for a silent version to run in the background 6. Copy over the ./dev/.env.prod.docker-compose.template to `./dev/.env.prod.docker-compose.template` and change the default passwords 7. Change the ID in the `./client/env.production` to a custom ID @@ -30,4 +36,4 @@ The client will need to be built with a different Client ID for accessing the se ### system.d service -Service that will automatically start and launch the server \ No newline at end of file +Service that will automatically start and launch the server diff --git a/bats_ai/settings.py b/bats_ai/settings.py index c780ed7..97925a5 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path from composed_configuration import ( @@ -10,7 +11,7 @@ ProductionBaseConfiguration, TestingBaseConfiguration, ) -from composed_configuration._configuration import _BaseConfiguration, HttpsMixin +from composed_configuration._configuration import _BaseConfiguration from configurations import values @@ -78,7 +79,8 @@ class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): - SECRET_KEY = 'secretkey' # Dummy value for local development configuration + SECRET_KEY = values.SecretValue() + baseHost = os.environ['SERVERHOSTNAME'] EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' MINIO_STORAGE_ENDPOINT = values.Value( @@ -95,8 +97,9 @@ class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY = 'READ_WRITE' MINIO_STORAGE_MEDIA_USE_PRESIGNED = True MINIO_STORAGE_MEDIA_URL = 'http://127.0.0.1:9000/django-storage' - ALLOWED_HOSTS = ['batdetectai.kitware.com'] - CSRF_TRUSTED_ORIGINS = ["https://batdetectai.kitware.com", "https://www.batdetectai.kitware.com"] + ALLOWED_HOSTS = [baseHost] + CSRF_TRUSTED_ORIGINS = [f'https://{baseHost}', f'https://{baseHost}'] + CORS_ORIGIN_WHITELIST = [f'https://{baseHost}', f'https://{baseHost}'] class ProductionConfiguration(BatsAiMixin, ProductionBaseConfiguration): diff --git a/client/README.md b/client/README.md index 35a6427..4bf6c06 100644 --- a/client/README.md +++ b/client/README.md @@ -30,8 +30,8 @@ git grep CHANGEME ## Recommended IDE Setup -- [VSCode](https://code.visualstudio.com/) -- [Volar](https://marketplace.visualstudio.com/items?itemName=johnsoncodehk.volar) +* [VSCode](https://code.visualstudio.com/) +* [Volar](https://marketplace.visualstudio.com/items?itemName=johnsoncodehk.volar) ## Type Support For `.vue` Imports in TS diff --git a/dev/.env.prod.docker-compose.template b/dev/.env.prod.docker-compose.template index 3e265a5..f03ed11 100644 --- a/dev/.env.prod.docker-compose.template +++ b/dev/.env.prod.docker-compose.template @@ -8,6 +8,6 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_ENDPOINT=minio:9000 -DJANGO_CORS_ORIGIN_WHITELIST=http://batdetectai.kitware.com -DJANGO_EMAIL_URL=submission://USER:PASSWORD@smtp.sendgrid.com -DJANGO_ALLOWED_HOSTS=['batdetectai.kitware.com'] +SERVERHOSTNAME=batdetectai.kitware.com +DJANGO_SECRET_KEY=changeme +ACME_EMAIL=Bryon.Lewis@kitware.com diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 61dbd63..e47f978 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -6,6 +6,7 @@ services: restart: always image: traefik:v2.4 container_name: traefik + env_file: ./dev/.env.prod.docker-compose networks: - django-nginx command: > @@ -26,7 +27,7 @@ services: # Traefik HTTPS Redirect - "traefik.enable=true" - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=Host(`batdetectai.kitware.com`)" + - "traefik.http.routers.http-catchall.rule=Host(`${SERVERHOSTNAME}``)" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -84,13 +85,14 @@ services: build: context: . dockerfile: ./dev/client.Dockerfile + env_file: ./dev/.env.prod.docker-compose networks: - django-nginx depends_on: - django labels: - "traefik.http.routers.client-rtr.entrypoints=websecure" - - "traefik.http.routers.client-rtr.rule=Host(`batdetectai.kitware.com`)" + - "traefik.http.routers.client-rtr.rule=Host(`${SERVERHOSTNAME}`)" - "traefik.enable=true" - "traefik.http.services.client-svc.loadbalancer.server.port=80" postgres: From f31c8415286cce9780dcc846bc666764fc8ef999 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 20:00:34 -0500 Subject: [PATCH 36/40] update docker --- docker-compose.prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e47f978..505ca33 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -27,7 +27,7 @@ services: # Traefik HTTPS Redirect - "traefik.enable=true" - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.rule=Host(`${SERVERHOSTNAME}``)" + - "traefik.http.routers.http-catchall.rule=Host(`${SERVERHOSTNAME:-batdetectai.kitware.com}`)" - "traefik.http.routers.http-catchall.middlewares=redirect-to-https-mddl@docker" - "traefik.http.middlewares.redirect-to-https-mddl.redirectscheme.scheme=https" volumes: @@ -92,7 +92,7 @@ services: - django labels: - "traefik.http.routers.client-rtr.entrypoints=websecure" - - "traefik.http.routers.client-rtr.rule=Host(`${SERVERHOSTNAME}`)" + - "traefik.http.routers.client-rtr.rule=Host(`${SERVERHOSTNAME:-batdetectai.kitware.com}`)" - "traefik.enable=true" - "traefik.http.services.client-svc.loadbalancer.server.port=80" postgres: From 096ce32e972890172a7893ce1b104728444b71d6 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 20 Feb 2024 20:11:16 -0500 Subject: [PATCH 37/40] linting fix --- DEPLOYMENT.md | 12 ++++++++---- bats_ai/settings.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index ca7483c..1e9ce8c 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -27,11 +27,15 @@ for accessing the server. 3. Run `docker compose run --rm django ./manage.py makeclient \ --username your.super.user@email.address \ --uri https://batdetectai.kitware.com/` -4. Run `docker compose run --rm django ./manage.py collectstatic`to collect the static files -5. Run `docker compose -f docker-compose.prod.yml up` to start the server add `-d` for a silent version to run in the background -6. Copy over the ./dev/.env.prod.docker-compose.template to `./dev/.env.prod.docker-compose.template` and change the default passwords +4. Run `docker compose run --rm django ./manage.py collectstatic` + to collect the static files +5. Run `docker compose -f docker-compose.prod.yml up` to start the server + add `-d` for a silent version to run in the background +6. Copy over the ./dev/.env.prod.docker-compose.template + to `./dev/.env.prod.docker-compose.template` and change the default passwords 7. Change the ID in the `./client/env.production` to a custom ID -8. After creating the basic application log into the django admin `batdetectai.kitware.com/admin` and change the ApplicationId to the ID in the `./client.env.production` +8. After creating the basic application log into the django admin `batdetectai.kitware.com/admin` + and change the ApplicationId to the ID in the `./client.env.production` 9. Test logging in/out and uploading data to the server. ### system.d service diff --git a/bats_ai/settings.py b/bats_ai/settings.py index 97925a5..11c29dd 100644 --- a/bats_ai/settings.py +++ b/bats_ai/settings.py @@ -80,7 +80,9 @@ class TestingConfiguration(BatsAiMixin, TestingBaseConfiguration): class KitwareConfiguration(BatsAiMixin, _BaseConfiguration): SECRET_KEY = values.SecretValue() - baseHost = os.environ['SERVERHOSTNAME'] + baseHost = 'batdetectai.kitware.com' + if 'SERVERHOSTNAME' in os.environ: + baseHost = os.environ['SERVERHOSTNAME'] EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FILE_STORAGE = 'minio_storage.storage.MinioMediaStorage' MINIO_STORAGE_ENDPOINT = values.Value( From ae16c2e0742a49855d77e23b0040b107e0c21008 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 21 Feb 2024 07:43:30 -0500 Subject: [PATCH 38/40] add server hostname to environment --- docker-compose.prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 505ca33..234e2a1 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -53,6 +53,8 @@ services: - django-nginx volumes: - .:/opt/django-project + environment: + - SERVERHOSTNAME=${SERVERHOSTNAME:-batdetectai.kitware.com} ports: - 8000:8000 depends_on: From c6078f4c4607fadf86fdfcbb233c827f27f97e11 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 21 Feb 2024 08:28:23 -0500 Subject: [PATCH 39/40] deployment docs update --- DEPLOYMENT.md | 50 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 1e9ce8c..578bb83 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -27,17 +27,55 @@ for accessing the server. 3. Run `docker compose run --rm django ./manage.py makeclient \ --username your.super.user@email.address \ --uri https://batdetectai.kitware.com/` -4. Run `docker compose run --rm django ./manage.py collectstatic` +4. Run `docker compose run --rm django ./manage.py loaddata species` to load species + data into the database +5. Run `docker compose run --rm django ./manage.py collectstatic` to collect the static files -5. Run `docker compose -f docker-compose.prod.yml up` to start the server +6. Run `docker compose -f docker-compose.prod.yml up` to start the server add `-d` for a silent version to run in the background -6. Copy over the ./dev/.env.prod.docker-compose.template +7. Copy over the ./dev/.env.prod.docker-compose.template to `./dev/.env.prod.docker-compose.template` and change the default passwords -7. Change the ID in the `./client/env.production` to a custom ID -8. After creating the basic application log into the django admin `batdetectai.kitware.com/admin` +8. Change the ID in the `./client/env.production` to a custom ID - this will + probably require a `docker compose build` to build the app afterwards +9. After creating the basic application log into the django admin `batdetectai.kitware.com/admin` and change the ApplicationId to the ID in the `./client.env.production` -9. Test logging in/out and uploading data to the server. +10. Test logging in/out and uploading data to the server. ### system.d service Service that will automatically start and launch the server +Create this at `/etc/systemd/system` using sudo + +```systemd +[Unit] +Description=batai-server +Requires=docker.service +After=docker.service + +[Service] +ExecStartPre=/bin/sleep 10 +Environment=PATH=/usr/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin +Restart=always +User=bryon +Group=docker +TimeoutStartSec=300 +RestartSec=20 +WorkingDirectory=/home/bryon/batai +# Shutdown container (if running) when unit is started +ExecStartPre=docker compose down +# Start container when unit is started +ExecStart=docker compose -f docker-compose.prod.yml up +# Stop container when unit is stopped +ExecStop=docker compose down + +[Install] +WantedBy=multi-user.target +``` + +After run `sudo systemctl enable batai.service` +Then to start you can use `sudo systemctl start batai.service` +Stopping: `sudo systemctl stop batai.service` + +# User Management + +There is no email server connected up so users need to be individually approved and their email verified by an admin From a87023c2a08d568f1b4e4c11a073a9d8de80c6e3 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 21 Feb 2024 08:32:37 -0500 Subject: [PATCH 40/40] deployment docs update --- DEPLOYMENT.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 578bb83..1fe1162 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -76,6 +76,7 @@ After run `sudo systemctl enable batai.service` Then to start you can use `sudo systemctl start batai.service` Stopping: `sudo systemctl stop batai.service` -# User Management +### User Management -There is no email server connected up so users need to be individually approved and their email verified by an admin +There is no email server connected up so users need to be +individually approved and their email verified by an admin