Multithreaded send batches

Dockerfile

@@ -1,42 +1,29 @@
 FROM python:3.12
 LABEL maintainer="Miguel Galves <[email protected]>"
 
-# We need sudo and nginx to run
-RUN apt-get update && apt-get -y install sudo nginx emacs gettext
+# Install system dependencies
+RUN apt-get update && apt-get -y install sudo gettext
 
 # Creating our local user and group
-RUN groupadd nginx
-RUN adduser --disabled-password --home /home/wmb --shell /bin/bash wmb 
+RUN adduser --disabled-password --home /home/wmb --shell /bin/bash wmb
 RUN adduser wmb sudo
-RUN adduser wmb nginx
 
 # We dont want any password when running SUDO
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
-# NGINX configuration
-COPY etc/nginx.conf /etc/nginx/sites-enabled/default
-
 # changing to our local user
 USER wmb
-RUN mkdir /home/wmb/logs && chmod 777 /home/wmb/logs && chown wmb:nginx /home/wmb/logs
-RUN mkdir /home/wmb/logs/nginx && chmod 777 /home/wmb/logs/nginx && chown wmb:nginx /home/wmb/logs/nginx
-RUN mkdir /home/wmb/www
-RUN mkdir /home/wmb/www/src
-
-# Needed for nginx
-RUN chmod o+x /home/wmb &&chmod o+x /home/wmb/www
-
-COPY bin/cmd_run.sh /home/wmb/www 
-RUN sudo chown wmb:wmb /home/wmb/www/cmd_run.sh && sudo chmod 777 /home/wmb/www/cmd_run.sh
+RUN mkdir -p /home/wmb/www/{src,static}
 
 COPY requirements.txt /home/wmb/www
-RUN sudo chown wmb:wmb /home/wmb/www/requirements.txt
+RUN sudo chown wmb:wmb -R /home/wmb/www
 
 WORKDIR /home/wmb/www/
 
 RUN pip install -r requirements.txt
 
 ENV PATH="${PATH}:/home/wmb/.local/bin"
+ENV DJANGO_SETTINGS_MODULE=qsts3.settings
+ENV PYTHONPATH="${PYTHONPATH}:/home/wmb/www/src"
 
-EXPOSE 8000 80
-
+WORKDIR /home/wmb/www/src

Makefile

@@ -8,15 +8,17 @@ VERSION ?= $(shell date +"%Y%m%d_%H%M")
 build:
 	docker build -t ${IMAGE} -f Dockerfile .
 
+run:
+	docker-compose up -d
 
-shell: 
-	docker run --rm -ti --env-file etc/env -p 8765:80 -p 8000:8000 -v ${ROOT_DIR}/src:/home/wmb/www/src ${IMAGE} bash
+shell:
+	docker-compose exec app bash
 
-run: 
-	docker run --rm -ti --env-file etc/env -p 8765:80 -p 8000:8000 -v ${ROOT_DIR}/src:/home/wmb/www/src ${IMAGE} /home/wmb/www/cmd_run.sh
+watch:
+	docker-compose logs app -f
 
 test:
-	docker run --rm -ti --env-file etc/env -v ${ROOT_DIR}/src:/home/wmb/www/src ${IMAGE} bash -c "cd src; python3 manage.py test"
+	docker-compose exec app django-admin test
 
 integration:
-	docker run --rm -ti --env-file etc/env -v ${ROOT_DIR}/src:/home/wmb/www/src ${IMAGE} bash -c "cd src; python3 manage.py test integration"
+	docker-compose exec app django-admin test integration

README.md

@@ -7,6 +7,7 @@ Repository for the development of a new version of QuickStatements
 Required tools:
 
 * Docker
+* docker-compose
 * Make
 
 To build the development container
@@ -15,12 +16,6 @@ To build the development container
 > make build
 ```
 
-To run a shell inside the container
-
-```bash
-> make shell
-```
-
 Make sure that you have an env file inside the local etc/ dir. This file contains all the **ENVIRONMENT VARIABLES** used by the system and must never be added to your git repo.
 
 To generate a good secret key you can run with python 3.6+
@@ -29,26 +24,21 @@ To generate a good secret key you can run with python 3.6+
 python -c "import secrets; print(secrets.token_urlsafe())"
 ```
 
-If you are running this container for the first time, you have to initialize the database and create a superuser for the Django ADMIN
+To start all services (database, web server, run a shell inside the container)
 
 ```bash
-> cd src
-> python manage.py migrate
-> python manage.py createsuperuser
+> make run
 ```
 
-Now that everything is set up, we can start **Quickstatements**. We have 2 ways of doing that:
+If everything is correct, **Quickstatements** will be available at http://localhost:8000/
 
-* from inside the container, running 
-```bash 
->./cmd_run.sh
-```
-* from our Makefile, running 
-```bash 
-> make run
+If you are running this for the first time, you have to create a superuser for the Django admin. You can access the container with `make shell`. From there:
+
+```bash
+> django-admin createsuperuser
 ```
 
-Now **Quickstatements** is available at http://localhost:8765/
+will run an interactive command which will ask you for your username, password, email, etc... follow the prompts and you should be able to login to the admin at `http://localhost:8000/admin`
 
 ## Wikibase server
 

docker-compose.yml

@@ -1,12 +1,36 @@
 version: '3.8'
 
+x-application-service: &application-service
+  build:
+    context: ./
+    dockerfile: Dockerfile
+  depends_on:
+    mariadb:
+      condition: service_healthy
+  restart: unless-stopped
+  environment: &application-environment
+    DJANGO_SETTINGS_MODULE: qsts3.settings
+    DB_NAME: quickstatements
+    DB_USER: root
+    DB_PASSWORD: "${DB_ROOT_PASSWORD}"
+    DB_HOST: mariadb
+    STATIC_ROOT: /home/wmb/www/static
+    QSTS_DEBUG: "true"
+  env_file:
+    - ./etc/env
+  volumes:
+    - ./src:/home/wmb/www/src
+    - static-data:/home/wmb/www/static
+
 services:
   mariadb:
     image: mariadb
-    container_name: "${DB_HOST}"
+    restart: unless-stopped
     environment:
+      MARIADB_ROOT_PASSWORD: "${DB_ROOT_PASSWORD}"
+      MARIADB_DATABASE: quickstatements
       MYSQL_ROOT_PASSWORD: "${DB_ROOT_PASSWORD}"
-      MYSQL_DATABASE: "${DB_NAME}"      
+      MYSQL_DATABASE: quickstatements
     expose:
       - "3306"
     volumes:
@@ -17,25 +41,42 @@ services:
       interval: 10s
       timeout: 5s
       retries: 3
-    restart: always
-
-
-  quickstatements:
-    build:
-      context: ./
-      dockerfile: Dockerfile
-    depends_on:
-      mariadb:
-        condition: service_healthy
-    env_file:
-      - ./etc/env
+
+  app:
+    <<: *application-service
+    command: django-admin runserver 0.0.0.0:8000
+
+  run_send_batches:
+    <<: *application-service
+    command: django-admin send_batches
+
+  webserver:
+    image: nginx
+    restart: unless-stopped
+    configs:
+      - source: nginx_config
+        target: /etc/nginx/nginx.conf
+        mode: 0640
     ports:
-      - "8765:80"
-      - "8000:8000"
+      - "8000:80"
     volumes:
-      - ./src:/home/wmb/www/src
-    command: /home/wmb/www/cmd_run.sh
-    restart: always
+      - static-data:/static
+
+  init_collect_static:
+    <<: *application-service
+    command: django-admin collectstatic --no-input
+    restart: on-failure
+
+  init_migrations:
+    <<: *application-service
+    command: django-admin migrate --no-input
+    restart: on-failure
+
+
+configs:
+  nginx_config:
+    file: ./etc/nginx.conf
 
 volumes:
   mariadb_data:
+  static-data:

etc/nginx.conf

@@ -1,22 +1,33 @@
+worker_processes 1;
 
+events {
+  worker_connections 1024;
+}
 
-server {
-    listen 80;
-
-    access_log /home/wmb/logs/nginx/qsts3-access.log;
-    error_log /home/wmb/logs/nginx/qsts3-error.log;
+http {
+  include mime.types;
 
+  server {
+    listen 80;
+    server_name _;
+    server_tokens off;
     client_max_body_size 512m;
     keepalive_requests 100;
     keepalive_timeout 35s;
 
     location / {
-        include uwsgi_params;
-        uwsgi_pass unix:/home/wmb/www/django.sock;
+      proxy_pass http://app:8000;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header Host $host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     }
 
     location /static/ {
-        autoindex off;
-        alias   /home/wmb/www/src/static/;
+      autoindex off;
+      alias   /static/;
     }
+  }
 }

requirements.txt

@@ -1,5 +1,4 @@
 Django==5.0.9
-uwsgi==2.0.26
 requests==2.32.3
 requests-mock==1.10.0
 Authlib==1.3.1

setup.cfg

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 99
+exclude = build,dist,.git,migrations
+
+[isort]
+profile = "black"
+line_length = 99