Merge pull request #188 from WikiMovimentoBrasil/2025-02-mariadb-depl…

…oyment

 Add Docker Compose Deployment with MariaDB

.github/workflows/ci.yml

@@ -10,6 +10,7 @@ env:
   DJANGO_SECRET_KEY: dummysecretkey123
   OAUTH_CLIENT_ID: dummyoauthclientid
   OAUTH_CLIENT_SECRET: dummyoauthclientsecret
+  DB_ENGINE: django.db.backends.sqlite3
 
 jobs:
   build:

bin/cmd_run.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
+
 sudo service nginx start
 cd /home/wmb/www/src && python3 manage.py collectstatic --no-input && chmod -R 777 /home/wmb/www/src/static
 cd /home/wmb/www/src && python3 manage.py migrate --no-input
-cd /home/wmb/www/src && uwsgi --ini app.ini
+cd /home/wmb/www/src && uwsgi --ini app.ini

deploy.sh

@@ -2,21 +2,21 @@
 
 set -eu
 
-cd ~/www/python || exit
+# cd ~/www/python || exit
 
-echo "==> Updating git repository..."
-git pull
+# echo "==> Updating git repository..."
+# git pull
 
-echo "==> Entering python shell..."
-toolforge webservice --backend=kubernetes python3.11 shell -- \
-  webservice-python-bootstrap && \
-  source venv/bin/activate && \
-  echo "==> Installing dependencies..." && \
-  pip install -r requirements.txt && \
-  echo "==> Running migrations..." && \
-  python3 src/manage.py migrate && \
-  echo "==> Collecting static files..." && \
-  python3 src/manage.py collectstatic --noinput
+# echo "==> Entering python shell..."
+# toolforge webservice --backend=kubernetes python3.11 shell -- \
+#   webservice-python-bootstrap && \
+#   source venv/bin/activate && \
+#   echo "==> Installing dependencies..." && \
+#   pip install -r requirements.txt && \
+#   echo "==> Running migrations..." && \
+#   python3 src/manage.py migrate && \
+#   echo "==> Collecting static files..." && \
+#   python3 src/manage.py collectstatic --noinput
 
-echo "==> Restarting webservice..."
-toolforge webservice --backend=kubernetes python3.11 restart
+# echo "==> Restarting webservice..."
+# toolforge webservice --backend=kubernetes python3.11 restart

docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3.8'
+
+services:
+  mariadb:
+    image: mariadb
+    container_name: "${DB_HOST}"
+    environment:
+      MYSQL_ROOT_PASSWORD: "${DB_ROOT_PASSWORD}"
+      MYSQL_DATABASE: "${DB_NAME}"      
+    expose:
+      - "3306"
+    volumes:
+      - mariadb_data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    restart: always
+
+
+  quickstatements:
+    build:
+      context: ./
+      dockerfile: Dockerfile
+    depends_on:
+      mariadb:
+        condition: service_healthy
+    env_file:
+      - ./etc/env
+    ports:
+      - "8765:80"
+      - "8000:8000"
+    volumes:
+      - ./src:/home/wmb/www/src
+    command: /home/wmb/www/cmd_run.sh
+    restart: always
+
+volumes:
+  mariadb_data:

etc/env.SAMPLE

@@ -7,5 +7,13 @@ BASE_REST_URL=<optional>
 # example: https://test.wikidata.org/w/rest.php
 # defaults to: https://www.wikidata.org/w/rest.php
 
+DB_ROOT_PASSWORD=db_root_password
+DB_ENGINE=django.db.backends.mysql
+DB_NAME=db_name
+DB_USER=root
+DB_PASSWORD=db_root_password
+DB_HOST=maria_db
+DB_PORT=3306
+
 TOOLFORGE_TOOL_NAME=
 # only use when deploying to Toolforgre

requirements.txt

@@ -8,4 +8,4 @@ djangorestframework==3.15.2
 markdown==3.7      # Markdown support for the browsable API.
 django-filter==24.3  #
 jsonpatch==1.33
-
+mysqlclient>=2.0.0

src/api/tests/test_batch_command_list_view.py

@@ -36,6 +36,9 @@ def test_batch_command_list_authenticated_request(self):
         self.assertFalse(BatchCommand.objects.count())
         batch = v1.parse("My batch", "myuser", "CREATE||-Q1234|P1|12||Q222|P4|9~0.1")
         batch.save_batch_and_preview_commands()
+        commands = batch.commands()
+        cmd1_pk = commands[0].pk
+        cmd2_pk = commands[1].pk
 
         self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token.key)
 
@@ -51,15 +54,15 @@ def test_batch_command_list_authenticated_request(self):
 
         c0 = data["commands"][0]
         self.assertEqual(c0["index"], 0)
-        self.assertEqual(c0["url"], "http://testserver/api/v1/commands/1")
+        self.assertEqual(c0["url"], f"http://testserver/api/v1/commands/{cmd1_pk}")
         self.assertEqual(c0["action"], "CREATE")
         self.assertEqual(c0["json"], {"action": "create", "type": "item"})
         self.assertEqual(c0["response_json"], {})
         self.assertEqual(c0["status"], 0)
 
         c1 = data["commands"][1]
         self.assertEqual(c1["index"], 1)
-        self.assertEqual(c1["url"], "http://testserver/api/v1/commands/2")
+        self.assertEqual(c1["url"], f"http://testserver/api/v1/commands/{cmd2_pk}")
         self.assertEqual(c1["action"], "REMOVE")
         self.assertEqual(
             c1["json"],
@@ -84,7 +87,7 @@ def test_batch_command_list_paginated_authenticated_request(self):
         response = self.client.get(f"http://testserver/api/v1/batches/{batch.pk}/commands/")
         self.assertEqual(response.status_code, 200)
         data = response.data
-        self.assertEqual(data["links"]["next"], "http://testserver/api/v1/batches/1/commands/?page=2")
+        self.assertEqual(data["links"]["next"], f"http://testserver/api/v1/batches/{batch.pk}/commands/?page=2")
         self.assertEqual(data["links"]["previous"], None)
         self.assertEqual(data["total"], 250)
         self.assertEqual(data["page_size"], 100)
@@ -94,8 +97,8 @@ def test_batch_command_list_paginated_authenticated_request(self):
         self.assertEqual(response.status_code, 200)
         data = response.data
         self.assertEqual(data["batch"], {"pk": batch.pk, "url": f"http://testserver/api/v1/batches/{batch.pk}/"})
-        self.assertEqual(data["links"]["next"], "http://testserver/api/v1/batches/1/commands/?page=3")
-        self.assertEqual(data["links"]["previous"], "http://testserver/api/v1/batches/1/commands/")
+        self.assertEqual(data["links"]["next"], f"http://testserver/api/v1/batches/{batch.pk}/commands/?page=3")
+        self.assertEqual(data["links"]["previous"], f"http://testserver/api/v1/batches/{batch.pk}/commands/")
         self.assertEqual(data["total"], 250)
         self.assertEqual(data["page_size"], 100)
 
@@ -104,7 +107,7 @@ def test_batch_command_list_paginated_authenticated_request(self):
         data = response.data
         self.assertEqual(data["batch"], {"pk": batch.pk, "url": f"http://testserver/api/v1/batches/{batch.pk}/"})
         self.assertEqual(data["links"]["next"], None)
-        self.assertEqual(data["links"]["previous"], "http://testserver/api/v1/batches/1/commands/?page=2")
+        self.assertEqual(data["links"]["previous"], f"http://testserver/api/v1/batches/{batch.pk}/commands/?page=2")
         self.assertEqual(data["total"], 250)
         self.assertEqual(data["page_size"], 50)
 

src/qsdev.cnf

@@ -0,0 +1,7 @@
+[client]
+host = mariadb
+port = 3306
+database = your_database_name
+user = your_user
+password = your_user_password
+default-character-set = utf8

src/qsts3/settings.py

@@ -27,7 +27,7 @@
 SECRET_KEY = os.getenv("DJANGO_SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ["qs-dev.toolforge.org", "localhost"]
 
@@ -83,11 +83,16 @@
 # Database
 # https://docs.djangoproject.com/en/5.0/ref/settings/#databases
 
+# MariaDB setup
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": BASE_DIR / "db.sqlite3",
-    }
+        "ENGINE": os.getenv("DB_ENGINE", "django.db.backends.mysql"),
+        "NAME": os.getenv("DB_NAME", "your_db_name"),
+        "USER": os.getenv("DB_USER", "your_db_user"),
+        "PASSWORD": os.getenv("DB_PASSWORD", "your_db_password"),
+        "HOST": os.getenv("DB_HOST", "mariadb"),
+        "PORT": os.getenv("DB_PORT", 3306),
+    },
 }
 
 

src/web/tests/test_views.py

@@ -594,15 +594,15 @@ def test_batch_report(self, mocker):
         self.assertEqual(response.status_code, 200)
         self.assertTemplateUsed("batch.html")
         self.assertNotInRes(f"""<form method="GET" action="/batch/{pk}/report/">""", response)
-        self.assertNotInRes(f"""<input type="submit" value="Download report">""", response)
+        self.assertNotInRes("""<input type="submit" value="Download report">""", response)
 
         batch.run()
 
         response = self.client.get(f"/batch/{pk}/")
         self.assertEqual(response.status_code, 200)
         self.assertTemplateUsed("batch.html")
         self.assertInRes(f"""<form method="GET" action="/batch/{pk}/report/">""", response)
-        self.assertInRes(f"""<input type="submit" value="Download report">""", response)
+        self.assertInRes("""<input type="submit" value="Download report">""", response)
 
         response = self.client.post(f"/batch/{pk}/report/")
         self.assertEqual(response.status_code, 405)
@@ -612,8 +612,8 @@ def test_batch_report(self, mocker):
         self.assertEqual(response.headers["Content-Disposition"], f'attachment; filename="batch-{pk}-report.csv"')
         result = (
             """b'batch_id,index,operation,status,error,message,entity_id,raw_input\\r\\n"""
-            """1,0,set_statement,Done,,,Q1234,Q1234|P2|Q1\\r\\n"""
-            """1,1,set_label,Done,,,Q11,"Q11|Len|""label""\"\\r\\n\'"""
+            f"""{pk},0,set_statement,Done,,,Q1234,Q1234|P2|Q1\\r\\n"""
+            f"""{pk},1,set_label,Done,,,Q11,"Q11|Len|""label""\"\\r\\n\'"""
         )
         self.assertEqual(result, str(response.content).strip())