From a65c58a342af260e360463bd4061ef2125d079f0 Mon Sep 17 00:00:00 2001 From: Han Mai Date: Thu, 23 May 2024 21:46:15 +0800 Subject: [PATCH 1/2] Add docker compose, support docker image local build, lock version of mysql and bcrypt --- docker/Dockerfile.local | 61 +++++++++++++++++++++++++++++++++++++++ docker/README.md | 38 ++++++++++++++++++++++++ docker/buildLocalImage.sh | 18 ++++++++++++ docker/compose.yml | 50 ++++++++++++++++++++++++++++++++ docker/requirements.txt | 2 +- 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile.local create mode 100644 docker/README.md create mode 100644 docker/buildLocalImage.sh create mode 100644 docker/compose.yml diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local new file mode 100644 index 00000000..e97757e1 --- /dev/null +++ b/docker/Dockerfile.local @@ -0,0 +1,61 @@ +FROM ubuntu:latest +LABEL "org.opencontainers.image.authors"="Shamal Faily " +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y build-essential \ + python3-dev \ + mysql-client \ + graphviz \ + python3-pip \ + python3-numpy \ + python3-mysqldb \ + git \ + default-libmysqlclient-dev \ + python3-apt \ + libxml2-dev \ + libxslt1-dev \ + libssl-dev \ + apache2 \ + apache2-dev \ + poppler-utils \ + python3-setuptools \ + apt-transport-https \ + ca-certificates + +RUN pip3 install wheel --break-system-packages +#Installing Python modules +COPY requirements.txt / +RUN pip3 install -r requirements.txt --break-system-packages +COPY wsgi_requirements.txt / +RUN pip3 install -r wsgi_requirements.txt --break-system-packages + +#Environment Variable starts from here +ENV CAIRIS_SRC=/cairis/cairis +ENV CAIRIS_CFG_DIR=/cairis/docker +ENV CAIRIS_CFG=/cairis.cnf +ENV PYTHONPATH=/cairis + +RUN mkdir /tmpDocker && \ + mkdir /images + +# Copy from local repo +COPY cairis /cairis/cairis + +# Grant permission to executable files +RUN chmod +x /cairis/cairis/bin/*.* && \ + chmod +x /cairis/cairis/config/*.sh + +COPY cairis.cnf / +COPY setupDb.sh / +COPY createdb.sql / +COPY addAccount.sh / +COPY register_user.html /cairis/cairis/daemon/templates/security + +RUN /cairis/cairis/bin/installUI.sh + +EXPOSE 8000 + +RUN apt-get remove --purge -y git && \ + apt-get autoremove -y + +CMD ["./setupDb.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..4486f8a9 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,38 @@ +# INSTRUCTION TO BUILD AND BRING UP SETUP WITH DOCKER + +## 1. Build docker image + +By default, the cairis backend source code is fetched from github repo. The below script will help to build docker image locally. +You can specify the image version. By default, it is "latest". + +```sh +./buildLocalImage.sh +``` + +## 2. Startup servers + +Foreground mode (to exit, press CTRL + C): + +```sh +docker compose up +``` + +Background mode: + +```sh +docker compose up -d +``` + +To stop the servers: + +```sh +docker compose down +``` + +## 3. Create a user + +"cairis-CAIRIS-1" is the name of docker container of the cairis server. + +```sh +docker exec -t cairis-CAIRIS-1 ./addAccount.sh test@test.com test TestUser +``` diff --git a/docker/buildLocalImage.sh b/docker/buildLocalImage.sh new file mode 100644 index 00000000..94bfa661 --- /dev/null +++ b/docker/buildLocalImage.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Get version from parameter +VERSION=$1 + +if [[ "$VERSION" == "" ]]; then + VERSION=latest +fi + +# Copy from local repo +cp -R ../cairis ./ + +# Build +docker build -t shamalfaily/cairis:$VERSION -f Dockerfile.local . + +# Clean up +rm -rf cairis diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 00000000..241cb6bd --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,50 @@ +name: cairis +services: + cairis-mysql: + image: mysql:5.7 + environment: + - MYSQL_ROOT_PASSWORD=my-secret-pw + command: --thread_stack=256K --max_sp_recursion_depth=255 --log_bin_trust_function_creators=1 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - cairisMysqlData:/var/lib/mysql + networks: + - cairisnet + healthcheck: + test: ["CMD-SHELL", "mysql -h localhost -P 3306 -uroot -pmy-secret-pw -e \"SELECT 1\" && exit 0 || exit 1"] + interval: 1m + timeout: 10s + retries: 3 + start_period: 45s + start_interval: 15s + + cairis-docs: + image: shamalfaily/cairis-docs + depends_on: + - cairis-mysql + volumes: + - cairisDocumentation:/tmpDocker + - cairisImage:/images + networks: + - cairisnet + + CAIRIS: + image: shamalfaily/cairis:latest + depends_on: + cairis-mysql: + condition: service_healthy + restart: true + ports: + - 80:8000 + volumes: + - cairisDocumentation:/tmpDocker + - cairisImage:/images + networks: + - cairisnet + +volumes: + cairisMysqlData: + cairisDocumentation: + cairisImage: + +networks: + cairisnet: diff --git a/docker/requirements.txt b/docker/requirements.txt index 1129cb3a..bdd1cadd 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -24,7 +24,7 @@ mako>=1.0.4 lxml>=3.6.4 openpyxl>=2.4.0 SQLAlchemy>=2.0.0 -bcrypt>=3.1.6 +bcrypt==3.1.6 python-magic>=0.4.15 email_validator>=1.0.5 xlsxwriter>=1.2.7 From e08e71a6420c84d6fca5a8cb356a492048c8c4f3 Mon Sep 17 00:00:00 2001 From: Han Mai Date: Sat, 25 May 2024 15:11:01 +0800 Subject: [PATCH 2/2] Fix docker container keep flushing data on startup --- docker/Dockerfile.local | 55 ++++++++++++++++++---------------------- docker/README.md | 30 +++++++++++++++++----- docker/addAccount.sh | 2 +- docker/compose.yml | 10 +++++--- docker/init-test-data.sh | 13 ++++++++++ 5 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 docker/init-test-data.sh diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index e97757e1..fbbad52c 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -1,33 +1,29 @@ -FROM ubuntu:latest +# syntax=docker/dockerfile:1 +# Build UI +FROM node:16 as build_ui + +ADD --keep-git-dir=false https://github.com/cairis-platform/cairis-ui.git#master /cairis-ui +RUN cd /cairis-ui && \ + yarn install --ignore-engines && \ + yarn run build + +# Build CAIRIS backend +FROM python:3.12 LABEL "org.opencontainers.image.authors"="Shamal Faily " ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y build-essential \ - python3-dev \ - mysql-client \ - graphviz \ - python3-pip \ - python3-numpy \ - python3-mysqldb \ - git \ - default-libmysqlclient-dev \ - python3-apt \ - libxml2-dev \ - libxslt1-dev \ - libssl-dev \ +RUN apt-get update && apt-get install -y \ apache2 \ apache2-dev \ + mariadb-client \ poppler-utils \ - python3-setuptools \ - apt-transport-https \ - ca-certificates + netpbm \ + graphviz -RUN pip3 install wheel --break-system-packages #Installing Python modules COPY requirements.txt / -RUN pip3 install -r requirements.txt --break-system-packages COPY wsgi_requirements.txt / -RUN pip3 install -r wsgi_requirements.txt --break-system-packages +RUN pip install wheel --break-system-packages && \ + pip install -r requirements.txt -r wsgi_requirements.txt --break-system-packages #Environment Variable starts from here ENV CAIRIS_SRC=/cairis/cairis @@ -35,9 +31,6 @@ ENV CAIRIS_CFG_DIR=/cairis/docker ENV CAIRIS_CFG=/cairis.cnf ENV PYTHONPATH=/cairis -RUN mkdir /tmpDocker && \ - mkdir /images - # Copy from local repo COPY cairis /cairis/cairis @@ -46,16 +39,18 @@ RUN chmod +x /cairis/cairis/bin/*.* && \ chmod +x /cairis/cairis/config/*.sh COPY cairis.cnf / -COPY setupDb.sh / -COPY createdb.sql / COPY addAccount.sh / COPY register_user.html /cairis/cairis/daemon/templates/security -RUN /cairis/cairis/bin/installUI.sh +# Copy UI build +COPY --from=build_ui /cairis-ui/dist /cairis/cairis/dist EXPOSE 8000 -RUN apt-get remove --purge -y git && \ - apt-get autoremove -y +RUN mkdir /tmpDocker && \ + mkdir /images && \ + chown www-data /images && \ + chgrp www-data /images && \ + chmod 1777 /tmpDocker -CMD ["./setupDb.sh"] +CMD ["mod_wsgi-express", "start-server", "/cairis/cairis/bin/cairis.wsgi", "--user", "www-data", "--group", "www-data"] diff --git a/docker/README.md b/docker/README.md index 4486f8a9..de317809 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,26 +9,44 @@ You can specify the image version. By default, it is "latest". ./buildLocalImage.sh ``` -## 2. Startup servers +## 2. Server startup and shutdown -Foreground mode (to exit, press CTRL + C): +To create and start containers in backgroud mode: ```sh -docker compose up +docker compose up -d ``` -Background mode: +To view container logs: ```sh -docker compose up -d +docker compose logs -f +``` + +To stop: + +```sh +docker compose stop +``` + +To start: + +```sh +docker compose start ``` -To stop the servers: +To stop and remove the containers: ```sh docker compose down ``` +Your data is persistent even you remove the containers. To completely delete all data: + +```sh +$ docker volume rm cairis_cairisDocumentation cairis_cairisImage cairis_cairisMysqlData +``` + ## 3. Create a user "cairis-CAIRIS-1" is the name of docker container of the cairis server. diff --git a/docker/addAccount.sh b/docker/addAccount.sh index 44aebecf..e8bc0c56 100755 --- a/docker/addAccount.sh +++ b/docker/addAccount.sh @@ -8,4 +8,4 @@ USERNAME=$1 PASSWD=$2 FULLNAME=$3 -$CAIRIS_SRC/bin/add_cairis_user.py $USERNAME $PASSWD $FULLNAME +python3 $CAIRIS_SRC/bin/add_cairis_user.py $USERNAME $PASSWD $FULLNAME diff --git a/docker/compose.yml b/docker/compose.yml index 241cb6bd..98606ff6 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -7,15 +7,19 @@ services: command: --thread_stack=256K --max_sp_recursion_depth=255 --log_bin_trust_function_creators=1 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci volumes: - cairisMysqlData:/var/lib/mysql + - ./createdb.sql:/docker-entrypoint-initdb.d/00-createdb.sql:ro + - ./init-test-data.sh:/docker-entrypoint-initdb.d/01-init-test-data.sh:ro + - ../cairis/sql/init.sql:/sql/init.sql + - ../cairis/sql/procs.sql:/sql/procs.sql networks: - cairisnet healthcheck: - test: ["CMD-SHELL", "mysql -h localhost -P 3306 -uroot -pmy-secret-pw -e \"SELECT 1\" && exit 0 || exit 1"] + test: ["CMD-SHELL", "[ -f /var/lib/mysql/db_init_done ] && mysql -h localhost -P 3306 -uroot -pmy-secret-pw -e \"SELECT 1\" && exit 0 || exit 1"] interval: 1m timeout: 10s retries: 3 - start_period: 45s - start_interval: 15s + start_period: 2m30s + start_interval: 10s cairis-docs: image: shamalfaily/cairis-docs diff --git a/docker/init-test-data.sh b/docker/init-test-data.sh new file mode 100644 index 00000000..7adabbf2 --- /dev/null +++ b/docker/init-test-data.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -x + +mysql --user=cairis_test --password=cairis_test --database=cairis_test_default < /sql/init.sql +mysql --user=cairis_test --password=cairis_test --database=cairis_test_default < /sql/procs.sql +mysql --user=root --password=my-secret-pw <