From 95d5434618aa959af673639283b4c58fbd974f4a Mon Sep 17 00:00:00 2001 From: fidoriel <49869342+fidoriel@users.noreply.github.com> Date: Fri, 7 Jun 2024 23:10:56 +0200 Subject: [PATCH] new ci and dockerfiles --- .github/workflows/main.yml | 91 ++++++++++++++++++++++++++ Makefile | 47 ------------- README.md | 2 + backends/Dockerfile | 23 +++++++ backends/docker/Dockerfile.development | 18 ----- backends/docker/Dockerfile.production | 22 ------- backends/docker/startup.sh | 2 - docker-compose.dev.yml | 53 +++++++++++++++ docker-compose.yml | 60 ++++++++--------- frontend/Dockerfile | 22 +++++++ frontend/docker/Dockerfile.development | 21 ------ frontend/docker/Dockerfile.production | 31 --------- frontend/docker/httpd.conf | 21 ------ frontend/docker/startup.sh | 32 --------- frontend/ore/settings.py | 5 ++ frontend/ore/templates/index.html | 4 +- frontend/ore/templatetags/settings.py | 5 +- 17 files changed, 230 insertions(+), 229 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 Makefile create mode 100644 backends/Dockerfile delete mode 100644 backends/docker/Dockerfile.development delete mode 100644 backends/docker/Dockerfile.production delete mode 100644 backends/docker/startup.sh create mode 100644 docker-compose.dev.yml create mode 100644 frontend/Dockerfile delete mode 100644 frontend/docker/Dockerfile.development delete mode 100644 frontend/docker/Dockerfile.production delete mode 100644 frontend/docker/httpd.conf delete mode 100644 frontend/docker/startup.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..5599b6f9 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,91 @@ +name: Build + +on: + pull_request: + push: + branches: [master] + tags: + - "*" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build_docker: + name: build docker + runs-on: ubuntu-latest + + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to ghcr + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Docker fe-meta + id: fe-meta + uses: docker/metadata-action@v4 + env: + DOCKER_METADATA_PR_HEAD_SHA: true + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + tags: | + type=raw,value=${{ steps.extract_branch.outputs.branch }},enable=${{ github.ref != 'refs/heads/master' && steps.extract_branch.outputs.branch != '' }},prefix=fe- + type=semver,pattern={{raw}},prefix=fe- + type=raw,value=fe-latest,enable=${{ github.ref == 'refs/heads/master' }} + + - name: Build and push fe + uses: docker/build-push-action@v4 + with: + context: . + file: frontend/Dockerfile + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.fe-meta.outputs.tags }} + platforms: linux/amd64 + + - name: Docker be-meta + id: be-meta + uses: docker/metadata-action@v4 + env: + DOCKER_METADATA_PR_HEAD_SHA: true + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + tags: | + type=raw,value=${{ steps.extract_branch.outputs.branch }},enable=${{ github.ref != 'refs/heads/master' && steps.extract_branch.outputs.branch != '' }},prefix=be- + type=semver,pattern={{raw}},prefix=be- + type=raw,value=be-latest,enable=${{ github.ref == 'refs/heads/master' }} + + - name: Build and push be + uses: docker/build-push-action@v4 + with: + context: . + file: backends/Dockerfile + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.be-meta.outputs.tags }} + platforms: linux/amd64 \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 29db7ea5..00000000 --- a/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -dev-build: - docker-compose build - docker-compose up -d - docker exec -w /ore-front ore-front-dev scons - docker exec -w /ore-back ore-back-dev cmake . - docker exec -w /ore-back ore-back-dev make - docker-compose down - -dev-up: - docker-compose up - -dev-down: - docker-compose down - -dev-clean: - docker exec ore-front-dev /usr/bin/scons -C /ore-front -c - docker exec -w /ore-back ore-back-dev make clean - -dev-back-log: - docker logs ore-back-dev -f - -dev-front-log: - docker logs ore-front-dev -f - -dev-front-shell: - docker exec -it ore-front-dev bash - -dev-back-shell: - docker exec -it ore-back-dev bash - -prod-build: dev-build - # Use dev containers to build neccessary files - docker-compose up -d - docker exec -w /ore-front ore-front-dev ./manage.py collectstatic --noinput --configuration=Dev - docker-compose down - # Build production containers - docker build -t troeger/ore-front:0.8.4 -f frontend/docker/Dockerfile.production . - docker build -t troeger/ore-back:0.8.4 -f backends/docker/Dockerfile.production . - -prod-push: - docker login - docker push troeger/ore-front:0.8.4 - docker push troeger/ore-back:0.8.4 - -# Update version numbers, commit and tag -bumpversion: - bumpversion --allow-dirty patch diff --git a/README.md b/README.md index 299f3ee1..c30ca2cc 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ # ORE - The Open Reliability Editor (former FuzzEd) +> This is a legacy application without maintenance. This is not meant to run anywhere exposed to the internet. Please make sure to run a reverse proxy with basicauth in front. All versions are pinned so that the `Dockerfile` can build. For a working example please see `docker-compose.yml`. Make sure to remember your username. Because there is no import, you can back up you db with all graphs `docker compose exec ore-db pg_dump -U ore -d ore > backup.sql` and load with `docker-compose exec -T ore-db psql -U ore -d ore < backup.sql`. If the database is already used, you can drop the old one with `docker-compose exec ore-db dropdb -U ore ore` and create an empty new one with `docker-compose exec ore-db createdb -U ore ore`. Make sure to keep all data safe. + Note: FuzzEd becomes ORE. We are in the middle of that process, so don't get confused while both names are still in use. ORE is an browser-based editor for drawing and analyzing dependability models. The currently supported types are: diff --git a/backends/Dockerfile b/backends/Dockerfile new file mode 100644 index 00000000..1ea0fb0d --- /dev/null +++ b/backends/Dockerfile @@ -0,0 +1,23 @@ +# Dockerfile for ORE backend + +FROM ubuntu:xenial + +RUN apt-get update \ + && apt-get install -y texlive \ + && rm -rf /var/lib/apt/lists/* + + RUN apt-get update \ + && apt-get install -y libpq-dev python python-pip libboost-dev libboost-graph-dev libboost-date-time-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev cmake gcc libxerces-c-dev xsdcxx \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install requests==2.23.0 certifi==2020.6.20 + +WORKDIR /ore-back/ +COPY backends/ /ore-back/ +COPY common/ /ore-common/ + +RUN cmake . +RUN make -j + +EXPOSE 8000 +CMD ["python", "daemon.py"] diff --git a/backends/docker/Dockerfile.development b/backends/docker/Dockerfile.development deleted file mode 100644 index fcfa6f26..00000000 --- a/backends/docker/Dockerfile.development +++ /dev/null @@ -1,18 +0,0 @@ -# Dockerfile for ORE backend - -FROM ubuntu:xenial - -RUN apt-get update \ - && apt-get install -y texlive \ - && rm -rf /var/lib/apt/lists/* - -RUN apt-get update \ - && apt-get install -y python python-pip libboost-dev libboost-graph-dev libboost-date-time-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev cmake gcc libxerces-c-dev xsdcxx \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install requests - -COPY backends/docker/startup.sh /startup.sh - -EXPOSE 8000 -CMD ["bash", "/startup.sh"] diff --git a/backends/docker/Dockerfile.production b/backends/docker/Dockerfile.production deleted file mode 100644 index 5edaa60b..00000000 --- a/backends/docker/Dockerfile.production +++ /dev/null @@ -1,22 +0,0 @@ -# Dockerfile for ORE backend - -FROM ubuntu:xenial - -RUN apt-get update \ - && apt-get install -y texlive \ - && rm -rf /var/lib/apt/lists/* - -RUN apt-get update \ - && apt-get install -y python python-pip libboost-dev libboost-graph-dev libboost-date-time-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev libxerces-c-dev xsdcxx \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install requests - -COPY backends/docker/startup.sh /ore-back/startup.sh -COPY backends/daemon.py /ore-back/daemon.py -COPY backends/daemon.ini /ore-back/daemon.ini -COPY backends/lib /ore-back/lib -COPY backends/rendering /ore-back/rendering - -EXPOSE 8000 -CMD ["bash", "/ore-back/startup.sh"] diff --git a/backends/docker/startup.sh b/backends/docker/startup.sh deleted file mode 100644 index 215e2c55..00000000 --- a/backends/docker/startup.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd /ore-back -python daemon.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..b697408a --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,53 @@ +version: '3' + +services: + ore-db-dev: + image: postgres:9 + container_name: ore-db-dev + environment: + POSTGRES_PASSWORD: ore + POSTGRES_USER: ore + POSTGRES_DB: ore + volumes: + - postgres-volume-dev:/var/lib/postgresql/data + + ore-front-dev: + image: ghcr.io/osmhpi/fuzzed:fe-latest + container_name: ore-front-dev + command: sh -c '/ore-front/manage.py migrate && ./manage.py runserver 0.0.0.0:8000' + build: + context: . + dockerfile: ./frontend/Dockerfile + environment: + ORE_BACKEND_DAEMON: http://ore-back-dev:8000 + ORE_SERVER_URL: http://ore-front-dev:8000 + ORE_SERVER: ore-front-dev + ORE_DB_NAME: ore + ORE_DB_USER: ore + ORE_DB_PASSWORD: ore + ORE_DB_HOST: ore-db-dev + DJANGO_CONFIGURATION: Dev + ORE_SECRET_KEY: 1234 + ORE_DEV_LOGIN: 1 + ports: + - 8000:8000 + depends_on: + - ore-db-dev + - ore-back-dev + volumes: + - ./frontend:/ore-front + - ./common:/ore-common + + ore-back-dev: # to build docker compose exec ore-back-dev cmake . && make + image: ghcr.io/osmhpi/fuzzed:be-latest + container_name: ore-back-dev + build: + context: . + dockerfile: ./backends/Dockerfile + volumes: + - ./backends:/ore-back + - ./common:/ore-common + - /ore-back/lib + +volumes: + postgres-volume-dev: diff --git a/docker-compose.yml b/docker-compose.yml index dd9986c8..7719738b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,49 +1,47 @@ version: '3' services: - ore-db-dev: + ore-db: image: postgres:9 - container_name: ore-db-dev + container_name: ore-db environment: POSTGRES_PASSWORD: ore POSTGRES_USER: ore POSTGRES_DB: ore + restart: always volumes: - - postgres-volume:/var/lib/postgresql/data - ore-front-dev: - image: troeger/ore-front:0.8.4 - container_name: ore-front-dev - build: + - postgres-volume:/var/lib/postgresql/data + + ore-front: + image: ghcr.io/osmhpi/fuzzed:fe-latest + container_name: ore-front + build: context: . - dockerfile: ./frontend/docker/Dockerfile.development + dockerfile: ./frontend/Dockerfile + restart: always environment: - ORE_BACKEND_DAEMON: http://ore-back-dev:8000 - ORE_SERVER_URL: http://localhost - ORE_SERVER: localhost # this adds an entry to allowed hosts - ORE_DB_NAME: ore - ORE_DB_USER: ore - ORE_DB_PASSWORD: ore - ORE_DB_HOST: ore-db-dev - ORE_MODE: development - tty: true + ORE_BACKEND_DAEMON: http://ore-back:8000 + ORE_SERVER_URL: http://ore-front:8000 + ORE_SERVER: ore-front # this adds a host to allowed hosts + ORE_DB_NAME: ore + ORE_DB_USER: ore + ORE_DB_PASSWORD: ore + ORE_DB_HOST: ore-db + ORE_SECRET_KEY: change-me-to-something-random-and-very-secret + ORE_DEV_LOGIN: 1 ports: - 8000:8000 depends_on: - - ore-db-dev - - ore-back-dev - volumes: - - ./frontend:/ore-front - - ./common:/ore-common - ore-back-dev: - image: troeger/ore-back:0.8.4 - container_name: ore-back-dev - build: + - ore-db + - ore-back + + ore-back: + image: ghcr.io/osmhpi/fuzzed:be-latest + container_name: ore-back + restart: always + build: context: . - dockerfile: ./backends/docker/Dockerfile.development - tty: true - volumes: - - ./backends:/ore-back - - ./common:/ore-common + dockerfile: ./backends/Dockerfile volumes: postgres-volume: diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..1559d94c --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,22 @@ +# Dockerfile for ORE web application + +FROM ubuntu:bionic + +ENV DJANGO_CONFIGURATION=Production +ENV DEBIANFRONTEND=nointeractive + +# Prepare Ansible environment +RUN apt-get update \ + && apt-get install -y python python-pip git nodejs \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install PyXB==1.2.6 defusedxml==0.7.1 psycopg2-binary==2.8.6 django==1.8.18 python-social-auth==0.2.21 python-openid==2.2.5 python-oauth2==1.1.1 django-require==1.0.11 django-robots==3.1.0 django-configurations==2.0 requests_oauthlib==1.1.0 uwsgi==2.0.18 + +RUN pip install git+https://github.com/django-tastypie/django-tastypie.git@256ebe1de9a78dfb5d4d6e938b813cf4c5c4ac1b + +WORKDIR /ore-front +COPY frontend/ /ore-front + +ENV PYTHONUNBUFFERED 1 +EXPOSE 8000 +CMD sh -c '/ore-front/manage.py collectstatic -v3 --noinput && /ore-front/manage.py migrate && uwsgi --http 0.0.0.0:8000 --wsgi-file ore/wsgi.py --static-map /static/=/ore-front/ore/static-release/' diff --git a/frontend/docker/Dockerfile.development b/frontend/docker/Dockerfile.development deleted file mode 100644 index 3831fa57..00000000 --- a/frontend/docker/Dockerfile.development +++ /dev/null @@ -1,21 +0,0 @@ -# Dockerfile for ORE web application - -FROM ubuntu:bionic - -ENV ORE_MODE development - -RUN apt-get update \ - && apt-get install -y python-dev python-pip scons npm git netcat \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install PyXB defusedxml psycopg2-binary django==1.8.18 python-social-auth==0.2.21 python-openid python-oauth2 django-require django-robots django-configurations==2.0 requests_oauthlib - -RUN pip install git+https://github.com/django-tastypie/django-tastypie.git@256ebe1de9a78dfb5d4d6e938b813cf4c5c4ac1b - -RUN npm install -g less mocha-phantomjs - -COPY frontend/docker/startup.sh /startup.sh - -ENV PYTHONUNBUFFERED 1 -EXPOSE 8000 -CMD ["bash", "/startup.sh"] diff --git a/frontend/docker/Dockerfile.production b/frontend/docker/Dockerfile.production deleted file mode 100644 index d283d2da..00000000 --- a/frontend/docker/Dockerfile.production +++ /dev/null @@ -1,31 +0,0 @@ -# Dockerfile for ORE web application - -FROM ubuntu:bionic - -ENV ORE_MODE production - -# Prepare Ansible environment -RUN apt-get update \ - && apt-get install -y python python-pip apache2 libapache2-mod-wsgi git netcat \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install PyXB defusedxml psycopg2-binary django==1.8.18 python-social-auth==0.2.21 python-openid python-oauth2 django-require django-robots django-configurations==2.0 requests_oauthlib - -RUN pip install git+https://github.com/django-tastypie/django-tastypie.git@256ebe1de9a78dfb5d4d6e938b813cf4c5c4ac1b - -RUN rm /etc/apache2/sites-enabled/000-default.conf -COPY frontend/docker/httpd.conf /etc/apache2/sites-enabled/ore.conf - -RUN a2enmod wsgi - -RUN apache2ctl configtest - -COPY frontend/docker/startup.sh /startup.sh -COPY frontend/manage.py /var/www/ -COPY frontend/ore /var/www/ore -RUN rm -r /var/www/ore/fixtures /var/www/ore/static /var/www/ore/tests - -ENV PYTHONUNBUFFERED 1 -EXPOSE 8000 -CMD ["bash", "/startup.sh"] - diff --git a/frontend/docker/httpd.conf b/frontend/docker/httpd.conf deleted file mode 100644 index 7dc08a0d..00000000 --- a/frontend/docker/httpd.conf +++ /dev/null @@ -1,21 +0,0 @@ - - ServerName front - - Alias /static /var/www/ore/static-release - - Options FollowSymLinks - AllowOverride None - Order deny,allow - Allow from all - - - WSGIDaemonProcess ore processes=5 threads=1 maximum-requests=1000 display-name=%{GROUP} python-path=/var/www - WSGIProcessGroup ore - WSGIScriptAlias / /var/www/ore/wsgi.py - WSGIPassAuthorization On - - ErrorLog ${APACHE_LOG_DIR}/ore.error.log - LogLevel info - CustomLog ${APACHE_LOG_DIR}/ore.access.log combined - - diff --git a/frontend/docker/startup.sh b/frontend/docker/startup.sh deleted file mode 100644 index 35529146..00000000 --- a/frontend/docker/startup.sh +++ /dev/null @@ -1,32 +0,0 @@ - -# Wait for postgres to come up -echo "Waiting for database at '$ORE_DB_HOST' ..." -while ! nc -z $ORE_DB_HOST 5432 2>/dev/null -do - let elapsed=elapsed+1 - if [ "$elapsed" -gt 10 ] - then - echo "Could not connect to database container." - exit 1 - fi - sleep 1; -done -echo "Database is up." - -if [ "$ORE_MODE" = "development" ] -then - # Assumes "frontend" directory to be mounted at /ore-front - cd /ore-front - export DJANGO_CONFIGURATION=Dev - ./manage.py migrate - ./manage.py runserver 0.0.0.0:8000 -fi - -if [ "$ORE_MODE" = "production" ] -then - cd /var/www - export DJANGO_CONFIGURATION=Production - ./manage.py migrate - rm -f /var/run/apache2/apache2.pid - apache2ctl -D FOREGROUND -fi diff --git a/frontend/ore/settings.py b/frontend/ore/settings.py index 5004eb13..aab64beb 100644 --- a/frontend/ore/settings.py +++ b/frontend/ore/settings.py @@ -217,6 +217,7 @@ class Dev(Common): 'oredeveloper@example.com', environ_prefix='ORE', environ_name='ADMIN_EMAIL'))),) +DEV_LOGIN = bool(os.environ.get('ORE_DEV_LOGIN', False)) class Production(Common): DEBUG = False TEMPLATE_DEBUG = False @@ -234,3 +235,7 @@ class Production(Common): LOGGING['loggers']['django.request']['handlers'] = ['mail_admins'] LOGGING['loggers']['ore']['handlers'] = ['console'] FOOTER = values.Value('ORE Development Team', environ_prefix='ORE') + + SOCIAL_AUTH_USERNAME_FORM_URL = '/' + SOCIAL_AUTH_USERNAME_FORM_HTML = 'dev_login.html' if DEV_LOGIN is True else None + AUTHENTICATION_BACKENDS = Common.AUTHENTICATION_BACKENDS + ('social.backends.username.UsernameAuth',) if DEV_LOGIN is True else Common.AUTHENTICATION_BACKENDS diff --git a/frontend/ore/templates/index.html b/frontend/ore/templates/index.html index 930be100..8802e9f0 100644 --- a/frontend/ore/templates/index.html +++ b/frontend/ore/templates/index.html @@ -86,8 +86,8 @@
Please use one of these authentication methods:
- {% get_debug_status as debug %} - {% if debug %} + {% get_dev_login as dev_login %} + {% if dev_login %}
Developer Login
diff --git a/frontend/ore/templatetags/settings.py b/frontend/ore/templatetags/settings.py index c6bbafed..cd38a298 100644 --- a/frontend/ore/templatetags/settings.py +++ b/frontend/ore/templatetags/settings.py @@ -1,6 +1,7 @@ from django import template from django.conf import settings import ore +import os register = template.Library() @@ -18,5 +19,5 @@ def setting(name): @register.assignment_tag -def get_debug_status(): - return settings.DEBUG is True +def get_dev_login(): + return bool(os.environ.get('ORE_DEV_LOGIN', False)) is True or settings.DEBUG is True