From 1aaed7aa1c35987564af13e49030c8f00fa5cbb7 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 12:47:01 +0100 Subject: [PATCH 01/51] #227 add script for building backend deb package --- scripts/backend_deb.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 scripts/backend_deb.sh diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh new file mode 100755 index 000000000..8f40e1f01 --- /dev/null +++ b/scripts/backend_deb.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# deb constants +version=1.0 +revision=1 +name=eak-backend +architecture=amd64 +maintainer="The Ehrenamtskarte Team " +description="Backend server for the Ehrenamtskarte app" + +fullname=${name}_${version}-${revision}_${architecture} +debworkdir=build/$fullname +debfile=build/${fullname}.deb +ctrlfile=${debworkdir}/DEBIAN/control + +# build backend +./gradlew build + +# init deb workdir +mkdir -p $debworkdir +rm -rf ${debworkdir}/* +mkdir ${debworkdir}/DEBIAN +touch $ctrlfile +echo "Package: $name" >> $ctrlfile +echo "Version: $version" >> $ctrlfile +echo "Architecture: $architecture" >> $ctrlfile +echo "Maintainer: $maintainer" >> $ctrlfile +echo "Description: $description" >> $ctrlfile + +# copy files to deb workdir +mkdir -p ${debworkdir}/var/ehrenamtskarte/backend +tar -xf build/distributions/backend.tar -C ${debworkdir}/var/ehrenamtskarte + +# build the deb +dpkg-deb --build --root-owner-group $debworkdir $debfile From a17bd833d10c98fe0e90cf834a1181376b0d4cff Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 13:16:55 +0100 Subject: [PATCH 02/51] #227 add config for circle ci to build backend --- .circleci/config.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..1202674b1 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2.1 +orbs: + gradle: circleci/gradle@2.2.0 +workflows: + backend: + jobs: + - build + +jobs: + build: + environment: + # Configure the JVM and Gradle to avoid OOM errors + _JAVA_OPTIONS: "-Xmx3g" + GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2" + docker: + - image: circleci/openjdk:11.0.3-jdk-stretch + working_directory: ~/root/backend + steps: + - checkout: + path: ~/root + - restore_cache: + key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + - restore_cache: + key: v1-gradle-cache-{{ checksum "build.gradle.kts" }} + - run: ./gradlew build + - save_cache: + paths: + - .gradle/wrapper + key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + - save_cache: + paths: + - .gradle/caches + key: v1-gradle-cache-{{ checksum "build.gradle.kts" }} + - store_artifacts: + path: build/libs From 72a660abfb5f94f30d3993c2b85e4f691b0882a6 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 16:20:46 +0100 Subject: [PATCH 03/51] #227 pack deb in pipeline --- .circleci/config.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1202674b1..e82a30f59 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,13 @@ workflows: backend: jobs: - build + - pack: + requires: + - build + - hold: + requires: + - pack + type: approval jobs: build: @@ -14,22 +21,36 @@ jobs: GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2" docker: - image: circleci/openjdk:11.0.3-jdk-stretch - working_directory: ~/root/backend + working_directory: ~/project/backend steps: - checkout: - path: ~/root + path: ~/project - restore_cache: - key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} - restore_cache: - key: v1-gradle-cache-{{ checksum "build.gradle.kts" }} + key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} - run: ./gradlew build - save_cache: paths: - .gradle/wrapper - key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} - save_cache: paths: - .gradle/caches - key: v1-gradle-cache-{{ checksum "build.gradle.kts" }} + key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} - store_artifacts: path: build/libs + - persist_to_workspace: + root: . # why is this not an optional parameter? :shaking_head: + paths: + - build/distributions/* + pack: + docker: + - image: cimg/base:2020.01 + working_directory: ~/project/backend + steps: + - checkout: + path: ~/project + - run: ../scripts/backend_deb.sh + - store_artifacts: + path: build/*.deb From 7426a89e8d1cc47fe15309e124e85ef814030f5d Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 16:38:14 +0100 Subject: [PATCH 04/51] #227 do not build again in deb script --- scripts/backend_deb.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh index 8f40e1f01..bb6f96c43 100755 --- a/scripts/backend_deb.sh +++ b/scripts/backend_deb.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -eo pipefail # deb constants version=1.0 @@ -13,9 +13,6 @@ debworkdir=build/$fullname debfile=build/${fullname}.deb ctrlfile=${debworkdir}/DEBIAN/control -# build backend -./gradlew build - # init deb workdir mkdir -p $debworkdir rm -rf ${debworkdir}/* From bed47c418fa0876ff4194cba8c7c4a04a5bc3f91 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 16:51:18 +0100 Subject: [PATCH 05/51] #227 improve circle ci config --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ scripts/backend_deb.sh | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e82a30f59..6292ddbfc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,15 @@ workflows: requires: - pack type: approval + filters: + branches: + only: main + - deploy: + requires: + - hold + filters: + branches: + only: main jobs: build: @@ -45,6 +54,23 @@ jobs: paths: - build/distributions/* pack: + docker: + - image: cimg/base:2020.01 + working_directory: ~/project/backend + steps: + - checkout: + path: ~/project + - attach_workspace: + at: /tmp/workspace + - run: cp -r /tmp/workspace/** . + - run: ../scripts/backend_deb.sh + - store_artifacts: + path: build/*.deb + - persist_to_workspace: + root: . + paths: + - build/*.deb + deploy: docker: - image: cimg/base:2020.01 working_directory: ~/project/backend diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh index bb6f96c43..2adb4038f 100755 --- a/scripts/backend_deb.sh +++ b/scripts/backend_deb.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eo pipefail +#!/bin/bash -e # deb constants version=1.0 From f1076df2edfddb9b4b4f87d83b03a24a12986be5 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 17:33:56 +0100 Subject: [PATCH 06/51] #227 make deb script configurable using params --- .circleci/config.yml | 13 ++++++------- scripts/backend_deb.sh | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6292ddbfc..848c5526d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,14 +62,13 @@ jobs: path: ~/project - attach_workspace: at: /tmp/workspace - - run: cp -r /tmp/workspace/** . - - run: ../scripts/backend_deb.sh + - run: ../scripts/backend_deb.sh -v 0.1 -t /tmp/workspace/build/distributions/*.tar - store_artifacts: - path: build/*.deb + path: ./*.deb - persist_to_workspace: root: . paths: - - build/*.deb + - ./*.deb deploy: docker: - image: cimg/base:2020.01 @@ -77,6 +76,6 @@ jobs: steps: - checkout: path: ~/project - - run: ../scripts/backend_deb.sh - - store_artifacts: - path: build/*.deb + - attach_workspace: + at: /tmp/workspace + - run: echo "TODO put to ftp folder" diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh index 2adb4038f..1caf06e2d 100755 --- a/scripts/backend_deb.sh +++ b/scripts/backend_deb.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/bin/bash +set -eo pipefail # deb constants version=1.0 @@ -8,15 +9,34 @@ architecture=amd64 maintainer="The Ehrenamtskarte Team " description="Backend server for the Ehrenamtskarte app" +# read input +while getopts v:r:a:n:t:h flag +do + case "${flag}" in + v) version=${OPTARG};; + r) revision=${OPTARG};; + a) architecture=${OPTARG};; + n) name=${OPTARG};; + t) tarfile=${OPTARG};; + h) + echo "$0 [-v version] [-r revision] [-a architecture] [-n name] -t backend_tar" + exit 0;; + esac +done + +if [[ -z "$tarfile" ]]; then + echo "Must provide input tar file using -t file" 1>&2 + exit 1 +fi + +debworkdir=$(mktemp -d) fullname=${name}_${version}-${revision}_${architecture} -debworkdir=build/$fullname -debfile=build/${fullname}.deb -ctrlfile=${debworkdir}/DEBIAN/control +debfile=${fullname}.deb # init deb workdir -mkdir -p $debworkdir -rm -rf ${debworkdir}/* mkdir ${debworkdir}/DEBIAN +ctrlfile=${debworkdir}/DEBIAN/control +echo "Creating control file in $ctrlfile …" touch $ctrlfile echo "Package: $name" >> $ctrlfile echo "Version: $version" >> $ctrlfile @@ -25,8 +45,13 @@ echo "Maintainer: $maintainer" >> $ctrlfile echo "Description: $description" >> $ctrlfile # copy files to deb workdir +echo "Copying $tarfile …" mkdir -p ${debworkdir}/var/ehrenamtskarte/backend -tar -xf build/distributions/backend.tar -C ${debworkdir}/var/ehrenamtskarte +tar -xf $tarfile -C ${debworkdir}/var/ehrenamtskarte # build the deb dpkg-deb --build --root-owner-group $debworkdir $debfile + +# clean up +echo "Cleaning up …" +rm -rf $debworkdir From d040302e766b0ba355fb6455e0fb7f8e14f6445f Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 17:45:17 +0100 Subject: [PATCH 07/51] #227 store deb as artifact for real --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 848c5526d..64f2d1272 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,8 +63,11 @@ jobs: - attach_workspace: at: /tmp/workspace - run: ../scripts/backend_deb.sh -v 0.1 -t /tmp/workspace/build/distributions/*.tar + - run: | + mkdir -p /tmp/artifacts + cp *.deb /tmp/artifacts - store_artifacts: - path: ./*.deb + path: /tmp/artifacts - persist_to_workspace: root: . paths: From fb6a28fa6961d9063ccbc5a948c5a17a517a29a7 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 18:09:54 +0100 Subject: [PATCH 08/51] #227 add systemd service file --- .circleci/config.yml | 2 +- scripts/backend_deb.sh | 24 ++++++++++++++---------- scripts/eak-backend.service | 13 +++++++++++++ 3 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 scripts/eak-backend.service diff --git a/.circleci/config.yml b/.circleci/config.yml index 64f2d1272..fc743c12b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,7 +62,7 @@ jobs: path: ~/project - attach_workspace: at: /tmp/workspace - - run: ../scripts/backend_deb.sh -v 0.1 -t /tmp/workspace/build/distributions/*.tar + - run: ../scripts/backend_deb.sh -v 0.1 -t /tmp/workspace/build/distributions/*.tar -s ../scripts/eak-backend.service - run: | mkdir -p /tmp/artifacts cp *.deb /tmp/artifacts diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh index 1caf06e2d..7e1eee4fc 100755 --- a/scripts/backend_deb.sh +++ b/scripts/backend_deb.sh @@ -10,7 +10,7 @@ maintainer="The Ehrenamtskarte Team " description="Backend server for the Ehrenamtskarte app" # read input -while getopts v:r:a:n:t:h flag +while getopts v:r:a:n:t:s:h flag do case "${flag}" in v) version=${OPTARG};; @@ -18,17 +18,13 @@ do a) architecture=${OPTARG};; n) name=${OPTARG};; t) tarfile=${OPTARG};; + s) servicefile=${OPTARG};; h) - echo "$0 [-v version] [-r revision] [-a architecture] [-n name] -t backend_tar" + echo "$0 [-v version] [-r revision] [-a architecture] [-n name] [-t backend_tar] [-s service_file]" exit 0;; esac done -if [[ -z "$tarfile" ]]; then - echo "Must provide input tar file using -t file" 1>&2 - exit 1 -fi - debworkdir=$(mktemp -d) fullname=${name}_${version}-${revision}_${architecture} debfile=${fullname}.deb @@ -45,9 +41,17 @@ echo "Maintainer: $maintainer" >> $ctrlfile echo "Description: $description" >> $ctrlfile # copy files to deb workdir -echo "Copying $tarfile …" -mkdir -p ${debworkdir}/var/ehrenamtskarte/backend -tar -xf $tarfile -C ${debworkdir}/var/ehrenamtskarte +if [[ -n "$tarfile" ]]; then + echo "Copying $tarfile …" + mkdir -p ${debworkdir}/opt/ehrenamtskarte/backend + tar -xf $tarfile -C ${debworkdir}/opt/ehrenamtskarte +fi +if [[ -n "$servicefile" ]]; then + echo "Copying $servicefile …" + mkdir -p ${debworkdir}/etc/systemd/system + cp $servicefile ${debworkdir}/etc/systemd/system/${name}.service +fi + # build the deb dpkg-deb --build --root-owner-group $debworkdir $debfile diff --git a/scripts/eak-backend.service b/scripts/eak-backend.service new file mode 100644 index 000000000..07cbbaf32 --- /dev/null +++ b/scripts/eak-backend.service @@ -0,0 +1,13 @@ +[Unit] +Description=Backend for the ehrenamtskarte app +After=postgresql.service network-online.target +Wants=network-online.target systemd-networkd-wait-online.service + +StartLimitIntervalSec=500 +StartLimitBurst=5 + +[Service] +Type=simple +ExecStart=/opt/ehrenamtskarte/backend/bin/backend +Restart=on-failure +RestartSec=5s From 975e62aca1ec79d8b41b6157bc51b8f73bfda73e Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 18:52:33 +0100 Subject: [PATCH 09/51] #227 add backend to multiuser target --- scripts/eak-backend.service | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/eak-backend.service b/scripts/eak-backend.service index 07cbbaf32..402fcae17 100644 --- a/scripts/eak-backend.service +++ b/scripts/eak-backend.service @@ -1,7 +1,6 @@ [Unit] Description=Backend for the ehrenamtskarte app After=postgresql.service network-online.target -Wants=network-online.target systemd-networkd-wait-online.service StartLimitIntervalSec=500 StartLimitBurst=5 @@ -11,3 +10,6 @@ Type=simple ExecStart=/opt/ehrenamtskarte/backend/bin/backend Restart=on-failure RestartSec=5s + +[Install] +multi-user.target From 84afdc3d52ef612187ac598e4bede6e9a0cbec97 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 19:18:07 +0100 Subject: [PATCH 10/51] #227 deploy .deb using scp --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fc743c12b..1371ed823 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,8 +77,6 @@ jobs: - image: cimg/base:2020.01 working_directory: ~/project/backend steps: - - checkout: - path: ~/project - attach_workspace: at: /tmp/workspace - - run: echo "TODO put to ftp folder" + - run: scp /tmp/workspace/*.deb circleci@apt.ehrenamtskarte.app:/srv/local-apt-repository/ From 46078bca8218a8cb15168f386a9465d8c1123b39 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 21:35:28 +0100 Subject: [PATCH 11/51] #227 adapt deployment to server config --- .circleci/config.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1371ed823..4dd59269a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,13 +14,17 @@ workflows: type: approval filters: branches: - only: main + only: + - main + - 227-deb-packages - deploy: requires: - hold filters: branches: - only: main + only: + - main + - 227-deb-packages jobs: build: @@ -79,4 +83,7 @@ jobs: steps: - attach_workspace: at: /tmp/workspace - - run: scp /tmp/workspace/*.deb circleci@apt.ehrenamtskarte.app:/srv/local-apt-repository/ + - add_ssh_keys: + fingerprints: + - "5c:8a:70:dc:61:9d:fa:c2:c2:6d:2d:fc:4a:ab:d2:e3" + - run: scp /tmp/workspace/*.deb ci@apt.ehrenamtskarte.app:/srv/local-apt-repository/ From a0a025ae651cf789c14d90b2b565df80d5e54323 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 21:56:03 +0100 Subject: [PATCH 12/51] #227 check host key before scp-ing --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4dd59269a..79a0ba8c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,4 +86,6 @@ jobs: - add_ssh_keys: fingerprints: - "5c:8a:70:dc:61:9d:fa:c2:c2:6d:2d:fc:4a:ab:d2:e3" - - run: scp /tmp/workspace/*.deb ci@apt.ehrenamtskarte.app:/srv/local-apt-repository/ + - run: | + echo $APT_HOST_FINGERPRINT > known_hosts + scp -o UserKnownHostsFile=known_hosts /tmp/workspace/*.deb ci@apt.ehrenamtskarte.app:/srv/local-apt-repository From 1bb0d6406934c6afcc290ad94f047833c9eba916 Mon Sep 17 00:00:00 2001 From: Elias Keis Date: Thu, 4 Feb 2021 22:37:05 +0100 Subject: [PATCH 13/51] #227 enable deploy for main only --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79a0ba8c5..0286c1d01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,6 @@ workflows: branches: only: - main - - 227-deb-packages - deploy: requires: - hold @@ -24,7 +23,6 @@ workflows: branches: only: - main - - 227-deb-packages jobs: build: From 706780bc980260dae429b7e3e77ee39bb296d3b3 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 14:34:54 +0100 Subject: [PATCH 14/51] Retry to deploy --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0286c1d01..79a0ba8c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,7 @@ workflows: branches: only: - main + - 227-deb-packages - deploy: requires: - hold @@ -23,6 +24,7 @@ workflows: branches: only: - main + - 227-deb-packages jobs: build: From 89e03c850c7d56c9fee78136f1f88151e28b5ddf Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 14:48:56 +0100 Subject: [PATCH 15/51] Switch to sftp --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79a0ba8c5..8db823751 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,4 +88,4 @@ jobs: - "5c:8a:70:dc:61:9d:fa:c2:c2:6d:2d:fc:4a:ab:d2:e3" - run: | echo $APT_HOST_FINGERPRINT > known_hosts - scp -o UserKnownHostsFile=known_hosts /tmp/workspace/*.deb ci@apt.ehrenamtskarte.app:/srv/local-apt-repository + sftp -b - -v -o UserKnownHostsFile=known_hosts ci@apt.ehrenamtskarte.app:/ \<<< "put -r /tmp/workspace/*.deb" From 4d0abec1896883e04121c39507ed7b60631aa113 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 15:04:48 +0100 Subject: [PATCH 16/51] Fix service file --- scripts/eak-backend.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eak-backend.service b/scripts/eak-backend.service index 402fcae17..dcd9fb865 100644 --- a/scripts/eak-backend.service +++ b/scripts/eak-backend.service @@ -12,4 +12,4 @@ Restart=on-failure RestartSec=5s [Install] -multi-user.target +WantedBy=multi-user.target From 04c6e6b2183f690c4cfab8053a2758fdf78d06c9 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 15:11:06 +0100 Subject: [PATCH 17/51] Fix path --- .circleci/config.yml | 2 +- scripts/backend_deb.sh | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8db823751..d265976d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,4 +88,4 @@ jobs: - "5c:8a:70:dc:61:9d:fa:c2:c2:6d:2d:fc:4a:ab:d2:e3" - run: | echo $APT_HOST_FINGERPRINT > known_hosts - sftp -b - -v -o UserKnownHostsFile=known_hosts ci@apt.ehrenamtskarte.app:/ \<<< "put -r /tmp/workspace/*.deb" + sftp -b - -v -o UserKnownHostsFile=known_hosts ci@apt.ehrenamtskarte.app:/local-apt-repository/ \<<< "put -r /tmp/workspace/*.deb" diff --git a/scripts/backend_deb.sh b/scripts/backend_deb.sh index 7e1eee4fc..1ce152af8 100755 --- a/scripts/backend_deb.sh +++ b/scripts/backend_deb.sh @@ -8,6 +8,7 @@ name=eak-backend architecture=amd64 maintainer="The Ehrenamtskarte Team " description="Backend server for the Ehrenamtskarte app" +dependencies="default-jre" # read input while getopts v:r:a:n:t:s:h flag @@ -22,6 +23,9 @@ do h) echo "$0 [-v version] [-r revision] [-a architecture] [-n name] [-t backend_tar] [-s service_file]" exit 0;; + *) + echo "Unknown flag" + exit 1;; esac done @@ -30,32 +34,33 @@ fullname=${name}_${version}-${revision}_${architecture} debfile=${fullname}.deb # init deb workdir -mkdir ${debworkdir}/DEBIAN +mkdir "${debworkdir}/DEBIAN" ctrlfile=${debworkdir}/DEBIAN/control echo "Creating control file in $ctrlfile …" -touch $ctrlfile -echo "Package: $name" >> $ctrlfile -echo "Version: $version" >> $ctrlfile -echo "Architecture: $architecture" >> $ctrlfile -echo "Maintainer: $maintainer" >> $ctrlfile -echo "Description: $description" >> $ctrlfile +touch "$ctrlfile" +echo "Package: $name" >> "$ctrlfile" +echo "Version: $version" >> "$ctrlfile" +echo "Architecture: $architecture" >> "$ctrlfile" +echo "Maintainer: $maintainer" >> "$ctrlfile" +echo "Description: $description" >> "$ctrlfile" +echo "Depends: $dependencies" >> "$ctrlfile" # copy files to deb workdir if [[ -n "$tarfile" ]]; then echo "Copying $tarfile …" - mkdir -p ${debworkdir}/opt/ehrenamtskarte/backend - tar -xf $tarfile -C ${debworkdir}/opt/ehrenamtskarte + mkdir -p "${debworkdir}/opt/ehrenamtskarte/backend" + tar -xf "$tarfile" -C "${debworkdir}/opt/ehrenamtskarte" fi if [[ -n "$servicefile" ]]; then echo "Copying $servicefile …" - mkdir -p ${debworkdir}/etc/systemd/system - cp $servicefile ${debworkdir}/etc/systemd/system/${name}.service + mkdir -p "${debworkdir}/etc/systemd/system" + cp "$servicefile" "${debworkdir}/etc/systemd/system/${name}.service" fi # build the deb -dpkg-deb --build --root-owner-group $debworkdir $debfile +dpkg-deb --build --root-owner-group "$debworkdir" "$debfile" # clean up echo "Cleaning up …" -rm -rf $debworkdir +rm -rf "$debworkdir" From 4a53654490ea6c278bb0ad54eed003196cebe99b Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 15:32:12 +0100 Subject: [PATCH 18/51] Change postgres config --- backend/gradle.properties | 4 +--- .../ehrenamtskarte/backend/common/database/Database.kt | 8 ++++---- docker-compose.staging.yml | 2 +- docker/reverse_proxy/nginx-staging.conf | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/backend/gradle.properties b/backend/gradle.properties index 9d94d08f9..dd8780fc9 100644 --- a/backend/gradle.properties +++ b/backend/gradle.properties @@ -2,11 +2,9 @@ exposed_version=0.28.1 -app.postgres.host=localhost -app.postgres.port=5432 +app.postgres.url=jdbc:postgresql://localhost:5432/ehrenamtskarte app.postgres.user=postgres app.postgres.password=postgres -app.postgres.database=ehrenamtskarte app.import.xml=https://www.lbe.bayern.de/engagement-anerkennen/ehrenamtskarte/akzeptanzstellen/app-daten.xml app.import.json_1=https://bayerische_eak: diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/Database.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/Database.kt index 08302f376..1c4c2eb24 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/Database.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/Database.kt @@ -40,9 +40,9 @@ class Database { } val db by lazy { - connect("jdbc:postgresql://" + - "${System.getProperty("app.postgres.host")}:${System.getProperty("app.postgres.port")}" + - "/${System.getProperty("app.postgres.database")}", driver = "org.postgresql.Driver", - user = System.getProperty("app.postgres.user"), password = System.getProperty("app.postgres.password")) + connect( + System.getProperty("app.postgres.url"), driver = "org.postgresql.Driver", + user = System.getProperty("app.postgres.user"), password = System.getProperty("app.postgres.password") + ) } } diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 695c1b5cb..39e3c6394 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -3,7 +3,7 @@ services: backend: build: ./backend/ environment: - - BACKEND_OPTS=-Dapp.postgres.host=db-postgis -Dapp.postgres.user=postgres -Dapp.postgres.port=5432 -Dapp.postgres.password=postgres -Dapp.postgres.database=ehrenamtskarte + - BACKEND_OPTS=-Dapp.postgres.url=jdbc:postgresql://db-postgis:5432/ehrenamtskarte -Dapp.postgres.user=postgres -Dapp.postgres.password=postgres depends_on: - db-postgis networks: diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 52b7f1142..099b0b674 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -8,7 +8,7 @@ upstream backend { server { listen 80; - server_name vector.ehrenamtskarte.app; + server_name tiles.staging.ehrenamtskarte.app; location ~ /tiles/(?.*) { proxy_set_header X-Rewrite-URL $request_uri; @@ -26,7 +26,7 @@ server { server { listen 80; - server_name api.ehrenamtskarte.app; + server_name api.staging.ehrenamtskarte.app; location / { proxy_set_header Host $http_host; From 060838437da131ffa4dac1294125c2dc16475b67 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 15:51:41 +0100 Subject: [PATCH 19/51] Add unix socket support --- backend/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/build.gradle.kts b/backend/build.gradle.kts index 9201e7f27..a483b22d9 100644 --- a/backend/build.gradle.kts +++ b/backend/build.gradle.kts @@ -49,6 +49,8 @@ dependencies { implementation("org.jetbrains.exposed", "exposed-dao", exposed_version) implementation("org.jetbrains.exposed", "exposed-jdbc", exposed_version) implementation("org.postgresql", "postgresql", "42.2.18") + implementation("com.kohlschutter.junixsocket", "junixsocket-core", "2.3.2") + implementation("com.kohlschutter.junixsocket", "junixsocket-common", "2.3.2") implementation("net.postgis", "postgis-jdbc", "2.5.0") From 543b97c173aa87e8a31e05773f12748810e0153e Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:18:04 +0100 Subject: [PATCH 20/51] Make host configurable --- .../backend/common/webservice/WebService.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt index 62090c0f8..f990ba7f4 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt @@ -3,18 +3,21 @@ package app.ehrenamtskarte.backend.common.webservice import io.javalin.Javalin import io.javalin.http.staticfiles.Location -const val PORT = 7000 +const val DEFAULT_PORT = "7000" class WebService { fun start() { + val host = System.getProperty("app.host", "0.0.0.0") + val port = Integer.parseInt(System.getProperty("app.port", DEFAULT_PORT)) val app = Javalin.create { cfg -> + cfg.enableDevLogging() cfg.enableCorsForAllOrigins() cfg.addStaticFiles("/graphiql", "/graphiql", Location.CLASSPATH) - }.start(PORT) + }.start(host, port) - println("Server is running at http://localhost:${PORT}") - println("Goto http://localhost:${PORT}/graphiql for a graphical editor") + println("Server is running at http://${host}:${port}") + println("Goto http://${host}:${port}/graphiql for a graphical editor") val graphQLHandler = GraphQLHandler() app.post("/") { ctx -> graphQLHandler.handle(ctx) } From 8c5c777066fa8c3e541c4186dbc4be985dc7df7b Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:21:56 +0100 Subject: [PATCH 21/51] Remove hold step for now --- .circleci/config.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d265976d7..0f6697940 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,18 +8,9 @@ workflows: - pack: requires: - build - - hold: - requires: - - pack - type: approval - filters: - branches: - only: - - main - - 227-deb-packages - deploy: requires: - - hold + - pack filters: branches: only: From abc4a3c92d6ce5c8299996395c8c721ea20c86cd Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:22:10 +0100 Subject: [PATCH 22/51] Add comment --- .../app/ehrenamtskarte/backend/common/webservice/WebService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt index f990ba7f4..91b65ac01 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt @@ -11,7 +11,7 @@ class WebService { val port = Integer.parseInt(System.getProperty("app.port", DEFAULT_PORT)) val app = Javalin.create { cfg -> - cfg.enableDevLogging() + cfg.enableDevLogging() // FIXME: Disable in dev cfg.enableCorsForAllOrigins() cfg.addStaticFiles("/graphiql", "/graphiql", Location.CLASSPATH) }.start(host, port) From eb1ffbddfc4f0a0feb5b208ddbfc18ebe8a39b2c Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:26:29 +0100 Subject: [PATCH 23/51] Fix urls --- .idea/misc.xml | 2 +- docker/reverse_proxy/nginx-development.conf | 2 +- docker/reverse_proxy/www/style.json | 8 ++++---- docs/staging-setup.md | 18 +++++++++--------- frontend/lib/main_prod.dart | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 2ec46fbf5..c396398e3 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -4,7 +4,7 @@ - + \ No newline at end of file diff --git a/docker/reverse_proxy/nginx-development.conf b/docker/reverse_proxy/nginx-development.conf index 4839e56a2..4283e1a33 100644 --- a/docker/reverse_proxy/nginx-development.conf +++ b/docker/reverse_proxy/nginx-development.conf @@ -17,7 +17,7 @@ server { add_header 'Access-Control-Allow-Origin' '*'; sub_filter_once off; sub_filter_types application/json; - sub_filter "https://vector.ehrenamtskarte.app" "http://localhost:5002"; + sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; root /usr/share/nginx/html; index index.html; } diff --git a/docker/reverse_proxy/www/style.json b/docker/reverse_proxy/www/style.json index dd428bb26..c5a978201 100644 --- a/docker/reverse_proxy/www/style.json +++ b/docker/reverse_proxy/www/style.json @@ -22,7 +22,7 @@ "physical_stores": { "type": "vector", "tiles": [ - "https://vector.ehrenamtskarte.app/tiles/rpc/physical_stores/{z}/{x}/{y}.pbf" + "https://tiles.staging.ehrenamtskarte.app/tiles/rpc/physical_stores/{z}/{x}/{y}.pbf" ], "minZoom": 0, "maxZoom": 14 @@ -30,13 +30,13 @@ "physical_stores_clustered": { "type": "vector", "tiles": [ - "https://vector.ehrenamtskarte.app/tiles/rpc/physical_stores_clustered/{z}/{x}/{y}.pbf" + "https://tiles.staging.ehrenamtskarte.app/tiles/rpc/physical_stores_clustered/{z}/{x}/{y}.pbf" ], "minZoom": 0, "maxZoom": 14 } }, - "sprite": "https://vector.ehrenamtskarte.app/sprites/osm-liberty", + "sprite": "https://tiles.staging.ehrenamtskarte.app/sprites/osm-liberty", "glyphs": "https://api.maptiler.com/fonts/{fontstack}/{range}.pbf?key=zOWlRyA7SQOSf0eOTmmm", "layers": [ { @@ -1687,4 +1687,4 @@ } ], "id": "osm-liberty" -} \ No newline at end of file +} diff --git a/docs/staging-setup.md b/docs/staging-setup.md index 8f64553c7..c3f2d61e5 100644 --- a/docs/staging-setup.md +++ b/docs/staging-setup.md @@ -4,7 +4,7 @@ If you want o make the staging environment available on (api|vector).ehrenamtska ```nginx configuration server { - server_name vector.ehrenamtskarte.app; + server_name tiles.staging.ehrenamtskarte.app; location / { proxy_set_header Host $http_host; @@ -19,7 +19,7 @@ server { } server { - server_name api.ehrenamtskarte.app; + server_name tiles.staging.ehrenamtskarte.app; location / { proxy_set_header Host $http_host; @@ -34,7 +34,7 @@ server { } server { - server_name ehrenamtskarte.app; + server_name staging.ehrenamtskarte.app; root /var/www/html; @@ -48,32 +48,32 @@ server { } server { - if ($host = ehrenamtskarte.app) { + if ($host = staging.ehrenamtskarte.app) { return 301 https://$host$request_uri; } listen 80; - server_name ehrenamtskarte.app; + server_name staging.ehrenamtskarte.app; return 404; } server { - if ($host = api.ehrenamtskarte.app) { + if ($host = api.staging.ehrenamtskarte.app) { return 301 https://$host$request_uri; } listen 80; - server_name api.ehrenamtskarte.app; + server_name api.staging.ehrenamtskarte.app; return 404; } server { - if ($host = vector.ehrenamtskarte.app) { + if ($host = tiles.staging..ehrenamtskarte.app) { return 301 https://$host$request_uri; } listen 80; - server_name vector.ehrenamtskarte.app; + server_name tiles.staging..ehrenamtskarte.app; return 404; } ``` diff --git a/frontend/lib/main_prod.dart b/frontend/lib/main_prod.dart index 9f63c7bc8..3d4526a1d 100644 --- a/frontend/lib/main_prod.dart +++ b/frontend/lib/main_prod.dart @@ -6,7 +6,7 @@ import 'configuration.dart'; Future main() async { WidgetsFlutterBinding.ensureInitialized(); runApp(Configuration( - mapStyleUrl: "https://vector.ehrenamtskarte.app/style.json", - graphqlUrl: "https://api.ehrenamtskarte.app", + mapStyleUrl: "https://tiles.staging.ehrenamtskarte.app/style.json", + graphqlUrl: "https://api.staging.ehrenamtskarte.app", child: App())); } From bb8166a976bf0c38cd07dc8ae6b38d6b4a40acf3 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:33:58 +0100 Subject: [PATCH 24/51] Add dart start scripts --- .idea/runConfigurations/Run_Flutter__prod__.xml | 6 ++++++ .idea/runConfigurations/Run_Flutter__staging_.xml | 2 +- frontend/lib/main_prod.dart | 4 ++-- frontend/lib/main_staging.dart | 12 ++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 .idea/runConfigurations/Run_Flutter__prod__.xml create mode 100644 frontend/lib/main_staging.dart diff --git a/.idea/runConfigurations/Run_Flutter__prod__.xml b/.idea/runConfigurations/Run_Flutter__prod__.xml new file mode 100644 index 000000000..7d718e452 --- /dev/null +++ b/.idea/runConfigurations/Run_Flutter__prod__.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_Flutter__staging_.xml b/.idea/runConfigurations/Run_Flutter__staging_.xml index 30b5a0cb5..8d00895e0 100644 --- a/.idea/runConfigurations/Run_Flutter__staging_.xml +++ b/.idea/runConfigurations/Run_Flutter__staging_.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/frontend/lib/main_prod.dart b/frontend/lib/main_prod.dart index 3d4526a1d..5f1a76a1f 100644 --- a/frontend/lib/main_prod.dart +++ b/frontend/lib/main_prod.dart @@ -6,7 +6,7 @@ import 'configuration.dart'; Future main() async { WidgetsFlutterBinding.ensureInitialized(); runApp(Configuration( - mapStyleUrl: "https://tiles.staging.ehrenamtskarte.app/style.json", - graphqlUrl: "https://api.staging.ehrenamtskarte.app", + mapStyleUrl: "https://tiles.ehrenamtskarte.app/style.json", + graphqlUrl: "https://api.ehrenamtskarte.app", child: App())); } diff --git a/frontend/lib/main_staging.dart b/frontend/lib/main_staging.dart new file mode 100644 index 000000000..3d4526a1d --- /dev/null +++ b/frontend/lib/main_staging.dart @@ -0,0 +1,12 @@ +import 'package:flutter/material.dart'; + +import 'app.dart'; +import 'configuration.dart'; + +Future main() async { + WidgetsFlutterBinding.ensureInitialized(); + runApp(Configuration( + mapStyleUrl: "https://tiles.staging.ehrenamtskarte.app/style.json", + graphqlUrl: "https://api.staging.ehrenamtskarte.app", + child: App())); +} From a0cdf94f800135454a77920e15ae22424e8013c0 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:41:02 +0100 Subject: [PATCH 25/51] Add setting to enable production mode --- .../backend/common/webservice/WebService.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt index 91b65ac01..4cb6ec01c 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/WebService.kt @@ -7,13 +7,15 @@ const val DEFAULT_PORT = "7000" class WebService { fun start() { + val production = System.getProperty("app.production", "").isNotEmpty() val host = System.getProperty("app.host", "0.0.0.0") val port = Integer.parseInt(System.getProperty("app.port", DEFAULT_PORT)) val app = Javalin.create { cfg -> - - cfg.enableDevLogging() // FIXME: Disable in dev - cfg.enableCorsForAllOrigins() - cfg.addStaticFiles("/graphiql", "/graphiql", Location.CLASSPATH) + if (!production) { + cfg.enableDevLogging() + cfg.enableCorsForAllOrigins() + cfg.addStaticFiles("/graphiql", "/graphiql", Location.CLASSPATH) + } }.start(host, port) println("Server is running at http://${host}:${port}") From c0396f7b11a58cffa1ddafb80705ac619fb8233d Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 5 Feb 2021 16:42:32 +0100 Subject: [PATCH 26/51] Only for main --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f6697940..c1ee27b1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,6 @@ workflows: branches: only: - main - - 227-deb-packages jobs: build: From 3c936c84970d9d4519ee4ae6e81344c4dafad00f Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 09:58:21 +0100 Subject: [PATCH 27/51] Update intellij configs --- .idea/ehrenamtskarte.iml | 6 +----- administration/administration.iml | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.idea/ehrenamtskarte.iml b/.idea/ehrenamtskarte.iml index 74e11a643..80cc7391b 100644 --- a/.idea/ehrenamtskarte.iml +++ b/.idea/ehrenamtskarte.iml @@ -2,11 +2,7 @@ - - - - - + \ No newline at end of file diff --git a/administration/administration.iml b/administration/administration.iml index 8021953ed..2e839296b 100644 --- a/administration/administration.iml +++ b/administration/administration.iml @@ -2,7 +2,10 @@ - + + + + From f2cdbe7a9a095766f545f075a0be213636c9041b Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:08:51 +0100 Subject: [PATCH 28/51] Update configs and docs --- docker/reverse_proxy/nginx-development.conf | 8 ++++--- docker/reverse_proxy/nginx-staging.conf | 10 +++++---- docs/staging-setup.md | 23 ++++++++++++--------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docker/reverse_proxy/nginx-development.conf b/docker/reverse_proxy/nginx-development.conf index 4283e1a33..b63506b1a 100644 --- a/docker/reverse_proxy/nginx-development.conf +++ b/docker/reverse_proxy/nginx-development.conf @@ -1,12 +1,14 @@ +# Nginx config for routing within the docker network (development) + upstream martin { - server martin:3000; + server martin:3000; # Host within docker network } server { - listen 80; + listen 80 default_server; server_name localhost; - location ~ /tiles/(?.*) { + location ~ /(?.*) { proxy_set_header X-Rewrite-URL $request_uri; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 099b0b674..1fd3b2e1b 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -1,16 +1,18 @@ +# Nginx config for routing within the docker network (staging) + upstream martin { - server martin:3000; + server martin:3000; # Host within docker network } upstream backend { - server backend:7000; + server backend:7000; # Host within docker network } server { - listen 80; + listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; - location ~ /tiles/(?.*) { + location ~ /(?.*) { proxy_set_header X-Rewrite-URL $request_uri; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/docs/staging-setup.md b/docs/staging-setup.md index c3f2d61e5..d7fbb5132 100644 --- a/docs/staging-setup.md +++ b/docs/staging-setup.md @@ -1,8 +1,11 @@ # Setup of staging environment -If you want o make the staging environment available on (api|vector).ehrenamtskarte.app, then the following nginx config can be used on the host: +If you want o make the staging environment available on (api|tiles).ehrenamtskarte.app, then the following nginx config can be used on the host: ```nginx configuration +# Nginx config which tls support for staging environment. Forwards requests to reverse proxy inside the docker network. +# The config used in that reverse proxy can be nginx-development.conf or nginx-staging.conf. + server { server_name tiles.staging.ehrenamtskarte.app; @@ -12,14 +15,14 @@ server { } listen 443 ssl; - ssl_certificate /etc/letsencrypt/live/ehrenamtskarte.app/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/ehrenamtskarte.app/privkey.pem; + ssl_certificate /etc/letsencrypt/live/staging.ehrenamtskarte.app/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/staging.ehrenamtskarte.app/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; } server { - server_name tiles.staging.ehrenamtskarte.app; + server_name api.staging.ehrenamtskarte.app; location / { proxy_set_header Host $http_host; @@ -27,8 +30,8 @@ server { } listen 443 ssl; - ssl_certificate /etc/letsencrypt/live/ehrenamtskarte.app/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/ehrenamtskarte.app/privkey.pem; + ssl_certificate /etc/letsencrypt/live/staging.ehrenamtskarte.app/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/staging.ehrenamtskarte.app/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; } @@ -41,8 +44,8 @@ server { index index.html index.htm index.nginx-debian.html; listen 443 ssl; - ssl_certificate /etc/letsencrypt/live/ehrenamtskarte.app/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/ehrenamtskarte.app/privkey.pem; + ssl_certificate /etc/letsencrypt/live/staging.ehrenamtskarte.app/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/staging.ehrenamtskarte.app/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; } @@ -68,12 +71,12 @@ server { } server { - if ($host = tiles.staging..ehrenamtskarte.app) { + if ($host = tiles.staging.ehrenamtskarte.app) { return 301 https://$host$request_uri; } listen 80; - server_name tiles.staging..ehrenamtskarte.app; + server_name tiles.staging.ehrenamtskarte.app; return 404; } ``` From d8555547597d19be1a94e54db0a2037a89fcfb36 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:18:23 +0100 Subject: [PATCH 29/51] Update staging config --- docker/reverse_proxy/nginx-development.conf | 10 +-------- docker/reverse_proxy/nginx-staging.conf | 25 ++++++++++++++++----- docker/reverse_proxy/www/style.json | 4 ++-- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docker/reverse_proxy/nginx-development.conf b/docker/reverse_proxy/nginx-development.conf index b63506b1a..3aab2df28 100644 --- a/docker/reverse_proxy/nginx-development.conf +++ b/docker/reverse_proxy/nginx-development.conf @@ -8,20 +8,12 @@ server { listen 80 default_server; server_name localhost; + # TODO location ~ /(?.*) { proxy_set_header X-Rewrite-URL $request_uri; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://martin/$fwd_path$is_args$args; } - - location / { - add_header 'Access-Control-Allow-Origin' '*'; - sub_filter_once off; - sub_filter_types application/json; - sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; - root /usr/share/nginx/html; - index index.html; - } } diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 1fd3b2e1b..4107b1c1a 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -12,18 +12,31 @@ server { listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; + location /map { + alias /usr/share/nginx/html/index.html; + } + + location /style.json { + add_header 'Access-Control-Allow-Origin' '*'; + sub_filter_once off; + sub_filter_types application/json; + sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; + alias /usr/share/nginx/html/style.json; + } + + location /sprites { + add_header 'Access-Control-Allow-Origin' '*'; + + root /usr/share/nginx/html/sprites; + index index.html; + } + location ~ /(?.*) { proxy_set_header X-Rewrite-URL $request_uri; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://martin/$fwd_path$is_args$args; } - - location / { - add_header 'Access-Control-Allow-Origin' '*'; - root /usr/share/nginx/html; - index index.html; - } } server { diff --git a/docker/reverse_proxy/www/style.json b/docker/reverse_proxy/www/style.json index c5a978201..11ff135c2 100644 --- a/docker/reverse_proxy/www/style.json +++ b/docker/reverse_proxy/www/style.json @@ -22,7 +22,7 @@ "physical_stores": { "type": "vector", "tiles": [ - "https://tiles.staging.ehrenamtskarte.app/tiles/rpc/physical_stores/{z}/{x}/{y}.pbf" + "https://tiles.staging.ehrenamtskarte.app/rpc/physical_stores/{z}/{x}/{y}.pbf" ], "minZoom": 0, "maxZoom": 14 @@ -30,7 +30,7 @@ "physical_stores_clustered": { "type": "vector", "tiles": [ - "https://tiles.staging.ehrenamtskarte.app/tiles/rpc/physical_stores_clustered/{z}/{x}/{y}.pbf" + "https://tiles.staging.ehrenamtskarte.app/rpc/physical_stores_clustered/{z}/{x}/{y}.pbf" ], "minZoom": 0, "maxZoom": 14 From 78d8b7c07c8e1ee207a4c17b91a37c8c80ae47be Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:23:38 +0100 Subject: [PATCH 30/51] Update nginx config --- docker/reverse_proxy/nginx-staging.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 4107b1c1a..4500a56b1 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -12,11 +12,11 @@ server { listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; - location /map { + location = /map { alias /usr/share/nginx/html/index.html; } - location /style.json { + location = /style.json { add_header 'Access-Control-Allow-Origin' '*'; sub_filter_once off; sub_filter_types application/json; @@ -24,11 +24,11 @@ server { alias /usr/share/nginx/html/style.json; } - location /sprites { + location ^~ /sprites { add_header 'Access-Control-Allow-Origin' '*'; - root /usr/share/nginx/html/sprites; - index index.html; + root /usr/share/nginx/html/sprites; + index index.html; } location ~ /(?.*) { From f8cc06e648c58457eefa24ad6de97b809bfdf1f3 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:30:18 +0100 Subject: [PATCH 31/51] Update nginx config --- docker/reverse_proxy/nginx-staging.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 4500a56b1..42eca8a51 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -12,7 +12,7 @@ server { listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; - location = /map { + location = /map.html { alias /usr/share/nginx/html/index.html; } @@ -24,11 +24,10 @@ server { alias /usr/share/nginx/html/style.json; } - location ^~ /sprites { + location ~ /sprites { add_header 'Access-Control-Allow-Origin' '*'; root /usr/share/nginx/html/sprites; - index index.html; } location ~ /(?.*) { From 80ef3e972e8981a1fae7e727b9f83b9e6a17fe15 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:32:50 +0100 Subject: [PATCH 32/51] use try files --- docker/reverse_proxy/nginx-staging.conf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 42eca8a51..4a53b05c1 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -11,23 +11,27 @@ upstream backend { server { listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; + + root /usr/share/nginx/html location = /map.html { - alias /usr/share/nginx/html/index.html; + try_files map.html; } location = /style.json { add_header 'Access-Control-Allow-Origin' '*'; + sub_filter_once off; sub_filter_types application/json; sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; - alias /usr/share/nginx/html/style.json; + + try_files style.json; } location ~ /sprites { add_header 'Access-Control-Allow-Origin' '*'; - root /usr/share/nginx/html/sprites; + try_files $uri; } location ~ /(?.*) { From e5bae800367f86e8c566e75736b07d6eef8beda4 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:33:50 +0100 Subject: [PATCH 33/51] Update --- docker/reverse_proxy/nginx-staging.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 4a53b05c1..66dd8ce4f 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -12,7 +12,7 @@ server { listen 80 default_server; server_name tiles.staging.ehrenamtskarte.app; - root /usr/share/nginx/html + root /usr/share/nginx/html; location = /map.html { try_files map.html; From f2e8fa4525237103106dabb1a8a566ebfe615225 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:34:03 +0100 Subject: [PATCH 34/51] Update --- docker/reverse_proxy/nginx-staging.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 66dd8ce4f..6315da6ba 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -28,7 +28,7 @@ server { try_files style.json; } - location ~ /sprites { + location ^~ /sprites { add_header 'Access-Control-Allow-Origin' '*'; try_files $uri; From c7e7be6c2c1903007ee82e67bd6a22686b5b81c3 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:36:40 +0100 Subject: [PATCH 35/51] Update --- docker/reverse_proxy/nginx-staging.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 6315da6ba..e05b9de52 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -14,8 +14,8 @@ server { root /usr/share/nginx/html; - location = /map.html { - try_files map.html; + location = /map { + try_files $uri /index.html; } location = /style.json { @@ -25,13 +25,13 @@ server { sub_filter_types application/json; sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; - try_files style.json; + try_files $uri /style.json; } location ^~ /sprites { add_header 'Access-Control-Allow-Origin' '*'; - try_files $uri; + try_files $uri $uri; } location ~ /(?.*) { From 132e2ec17255c571216067049451c746e5c8aa76 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:41:49 +0100 Subject: [PATCH 36/51] Set index --- docker/reverse_proxy/nginx-staging.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index e05b9de52..3a87b1272 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -15,9 +15,9 @@ server { root /usr/share/nginx/html; location = /map { - try_files $uri /index.html; + index index.html; } - + location = /style.json { add_header 'Access-Control-Allow-Origin' '*'; From dc61d697f038f98015b4f4379b15109b81b2883d Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:43:00 +0100 Subject: [PATCH 37/51] Add try files --- docker/reverse_proxy/nginx-staging.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 3a87b1272..c8d667593 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -16,6 +16,7 @@ server { location = /map { index index.html; + try_files $uri /index.html; } location = /style.json { From 2db5bfd6b6268c095093a7294060ee212f018528 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:46:10 +0100 Subject: [PATCH 38/51] Add try files --- docker/reverse_proxy/nginx-staging.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index c8d667593..d98f06548 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -14,8 +14,7 @@ server { root /usr/share/nginx/html; - location = /map { - index index.html; + location = /map.html { try_files $uri /index.html; } From 6f4e3661aea66619e6ff7f16edf1c48526b82232 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 10:54:11 +0100 Subject: [PATCH 39/51] Fix config --- docker/reverse_proxy/nginx-development.conf | 22 ++++++++++++++++++- docker/reverse_proxy/nginx-staging.conf | 1 - .../www/{index.html => map.html} | 7 +++--- 3 files changed, 25 insertions(+), 5 deletions(-) rename docker/reverse_proxy/www/{index.html => map.html} (53%) diff --git a/docker/reverse_proxy/nginx-development.conf b/docker/reverse_proxy/nginx-development.conf index 3aab2df28..20507c826 100644 --- a/docker/reverse_proxy/nginx-development.conf +++ b/docker/reverse_proxy/nginx-development.conf @@ -8,7 +8,27 @@ server { listen 80 default_server; server_name localhost; - # TODO + root /usr/share/nginx/html; + + location = /map.html { + } + + location = /style.json { + add_header 'Access-Control-Allow-Origin' '*'; + + sub_filter_once off; + sub_filter_types application/json; + sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; + + try_files $uri /style.json; + } + + location ^~ /sprites { + add_header 'Access-Control-Allow-Origin' '*'; + + try_files $uri $uri; + } + location ~ /(?.*) { proxy_set_header X-Rewrite-URL $request_uri; proxy_set_header X-Forwarded-Host $host; diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index d98f06548..90ef06f3b 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -15,7 +15,6 @@ server { root /usr/share/nginx/html; location = /map.html { - try_files $uri /index.html; } location = /style.json { diff --git a/docker/reverse_proxy/www/index.html b/docker/reverse_proxy/www/map.html similarity index 53% rename from docker/reverse_proxy/www/index.html rename to docker/reverse_proxy/www/map.html index 6c860a0fe..2072cb07d 100644 --- a/docker/reverse_proxy/www/index.html +++ b/docker/reverse_proxy/www/map.html @@ -1,7 +1,8 @@ - + - - + + + Ehrenamtskarte Preview Map
From 460cceb1519aca71a68904fe0f53ac298f902659 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 11:06:00 +0100 Subject: [PATCH 40/51] Do not rewrite in staging --- docker/reverse_proxy/nginx-staging.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/reverse_proxy/nginx-staging.conf b/docker/reverse_proxy/nginx-staging.conf index 90ef06f3b..32e7cb8d6 100644 --- a/docker/reverse_proxy/nginx-staging.conf +++ b/docker/reverse_proxy/nginx-staging.conf @@ -20,10 +20,6 @@ server { location = /style.json { add_header 'Access-Control-Allow-Origin' '*'; - sub_filter_once off; - sub_filter_types application/json; - sub_filter "https://tiles.staging.ehrenamtskarte.app" "http://localhost:5002"; - try_files $uri /style.json; } From c055d393565e8403bbd18dad91531b818f7b9111 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sat, 6 Feb 2021 11:07:51 +0100 Subject: [PATCH 41/51] add access token --- docker/reverse_proxy/www/map.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/reverse_proxy/www/map.html b/docker/reverse_proxy/www/map.html index 2072cb07d..42a529310 100644 --- a/docker/reverse_proxy/www/map.html +++ b/docker/reverse_proxy/www/map.html @@ -7,6 +7,7 @@
- + Ehrenamtskarte Preview Map
-