Modify api-db and keycloak-db to support MySQL8

.github/workflows/test-db-migrations.yml

@@ -47,7 +47,7 @@ jobs:
       -
         name: Show initial migration logs
         run: |
-          docker compose -p lagoon logs api-init
+          docker compose -p lagoon logs api-db-init
       -
         name: Initiate rollback
         run: |

Makefile

@@ -94,13 +94,21 @@ KUBECTL = $(realpath ./local-dev/kubectl)
 JQ = $(realpath ./local-dev/jq)
 K3D = $(realpath ./local-dev/k3d)
 
+# which database vendor type to use, can be mariadb (default) or mysql
+# DATABASE_VENDOR = mariadb
+DATABASE_VENDOR = mysql
+DATABASE_DOCKERFILE = Dockerfile
+ifeq ($(DATABASE_VENDOR), mysql)
+DATABASE_DOCKERFILE = Dockerfile.mysql
+endif
+
 #######
 ####### Functions
 #######
 
 # Builds a docker image. Expects as arguments: name of the image, location of Dockerfile, path of
 # Docker Build Context
-docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
+docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
 
 docker_buildx_create = 	docker buildx create --name $(CI_BUILD_TAG) || echo  -e '$(CI_BUILD_TAG) builder already present\n'
 
@@ -186,13 +194,13 @@ $(build-services):
 
 # Dependencies of Service Images
 build/auth-server build/webhook-handler build/webhooks2tasks build/api: build/yarn-workspace-builder
-build/api-db: services/api-db/Dockerfile
+build/api-db: services/api-db/$(DATABASE_DOCKERFILE)
 build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
 build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile
-build/keycloak-db: services/keycloak-db/Dockerfile
+build/keycloak-db: services/keycloak-db/$(DATABASE_DOCKERFILE)
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
 build/tests: tests/Dockerfile
@@ -305,14 +313,17 @@ webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test
 
 .PHONY: publish-testlagoon-images
 publish-testlagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 # tag and push all images
 
 .PHONY: publish-uselagoon-images
 publish-uselagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 .PHONY: clean
 clean:
@@ -413,7 +424,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.11.0
 K3D_IMAGE = docker.io/rancher/k3s:v1.31.0-k3s1
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = main
+CHARTS_TREEISH = mysql-image-support
 TASK_IMAGES = task-activestandby
 
 # the name of the docker network to create
@@ -569,6 +580,7 @@ k3d/test: k3d/setup
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_ALL_DEPS=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
@@ -603,6 +615,7 @@ k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_INSTALL_REGISTRY=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
@@ -662,6 +675,7 @@ k3d/dev: build
 		&& $(MAKE) install-lagoon-core DOCKER_NETWORK=$(DOCKER_NETWORK) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \
 			HELM=$(HELM) KUBECTL=$(KUBECTL) \
 			JQ=$(JQ) \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \
 			$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \
 			$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \
@@ -786,6 +800,7 @@ k3d/retest:
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_ALL_DEPS=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \

docker-bake.hcl

@@ -15,6 +15,14 @@ variable "UPSTREAM_REPO" {
   default = "uselagoon"
 }
 
+variable "DATABASE_DOCKERFILE" {
+  default = "Dockerfile"
+}
+
+variable "DATABASE_VENDOR" {
+  default = "mariadb"
+}
+
 variable "UPSTREAM_TAG" {
   default = "latest"
 }
@@ -39,6 +47,7 @@ target "default"{
     LAGOON_VERSION = "${LAGOON_VERSION}"
     UPSTREAM_REPO = "${UPSTREAM_REPO}"
     UPSTREAM_TAG = "${UPSTREAM_TAG}"
+    DATABASE_VENDOR = "${DATABASE_VENDOR}"
   }
 }
 
@@ -131,6 +140,7 @@ target "api" {
 target "api-db" {
   inherits = ["default"]
   context = "services/api-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/api-db - the MariaDB database service for Lagoon API"
   }
@@ -206,6 +216,7 @@ target "keycloak" {
 target "keycloak-db" {
   inherits = ["default"]
   context = "services/keycloak-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/keycloak-db - the MariaDB database service for Lagoon Keycloak"
   }

docker-compose.local-dev.yaml

@@ -29,6 +29,7 @@ services:
   api:
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
     volumes:
+      - ./services/api/database:/app/services/api/database
       - ./services/api/src:/app/services/api/src
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist

services/api-db/Dockerfile

@@ -12,6 +12,10 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
+RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash
+
 USER mysql
 
 ENV MARIADB_DATABASE=infrastructure \

services/api-db/Dockerfile.mysql

@@ -0,0 +1,31 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+USER root
+
+COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
+RUN chown -R mysql /legacy-migration-scripts/ \
+    && /bin/fix-permissions /legacy-migration-scripts/
+
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
+COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
+
+USER mysql
+
+ENV MYSQL_DATABASE=infrastructure \
+    MYSQL_USER=api \
+    MYSQL_PASSWORD=api
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# do we need to keep these any more? v2.10.0 was long ago...
+# COPY ./rerun_initdb.sh /rerun_initdb.sh
+# COPY ./legacy_rerun_initdb.sh /legacy_rerun_initdb.sh
+
+CMD ["mysqld", "--sql_mode", ""]

services/api-db/mysql-init.bash

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ "$(ls -A /etc/mysql/conf.d/)" ]; then
+   ep /etc/mysql/conf.d/*
+fi
+
+if [ "${1:0:1}" = '-' ]; then
+  set -- mysqld "$@"
+fi
+
+wantHelp=
+for arg; do
+  case "$arg" in
+    -'?'|--help|--print-defaults|-V|--version)
+      wantHelp=1
+      break
+      ;;
+  esac
+done
+
+# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder
+# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container).
+# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting
+# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system
+# the data will be prefilled and persistent across container restarts.
+if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying"
+  else
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'"
+    CUR_DIR=${PWD}
+    cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -)
+    cd $CUR_DIR
+  fi
+fi
+
+ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf
+
+if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
+  if [ ! -d "/run/mysqld" ]; then
+    mkdir -p /var/run/mysqld
+    chown -R mysql:mysql /var/run/mysqld
+  fi
+
+  MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30}
+  MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1}
+
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MySQL directory already present, skipping creation"
+
+    echo "starting mysql for mysql upgrade."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+  else
+    echo "MySQL data directory not found, creating initial DBs"
+
+    /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr
+
+    echo "starting mysql for initdb.d import."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then
+      MYSQL_ROOT_PASSWORD=`pwgen 16 1`
+      echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD"
+    fi
+
+    MYSQL_DATABASE=${MYSQL_DATABASE:-""}
+    MYSQL_USER=${MYSQL_USER:-""}
+    MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}
+
+    tfile=`mktemp`
+    if [ ! -f "$tfile" ]; then
+        return 1
+    fi
+
+    cat << EOF > $tfile
+DROP DATABASE IF EXISTS test;
+USE mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
+DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
+FLUSH PRIVILEGES;
+
+EOF
+
+    if [ "$MYSQL_DATABASE" != "" ]; then
+      echo "[i] Creating database: $MYSQL_DATABASE"
+      echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile
+      if [ "$MYSQL_USER" != "" ]; then
+        echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD"
+        echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
+        echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile
+      fi
+    fi
+
+
+    cat $tfile
+    cat $tfile | mysql -v -u root
+    rm -v -f $tfile
+
+    echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "password=${MYSQL_ROOT_PASSWORD}"  >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+
+    for f in /docker-entrypoint-initdb.d/*; do
+      if [ -e "$f" ]; then
+        case "$f" in
+          *.sh)     echo "$0: running $f"; . "$f" ;;
+          *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;;
+          *)        echo "$0: ignoring $f" ;;
+        esac
+      fi
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+
+  fi
+
+  echo "done, now starting daemon"
+  touch /tmp/startup-init-complete
+  touch /tmp/mysql-init-complete
+
+fi

services/api-db/password-entrypoint.bash

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ ${API_DB_PASSWORD+x} ]; then
+    if [ "${LAGOON}" == "mysql" ]; then
+        export MYSQL_PASSWORD=${API_DB_PASSWORD}
+    else
+        export MARIADB_PASSWORD=${API_DB_PASSWORD}
+    fi
+fi