diff --git a/.github/workflows/test-db-migrations.yml b/.github/workflows/test-db-migrations.yml index e38d94750c..04c8a4963e 100644 --- a/.github/workflows/test-db-migrations.yml +++ b/.github/workflows/test-db-migrations.yml @@ -47,7 +47,7 @@ jobs: - name: Show initial migration logs run: | - docker compose -p lagoon logs api-init + docker compose -p lagoon logs api-db-init - name: Initiate rollback run: | diff --git a/Makefile b/Makefile index a19f43d704..eafac67161 100644 --- a/Makefile +++ b/Makefile @@ -94,13 +94,21 @@ KUBECTL = $(realpath ./local-dev/kubectl) JQ = $(realpath ./local-dev/jq) K3D = $(realpath ./local-dev/k3d) +# which database vendor type to use, can be mariadb (default) or mysql +# DATABASE_VENDOR = mariadb +DATABASE_VENDOR = mysql +DATABASE_DOCKERFILE = Dockerfile +ifeq ($(DATABASE_VENDOR), mysql) +DATABASE_DOCKERFILE = Dockerfile.mysql +endif + ####### ####### Functions ####### # Builds a docker image. Expects as arguments: name of the image, location of Dockerfile, path of # Docker Build Context -docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1) +docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1) docker_buildx_create = docker buildx create --name $(CI_BUILD_TAG) || echo -e '$(CI_BUILD_TAG) builder already present\n' @@ -186,13 +194,13 @@ $(build-services): # Dependencies of Service Images build/auth-server build/webhook-handler build/webhooks2tasks build/api: build/yarn-workspace-builder -build/api-db: services/api-db/Dockerfile +build/api-db: services/api-db/$(DATABASE_DOCKERFILE) build/api-redis: services/api-redis/Dockerfile build/actions-handler: services/actions-handler/Dockerfile build/backup-handler: services/backup-handler/Dockerfile build/broker: services/broker/Dockerfile build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile -build/keycloak-db: services/keycloak-db/Dockerfile +build/keycloak-db: services/keycloak-db/$(DATABASE_DOCKERFILE) build/keycloak: services/keycloak/Dockerfile build/logs2notifications: services/logs2notifications/Dockerfile build/tests: tests/Dockerfile @@ -305,14 +313,17 @@ webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test .PHONY: publish-testlagoon-images publish-testlagoon-images: - PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + # PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push # tag and push all images .PHONY: publish-uselagoon-images publish-uselagoon-images: - PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push - PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + # PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push + # PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push .PHONY: clean clean: @@ -413,7 +424,7 @@ STERN_VERSION = v2.6.1 CHART_TESTING_VERSION = v3.11.0 K3D_IMAGE = docker.io/rancher/k3s:v1.31.0-k3s1 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows] -CHARTS_TREEISH = main +CHARTS_TREEISH = mysql-image-support TASK_IMAGES = task-activestandby # the name of the docker network to create @@ -569,6 +580,7 @@ k3d/test: k3d/setup OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ SKIP_ALL_DEPS=true \ + CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \ LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ USE_CALICO_CNI=false \ LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ @@ -603,6 +615,7 @@ k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ SKIP_INSTALL_REGISTRY=true \ + CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \ LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ USE_CALICO_CNI=false \ LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ @@ -662,6 +675,7 @@ k3d/dev: build && $(MAKE) install-lagoon-core DOCKER_NETWORK=$(DOCKER_NETWORK) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ HELM=$(HELM) KUBECTL=$(KUBECTL) \ JQ=$(JQ) \ + CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \ OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \ $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \ $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \ @@ -786,6 +800,7 @@ k3d/retest: OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ SKIP_ALL_DEPS=true \ + CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \ LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ USE_CALICO_CNI=false \ LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ diff --git a/docker-bake.hcl b/docker-bake.hcl index 30df12fd1e..996f9eb450 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -15,6 +15,14 @@ variable "UPSTREAM_REPO" { default = "uselagoon" } +variable "DATABASE_DOCKERFILE" { + default = "Dockerfile" +} + +variable "DATABASE_VENDOR" { + default = "mariadb" +} + variable "UPSTREAM_TAG" { default = "latest" } @@ -39,6 +47,7 @@ target "default"{ LAGOON_VERSION = "${LAGOON_VERSION}" UPSTREAM_REPO = "${UPSTREAM_REPO}" UPSTREAM_TAG = "${UPSTREAM_TAG}" + DATABASE_VENDOR = "${DATABASE_VENDOR}" } } @@ -131,6 +140,7 @@ target "api" { target "api-db" { inherits = ["default"] context = "services/api-db" + dockerfile = "${DATABASE_DOCKERFILE}" labels = { "org.opencontainers.image.title": "lagoon-core/api-db - the MariaDB database service for Lagoon API" } @@ -206,6 +216,7 @@ target "keycloak" { target "keycloak-db" { inherits = ["default"] context = "services/keycloak-db" + dockerfile = "${DATABASE_DOCKERFILE}" labels = { "org.opencontainers.image.title": "lagoon-core/keycloak-db - the MariaDB database service for Lagoon Keycloak" } diff --git a/docker-compose.local-dev.yaml b/docker-compose.local-dev.yaml index f5e00b0ce5..53140cb57c 100644 --- a/docker-compose.local-dev.yaml +++ b/docker-compose.local-dev.yaml @@ -29,6 +29,7 @@ services: api: command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index" volumes: + - ./services/api/database:/app/services/api/database - ./services/api/src:/app/services/api/src - ./node-packages:/app/node-packages:delegated - /app/node-packages/commons/dist diff --git a/services/api-db/Dockerfile b/services/api-db/Dockerfile index 93098a7ade..57ecf32b8f 100644 --- a/services/api-db/Dockerfile +++ b/services/api-db/Dockerfile @@ -12,6 +12,10 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/ RUN chown -R mysql /legacy-migration-scripts/ \ && /bin/fix-permissions /legacy-migration-scripts/ +# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image +COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh +RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash + USER mysql ENV MARIADB_DATABASE=infrastructure \ diff --git a/services/api-db/Dockerfile.mysql b/services/api-db/Dockerfile.mysql new file mode 100644 index 0000000000..e77db81364 --- /dev/null +++ b/services/api-db/Dockerfile.mysql @@ -0,0 +1,31 @@ +ARG UPSTREAM_REPO +ARG UPSTREAM_TAG +FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest} + +ARG LAGOON_VERSION +ENV LAGOON_VERSION=$LAGOON_VERSION + +USER root + +COPY ./legacy-migration-scripts/* /legacy-migration-scripts/ +RUN chown -R mysql /legacy-migration-scripts/ \ + && /bin/fix-permissions /legacy-migration-scripts/ + +# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image +COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh +COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash + +USER mysql + +ENV MYSQL_DATABASE=infrastructure \ + MYSQL_USER=api \ + MYSQL_PASSWORD=api + # are these needed?? + # MYSQL_CHARSET=utf8 \ + # MYSQL_COLLATION=utf8_general_ci + +# do we need to keep these any more? v2.10.0 was long ago... +# COPY ./rerun_initdb.sh /rerun_initdb.sh +# COPY ./legacy_rerun_initdb.sh /legacy_rerun_initdb.sh + +CMD ["mysqld", "--sql_mode", ""] \ No newline at end of file diff --git a/services/api-db/mysql-init.bash b/services/api-db/mysql-init.bash new file mode 100644 index 0000000000..38ae24a854 --- /dev/null +++ b/services/api-db/mysql-init.bash @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$(ls -A /etc/mysql/conf.d/)" ]; then + ep /etc/mysql/conf.d/* +fi + +if [ "${1:0:1}" = '-' ]; then + set -- mysqld "$@" +fi + +wantHelp= +for arg; do + case "$arg" in + -'?'|--help|--print-defaults|-V|--version) + wantHelp=1 + break + ;; + esac +done + +# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder +# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container). +# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting +# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system +# the data will be prefilled and persistent across container restarts. +if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying" + else + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'" + CUR_DIR=${PWD} + cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -) + cd $CUR_DIR + fi +fi + +ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf + +if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then + if [ ! -d "/run/mysqld" ]; then + mkdir -p /var/run/mysqld + chown -R mysql:mysql /var/run/mysqld + fi + + MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30} + MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1} + + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MySQL directory already present, skipping creation" + + echo "starting mysql for mysql upgrade." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + else + echo "MySQL data directory not found, creating initial DBs" + + /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr + + echo "starting mysql for initdb.d import." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then + MYSQL_ROOT_PASSWORD=`pwgen 16 1` + echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" + fi + + MYSQL_DATABASE=${MYSQL_DATABASE:-""} + MYSQL_USER=${MYSQL_USER:-""} + MYSQL_PASSWORD=${MYSQL_PASSWORD:-""} + + tfile=`mktemp` + if [ ! -f "$tfile" ]; then + return 1 + fi + + cat << EOF > $tfile +DROP DATABASE IF EXISTS test; +USE mysql; +ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD'; +DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1'); +FLUSH PRIVILEGES; + +EOF + + if [ "$MYSQL_DATABASE" != "" ]; then + echo "[i] Creating database: $MYSQL_DATABASE" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile + if [ "$MYSQL_USER" != "" ]; then + echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile + fi + fi + + + cat $tfile + cat $tfile | mysql -v -u root + rm -v -f $tfile + + echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "password=${MYSQL_ROOT_PASSWORD}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + + for f in /docker-entrypoint-initdb.d/*; do + if [ -e "$f" ]; then + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.sql) echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;; + *) echo "$0: ignoring $f" ;; + esac + fi + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + + fi + + echo "done, now starting daemon" + touch /tmp/startup-init-complete + touch /tmp/mysql-init-complete + +fi \ No newline at end of file diff --git a/services/api-db/password-entrypoint.bash b/services/api-db/password-entrypoint.bash new file mode 100644 index 0000000000..1064d2c36f --- /dev/null +++ b/services/api-db/password-entrypoint.bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ ${API_DB_PASSWORD+x} ]; then + if [ "${LAGOON}" == "mysql" ]; then + export MYSQL_PASSWORD=${API_DB_PASSWORD} + else + export MARIADB_PASSWORD=${API_DB_PASSWORD} + fi +fi \ No newline at end of file diff --git a/services/api/database/migrations/20220908065119_initial_db.js b/services/api/database/migrations/20220908065119_initial_db.js index 75051bc77e..412cc63e90 100644 --- a/services/api/database/migrations/20220908065119_initial_db.js +++ b/services/api/database/migrations/20220908065119_initial_db.js @@ -9,18 +9,18 @@ .createTable('advanced_task_definition', function (table) { table.increments('id').notNullable().primary(); table.string('name', 300).notNullable(); - table.specificType('description', 'text').defaultTo('').notNullable(); + table.specificType('description', 'text').notNullable(); table.string('image', 2000).defaultTo(''); table.string('service', 100); table.string('type', 100).notNullable(); table.integer('environment'); table.integer('project'); - table.string('group_name', 2000); + table.string('group_name', 300); table.enu('permission', ['GUEST', 'DEVELOPER', 'MAINTAINER']).defaultTo('GUEST'); - table.specificType('command', 'text').defaultTo(''); + table.specificType('command', 'text'); table.string('confirmation_text', 2000); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); - table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.timestamp('deleted').notNullable(); table.unique(['name', 'environment', 'project', 'group_name'], {indexName: 'name', storageEngineIndexType: 'hash'}); }) .createTable('advanced_task_definition_argument', function (table) { @@ -76,8 +76,8 @@ table.string('name', 100); table.integer('project'); table.enu('deploy_type', ['branch', 'pullrequest', 'promote']).notNullable(); - table.string('deploy_base_ref', 100); - table.string('deploy_head_ref', 100); + table.string('deploy_base_ref', 250); + table.string('deploy_head_ref', 250); table.string('deploy_title', 300); table.enu('environment_type', ['production', 'development']).notNullable(); table.integer('auto_idle', 1).notNullable().defaultTo(1); @@ -89,7 +89,7 @@ table.string('openshift_project_pattern', 300); table.timestamp('updated').notNullable().defaultTo(knex.fn.now()); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); - table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.timestamp('deleted').notNullable(); table.unique(['project', 'name', 'deleted'], {indexName: 'project_name_deleted'}); }) .createTable('environment_backup', function (table) { @@ -98,7 +98,7 @@ table.string('source', 300); table.string('backup_id', 300).unique({indexName:'backup_id'}); table.timestamp('created').notNullable().defaultTo(knex.raw('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP')); - table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.timestamp('deleted').notNullable(); table.index('environment', 'backup_environment'); }) .createTable('environment_fact', function (table) { @@ -109,9 +109,9 @@ table.string('value', 300).notNullable(); table.enu('type', ['TEXT', 'URL', 'SEMVER']).defaultTo('TEXT'); table.string('source', 300).defaultTo(''); - table.specificType('description', 'text').defaultTo(''); + table.specificType('description', 'text'); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); - table.specificType('category', 'text').defaultTo(''); + table.specificType('category', 'text'); table.boolean('key_fact').notNullable().defaultTo(0); table.unique(['environment', 'name', 'source'], {indexName: 'environment_fact'}); }) @@ -126,17 +126,17 @@ table.integer('environment'); table.string('severity', 300).defaultTo(''); table.decimal('severity_score', 1, 1).defaultTo(0); - table.string('identifier', 300).notNullable(); - table.string('lagoon_service', 300).defaultTo(''); + table.string('identifier', 100).notNullable(); + table.string('lagoon_service', 100).defaultTo(''); table.string('source', 300).defaultTo(''); table.string('associated_package', 300).defaultTo(''); - table.specificType('description', 'text').defaultTo(''); - table.string('version', 300).defaultTo(''); + table.specificType('description', 'text'); + table.string('version', 100).defaultTo(''); table.string('fixed_version', 300).defaultTo(''); table.string('links', 300).defaultTo(''); table.json('data'); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); - table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.timestamp('deleted').notNullable(); table.unique(['environment', 'lagoon_service', 'version', 'identifier', 'deleted'], {indexName: 'environment'}); }) .createTable('environment_service', function (table) { @@ -235,7 +235,7 @@ table.integer('development_environments_limit'); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); table.string('private_key', 5000); - table.json('metadata').defaultTo('{}'); + table.json('metadata'); table.string('router_pattern', 300).defaultTo(null); }) .createTable('project_notification', function (table) { @@ -251,7 +251,7 @@ table.string('filename', 100).notNullable(); table.text('s3_key').notNullable(); table.datetime('created').notNullable().defaultTo(knex.fn.now()); - table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.datetime('deleted').notNullable(); }) .createTable('ssh_key', function (table) { table.increments('id').notNullable().primary(); @@ -295,7 +295,7 @@ table.integer('project', 11).notNullable(); table.integer('advanced_task_definition', 11).notNullable(); table.timestamp('created').notNullable().defaultTo(knex.fn.now()); - table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00'); + table.timestamp('deleted').notNullable(); }) } else { diff --git a/services/api/database/migrations/20230801000000_organizations.js b/services/api/database/migrations/20230801000000_organizations.js index 8c63375555..be8a466ad6 100644 --- a/services/api/database/migrations/20230801000000_organizations.js +++ b/services/api/database/migrations/20230801000000_organizations.js @@ -10,7 +10,7 @@ exports.up = async function(knex) { table.increments('id').notNullable().primary(); table.string('name', 300).unique({indexName: 'name'}); table.string('friendly_name', 300).notNullable(); - table.specificType('description', 'text').defaultTo('').notNullable(); + table.specificType('description', 'text').notNullable(); table.integer('quota_project').defaultTo(1).notNullable(); table.integer('quota_group').defaultTo(10).notNullable(); table.integer('quota_notification').defaultTo(10).notNullable(); diff --git a/services/api/database/migrations/20241204000000_source_type_enums.js b/services/api/database/migrations/20241204000000_source_type_enums.js new file mode 100644 index 0000000000..ef2afea9bb --- /dev/null +++ b/services/api/database/migrations/20241204000000_source_type_enums.js @@ -0,0 +1,22 @@ +exports.up = function(knex) { + // fix the way the enum values are stored to be lowercase for mysql strict + return Promise.all([ + knex('deployment') + .where('source_type', '=', 'API') + .update('source_type', 'api'), + knex('deployment') + .where('source_type', '=', 'WEBHOOK') + .update('source_type', 'webhook'), + knex('task') + .where('source_type', '=', 'API') + .update('source_type', 'api'), + ]); + }; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = function(knex) { +return knex.schema; +}; \ No newline at end of file diff --git a/services/api/database/migrations/20241204000100_mysql8_compatibility.js b/services/api/database/migrations/20241204000100_mysql8_compatibility.js new file mode 100644 index 0000000000..da891923f4 --- /dev/null +++ b/services/api/database/migrations/20241204000100_mysql8_compatibility.js @@ -0,0 +1,79 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + return knex.schema + .alterTable('advanced_task_definition', function (table) { + table.specificType('description', 'text').notNullable().alter(); + table.string('group_name', 300).alter(); + table.specificType('command', 'text').alter(); + table.timestamp('deleted').notNullable().alter(); + }) + .alterTable('environment', function (table) { + table.timestamp('deleted').notNullable().alter(); + table.string('deploy_base_ref', 250).alter(); + table.string('deploy_head_ref', 250).alter(); + }) + .alterTable('environment_fact', function (table) { + table.specificType('description', 'text').alter(); + table.specificType('category', 'text').alter(); + }) + .alterTable('environment_problem', function (table) { + table.string('identifier', 100).notNullable().alter(); + table.string('lagoon_service', 100).defaultTo('').alter(); + table.specificType('description', 'text').alter(); + table.string('version', 100).defaultTo('').alter(); + table.timestamp('deleted').notNullable().alter(); + }) + .alterTable('organization', function (table) { + table.specificType('description', 'text').notNullable().alter(); + }) + .alterTable('s3_file', function (table) { + table.timestamp('created').notNullable().defaultTo(knex.fn.now()).alter(); + table.timestamp('deleted').notNullable().alter(); + }) + .alterTable('workflow', function (table) { + table.timestamp('deleted').notNullable().alter(); + }) +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + // caveats around this are that the rollback can only work while data is still saved in keycloak attributes + // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations + // for any group project associations made after the attribute removal + return knex.schema + .alterTable('advanced_task_definition', function (table) { + table.specificType('description', 'text').defaultTo('').notNullable().alter(); + table.string('group_name', 2000).alter(); + table.specificType('command', 'text').defaultTo('').alter(); + }) + .alterTable('environment', function (table) { + table.string('deploy_base_ref', 100).alter(); + table.string('deploy_head_ref', 100).alter(); + }) + .alterTable('environment_fact', function (table) { + table.specificType('description', 'text').defaultTo('').alter(); + table.specificType('category', 'text').defaultTo('').alter(); + }) + .alterTable('environment_problem', function (table) { + table.string('identifier', 300).notNullable().alter(); + table.string('lagoon_service', 300).defaultTo('').alter(); + table.specificType('description', 'text').defaultTo('').alter(); + table.string('version', 300).defaultTo('').alter(); + }) + .alterTable('project', function (table) { + table.json('metadata').defaultTo('{}').alter(); + }) + .alterTable('s3_file', function (table) { + table.datetime('created').notNullable().defaultTo(knex.fn.now()).alter(); + table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00').alter(); + }) + .alterTable('organization', function (table) { + table.specificType('description', 'text').notNullable().defaultTo('').alter(); + }) +}; diff --git a/services/api/src/models/environment.ts b/services/api/src/models/environment.ts index ba2588e655..10de050f16 100644 --- a/services/api/src/models/environment.ts +++ b/services/api/src/models/environment.ts @@ -13,7 +13,7 @@ export interface Environment { openshiftProjectName?: string; // varchar(100) COLLATE utf8_bin DEFAULT NULL, updated?: string; // timestamp NOT NULL DEFAULT current_timestamp(), created?: string; // timestamp NOT NULL DEFAULT current_timestamp(), - deleted?: string; // timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + deleted?: string; route?: string; // varchar(300) COLLATE utf8_bin DEFAULT NULL, routes?: string; // text COLLATE utf8_bin DEFAULT NULL, autoIdle?: Boolean; // int(1) NOT NULL DEFAULT 1, diff --git a/services/api/src/resources/deployment/resolvers.ts b/services/api/src/resources/deployment/resolvers.ts index 28908dbc9c..28d815518f 100644 --- a/services/api/src/resources/deployment/resolvers.ts +++ b/services/api/src/resources/deployment/resolvers.ts @@ -343,6 +343,9 @@ export const addDeployment: ResolverFn = async ( if (!sourceType) { sourceType = "API" } + if (sourceType) { + sourceType.toLocaleLowerCase(); + } const { insertId } = await query( sqlClientPool, Sql.insertDeployment({ diff --git a/services/api/src/resources/task/helpers.ts b/services/api/src/resources/task/helpers.ts index db28a8c71e..62d8820efe 100644 --- a/services/api/src/resources/task/helpers.ts +++ b/services/api/src/resources/task/helpers.ts @@ -73,6 +73,9 @@ export const Helpers = (sqlClientPool: Pool, hasPermission, adminScopes) => { sourceUser: string; sourceType: string; }) => { + if (sourceType) { + sourceType.toLocaleLowerCase(); + } const { insertId } = await query( sqlClientPool, Sql.insertTask({ diff --git a/services/keycloak-db/Dockerfile b/services/keycloak-db/Dockerfile index 6f26eb61a3..81bb03972c 100644 --- a/services/keycloak-db/Dockerfile +++ b/services/keycloak-db/Dockerfile @@ -13,5 +13,8 @@ ENV MARIADB_DATABASE=keycloak \ COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf USER root +# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image +COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh +RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash RUN fix-permissions /etc/mysql/conf.d/ USER mysql diff --git a/services/keycloak-db/Dockerfile.mysql b/services/keycloak-db/Dockerfile.mysql new file mode 100644 index 0000000000..c941c0439f --- /dev/null +++ b/services/keycloak-db/Dockerfile.mysql @@ -0,0 +1,25 @@ +ARG UPSTREAM_REPO +ARG UPSTREAM_TAG +FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest} + +ARG LAGOON_VERSION +ENV LAGOON_VERSION=$LAGOON_VERSION + +ENV MYSQL_DATABASE=keycloak \ + MYSQL_USER=keycloak \ + MYSQL_PASSWORD=keycloak + # are these needed?? + # MYSQL_CHARSET=utf8 \ + # MYSQL_COLLATION=utf8_general_ci + +USER root +# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image +COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh +COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash +USER mysql + +# not used in mysql8 +# COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf +# USER root +# RUN fix-permissions /etc/mysql/conf.d/ +# USER mysql diff --git a/services/keycloak-db/mysql-init.bash b/services/keycloak-db/mysql-init.bash new file mode 100644 index 0000000000..38ae24a854 --- /dev/null +++ b/services/keycloak-db/mysql-init.bash @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$(ls -A /etc/mysql/conf.d/)" ]; then + ep /etc/mysql/conf.d/* +fi + +if [ "${1:0:1}" = '-' ]; then + set -- mysqld "$@" +fi + +wantHelp= +for arg; do + case "$arg" in + -'?'|--help|--print-defaults|-V|--version) + wantHelp=1 + break + ;; + esac +done + +# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder +# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container). +# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting +# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system +# the data will be prefilled and persistent across container restarts. +if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying" + else + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'" + CUR_DIR=${PWD} + cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -) + cd $CUR_DIR + fi +fi + +ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf + +if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then + if [ ! -d "/run/mysqld" ]; then + mkdir -p /var/run/mysqld + chown -R mysql:mysql /var/run/mysqld + fi + + MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30} + MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1} + + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MySQL directory already present, skipping creation" + + echo "starting mysql for mysql upgrade." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + else + echo "MySQL data directory not found, creating initial DBs" + + /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr + + echo "starting mysql for initdb.d import." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then + MYSQL_ROOT_PASSWORD=`pwgen 16 1` + echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" + fi + + MYSQL_DATABASE=${MYSQL_DATABASE:-""} + MYSQL_USER=${MYSQL_USER:-""} + MYSQL_PASSWORD=${MYSQL_PASSWORD:-""} + + tfile=`mktemp` + if [ ! -f "$tfile" ]; then + return 1 + fi + + cat << EOF > $tfile +DROP DATABASE IF EXISTS test; +USE mysql; +ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD'; +DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1'); +FLUSH PRIVILEGES; + +EOF + + if [ "$MYSQL_DATABASE" != "" ]; then + echo "[i] Creating database: $MYSQL_DATABASE" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile + if [ "$MYSQL_USER" != "" ]; then + echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile + fi + fi + + + cat $tfile + cat $tfile | mysql -v -u root + rm -v -f $tfile + + echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "password=${MYSQL_ROOT_PASSWORD}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + + for f in /docker-entrypoint-initdb.d/*; do + if [ -e "$f" ]; then + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.sql) echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;; + *) echo "$0: ignoring $f" ;; + esac + fi + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + + fi + + echo "done, now starting daemon" + touch /tmp/startup-init-complete + touch /tmp/mysql-init-complete + +fi \ No newline at end of file diff --git a/services/keycloak-db/password-entrypoint.bash b/services/keycloak-db/password-entrypoint.bash new file mode 100644 index 0000000000..2ada8a28b2 --- /dev/null +++ b/services/keycloak-db/password-entrypoint.bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ ${KEYCLOAK_DB_PASSWORD+x} ]; then + if [ "${LAGOON}" == "mysql" ]; then + export MYSQL_PASSWORD=${KEYCLOAK_DB_PASSWORD} + else + export MARIADB_PASSWORD=${KEYCLOAK_DB_PASSWORD} + fi +fi \ No newline at end of file diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile index 1aac568ab2..b86461eeb0 100644 --- a/services/keycloak/Dockerfile +++ b/services/keycloak/Dockerfile @@ -22,6 +22,8 @@ COPY --from=ubi-micro-build /mnt/rootfs / ARG LAGOON_VERSION ENV LAGOON_VERSION=$LAGOON_VERSION +ARG DATABASE_VENDOR +ENV DATABASE_VENDOR=$DATABASE_VENDOR USER root @@ -58,7 +60,7 @@ ENV TMPDIR=/tmp \ KEYCLOAK_ADMIN_PASSWORD=admin \ KEYCLOAK_REALM=lagoon \ KEYCLOAK_REALM_ROLES=admin \ - DB_VENDOR=mariadb \ + DB_VENDOR=${DATABASE_VENDOR} \ DB_ADDR=keycloak-db \ DB_USER=keycloak \ DB_PASSWORD=keycloak \ @@ -70,7 +72,7 @@ ENV TMPDIR=/tmp \ KEYCLOAK_AUTH_SERVER_CLIENT_SECRET=f605b150-7636-4447-abd3-70988786b330 \ KEYCLOAK_SERVICE_API_CLIENT_SECRET=d3724d52-34d1-4967-a802-4d178678564b \ KEYCLOAK_LAGOON_UI_OIDC_CLIENT_SECRET=20580a56-6fbc-11ef-9a5b-3b4da292aa54 \ - LAGOON_DB_VENDOR=mariadb \ + LAGOON_DB_VENDOR=${DATABASE_VENDOR} \ LAGOON_DB_DATABASE=infrastructure \ LAGOON_DB_USER=api \ LAGOON_DB_PASSWORD=api \ diff --git a/services/keycloak/entrypoints/default-keycloak-entrypoint.sh b/services/keycloak/entrypoints/default-keycloak-entrypoint.sh index 42562df1a7..ba2e12721f 100755 --- a/services/keycloak/entrypoints/default-keycloak-entrypoint.sh +++ b/services/keycloak/entrypoints/default-keycloak-entrypoint.sh @@ -2,7 +2,7 @@ export KC_DB=$DB_VENDOR export KC_DB_USERNAME=$DB_USER export KC_DB_PASSWORD=$DB_PASSWORD -export KC_DB_URL=jdbc:mariadb://$DB_ADDR:3306/$DB_DATABASE +export KC_DB_URL=jdbc:$DB_VENDOR://$DB_ADDR:3306/$DB_DATABASE export KC_PROXY=${KC_PROXY:-edge} export KC_HOSTNAME_STRICT=${KC_HOSTNAME_STRICT:-false}