From c334d97b0441857f986efc114f1994a2f6e2aad8 Mon Sep 17 00:00:00 2001
From: Toby Bellwood <toby.bellwood@amazee.io>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 01/12] chore: add mysql8 compatible migrations

---
 .github/workflows/test-db-migrations.yml      |  2 +-
 docker-compose.local-dev.yaml                 |  1 +
 .../migrations/20220908065119_initial_db.js   | 36 ++++++------
 .../20230801000000_organizations.js           |  2 +-
 .../20241231000000_mysql8_compatibility.js    | 57 +++++++++++++++++++
 .../default-keycloak-entrypoint.sh            |  2 +-
 6 files changed, 79 insertions(+), 21 deletions(-)
 create mode 100644 services/api/database/migrations/20241231000000_mysql8_compatibility.js

diff --git a/.github/workflows/test-db-migrations.yml b/.github/workflows/test-db-migrations.yml
index e38d94750c..04c8a4963e 100644
--- a/.github/workflows/test-db-migrations.yml
+++ b/.github/workflows/test-db-migrations.yml
@@ -47,7 +47,7 @@ jobs:
       -
         name: Show initial migration logs
         run: |
-          docker compose -p lagoon logs api-init
+          docker compose -p lagoon logs api-db-init
       -
         name: Initiate rollback
         run: |
diff --git a/docker-compose.local-dev.yaml b/docker-compose.local-dev.yaml
index f5e00b0ce5..53140cb57c 100644
--- a/docker-compose.local-dev.yaml
+++ b/docker-compose.local-dev.yaml
@@ -29,6 +29,7 @@ services:
   api:
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
     volumes:
+      - ./services/api/database:/app/services/api/database
       - ./services/api/src:/app/services/api/src
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist
diff --git a/services/api/database/migrations/20220908065119_initial_db.js b/services/api/database/migrations/20220908065119_initial_db.js
index 75051bc77e..412cc63e90 100644
--- a/services/api/database/migrations/20220908065119_initial_db.js
+++ b/services/api/database/migrations/20220908065119_initial_db.js
@@ -9,18 +9,18 @@
             .createTable('advanced_task_definition', function (table) {
                 table.increments('id').notNullable().primary();
                 table.string('name', 300).notNullable();
-                table.specificType('description', 'text').defaultTo('').notNullable();
+                table.specificType('description', 'text').notNullable();
                 table.string('image', 2000).defaultTo('');
                 table.string('service', 100);
                 table.string('type', 100).notNullable();
                 table.integer('environment');
                 table.integer('project');
-                table.string('group_name', 2000);
+                table.string('group_name', 300);
                 table.enu('permission', ['GUEST', 'DEVELOPER', 'MAINTAINER']).defaultTo('GUEST');
-                table.specificType('command', 'text').defaultTo('');
+                table.specificType('command', 'text');
                 table.string('confirmation_text', 2000);
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['name', 'environment', 'project', 'group_name'], {indexName: 'name', storageEngineIndexType: 'hash'});
             })
             .createTable('advanced_task_definition_argument', function (table) {
@@ -76,8 +76,8 @@
                 table.string('name', 100);
                 table.integer('project');
                 table.enu('deploy_type', ['branch', 'pullrequest', 'promote']).notNullable();
-                table.string('deploy_base_ref', 100);
-                table.string('deploy_head_ref', 100);
+                table.string('deploy_base_ref', 250);
+                table.string('deploy_head_ref', 250);
                 table.string('deploy_title', 300);
                 table.enu('environment_type', ['production', 'development']).notNullable();
                 table.integer('auto_idle', 1).notNullable().defaultTo(1);
@@ -89,7 +89,7 @@
                 table.string('openshift_project_pattern', 300);
                 table.timestamp('updated').notNullable().defaultTo(knex.fn.now());
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['project', 'name', 'deleted'], {indexName: 'project_name_deleted'});
             })
             .createTable('environment_backup', function (table) {
@@ -98,7 +98,7 @@
                 table.string('source', 300);
                 table.string('backup_id', 300).unique({indexName:'backup_id'});
                 table.timestamp('created').notNullable().defaultTo(knex.raw('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'));
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.index('environment', 'backup_environment');
             })
             .createTable('environment_fact', function (table) {
@@ -109,9 +109,9 @@
                 table.string('value', 300).notNullable();
                 table.enu('type', ['TEXT', 'URL', 'SEMVER']).defaultTo('TEXT');
                 table.string('source', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
+                table.specificType('description', 'text');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.specificType('category', 'text').defaultTo('');
+                table.specificType('category', 'text');
                 table.boolean('key_fact').notNullable().defaultTo(0);
                 table.unique(['environment', 'name', 'source'], {indexName: 'environment_fact'});
             })
@@ -126,17 +126,17 @@
                 table.integer('environment');
                 table.string('severity', 300).defaultTo('');
                 table.decimal('severity_score', 1, 1).defaultTo(0);
-                table.string('identifier', 300).notNullable();
-                table.string('lagoon_service', 300).defaultTo('');
+                table.string('identifier', 100).notNullable();
+                table.string('lagoon_service', 100).defaultTo('');
                 table.string('source', 300).defaultTo('');
                 table.string('associated_package', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
-                table.string('version', 300).defaultTo('');
+                table.specificType('description', 'text');
+                table.string('version', 100).defaultTo('');
                 table.string('fixed_version', 300).defaultTo('');
                 table.string('links', 300).defaultTo('');
                 table.json('data');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['environment', 'lagoon_service', 'version', 'identifier', 'deleted'], {indexName: 'environment'});
             })
             .createTable('environment_service', function (table) {
@@ -235,7 +235,7 @@
                 table.integer('development_environments_limit');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
                 table.string('private_key', 5000);
-                table.json('metadata').defaultTo('{}');
+                table.json('metadata');
                 table.string('router_pattern', 300).defaultTo(null);
             })
             .createTable('project_notification', function (table) {
@@ -251,7 +251,7 @@
                 table.string('filename', 100).notNullable();
                 table.text('s3_key').notNullable();
                 table.datetime('created').notNullable().defaultTo(knex.fn.now());
-                table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.datetime('deleted').notNullable();
             })
             .createTable('ssh_key', function (table) {
                table.increments('id').notNullable().primary();
@@ -295,7 +295,7 @@
                 table.integer('project', 11).notNullable();
                 table.integer('advanced_task_definition', 11).notNullable();
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
             })
         }
         else {
diff --git a/services/api/database/migrations/20230801000000_organizations.js b/services/api/database/migrations/20230801000000_organizations.js
index 8c63375555..be8a466ad6 100644
--- a/services/api/database/migrations/20230801000000_organizations.js
+++ b/services/api/database/migrations/20230801000000_organizations.js
@@ -10,7 +10,7 @@ exports.up = async function(knex) {
             table.increments('id').notNullable().primary();
             table.string('name', 300).unique({indexName: 'name'});
             table.string('friendly_name', 300).notNullable();
-            table.specificType('description', 'text').defaultTo('').notNullable();
+            table.specificType('description', 'text').notNullable();
             table.integer('quota_project').defaultTo(1).notNullable();
             table.integer('quota_group').defaultTo(10).notNullable();
             table.integer('quota_notification').defaultTo(10).notNullable();
diff --git a/services/api/database/migrations/20241231000000_mysql8_compatibility.js b/services/api/database/migrations/20241231000000_mysql8_compatibility.js
new file mode 100644
index 0000000000..a5d71954c0
--- /dev/null
+++ b/services/api/database/migrations/20241231000000_mysql8_compatibility.js
@@ -0,0 +1,57 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema
+    .alterTable('advanced_task_definition', function (table) {
+        table.specificType('description', 'text').notNullable().alter();
+        table.string('group_name', 300).alter();
+        table.specificType('command', 'text').alter();
+    })
+    .alterTable('environment_fact', function (table) {
+        table.specificType('description', 'text').alter();
+        table.specificType('category', 'text').alter();
+    })
+    .alterTable('environment_problem', function (table) {
+        table.string('identifier', 100).notNullable().alter();
+        table.string('lagoon_service', 100).defaultTo('').alter();
+        table.specificType('description', 'text').alter();
+        table.string('version', 100).defaultTo('').alter();
+    })
+    .alterTable('organization', function (table) {
+        table.specificType('description', 'text').notNullable().alter();
+    })
+    .alterTable('project', function (table) {
+        table.json('metadata').alter();
+    })
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    // caveats around this are that the rollback can only work while data is still saved in keycloak attributes
+    // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations
+    // for any group project associations made after the attribute removal
+    return knex.schema
+    .alterTable('advanced_task_definition', function (table) {
+        table.specificType('description', 'text').defaultTo('').notNullable().alter();
+        table.string('group_name', 300).alter();
+        table.specificType('command', 'text').defaultTo('').alter();
+    })
+    .alterTable('environment_fact', function (table) {
+        table.specificType('description', 'text').defaultTo('').alter();
+        table.specificType('category', 'text').defaultTo('').alter();
+    })
+    .alterTable('environment_problem', function (table) {
+        table.string('identifier', 100).notNullable().alter();
+        table.string('lagoon_service', 100).defaultTo('').alter();
+        table.specificType('description', 'text').defaultTo('').alter();
+        table.string('version', 100).defaultTo('').alter();
+    })
+    .alterTable('organization', function (table) {
+        table.specificType('description', 'text').notNullable().defaultTo('').alter();
+    })
+};
diff --git a/services/keycloak/entrypoints/default-keycloak-entrypoint.sh b/services/keycloak/entrypoints/default-keycloak-entrypoint.sh
index 42562df1a7..ba2e12721f 100755
--- a/services/keycloak/entrypoints/default-keycloak-entrypoint.sh
+++ b/services/keycloak/entrypoints/default-keycloak-entrypoint.sh
@@ -2,7 +2,7 @@
 export KC_DB=$DB_VENDOR
 export KC_DB_USERNAME=$DB_USER
 export KC_DB_PASSWORD=$DB_PASSWORD
-export KC_DB_URL=jdbc:mariadb://$DB_ADDR:3306/$DB_DATABASE
+export KC_DB_URL=jdbc:$DB_VENDOR://$DB_ADDR:3306/$DB_DATABASE
 
 export KC_PROXY=${KC_PROXY:-edge}
 export KC_HOSTNAME_STRICT=${KC_HOSTNAME_STRICT:-false}

From 3d14608b0ba90d2bbe5f6317dd37f001fe7d4dce Mon Sep 17 00:00:00 2001
From: Toby Bellwood <toby.bellwood@amazee.io>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 02/12] chore: add mysql8 to docker compose for testing

---
 docker-compose.yaml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 8b0ac7b293..16c1a4190f 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -5,9 +5,14 @@ volumes:
 
 services:
   api-db:
-    image: ${IMAGE_REPO:-lagoon}/api-db:${IMAGE_REPO_API_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
+    image: uselagoon/mysql-8.0:latest
     ports:
       - '3366:3306'
+    command: mysqld --sql_mode=""
+    environment:
+      - MYSQL_USER=api
+      - MYSQL_PASSWORD=api
+      - MYSQL_DATABASE=infrastructure
   webhook-handler:
     image: ${IMAGE_REPO:-lagoon}/webhook-handler:${IMAGE_REPO_WEBHOOK_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
@@ -158,13 +163,19 @@ services:
     environment:
       - KEYCLOAK_FRONTEND_URL=http://localhost:8088/auth/
       - KEYCLOAK_ADMIN_EMAIL=admin@example.com
+      - DB_VENDOR=mysql
+      - LAGOON_DB_VENDOR=mysql
       # Uncomment for local new relic tracking
       # - NEW_RELIC_LICENSE_KEY=
       # - NEW_RELIC_APP_NAME=keycloak-local
   keycloak-db:
-    image: ${IMAGE_REPO:-lagoon}/keycloak-db:${IMAGE_REPO_KEYCLOAK_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
+    image: uselagoon/mysql-8.0:latest
     ports:
       - '3336:3306'
+    environment:
+      - MYSQL_USER=keycloak
+      - MYSQL_PASSWORD=keycloak
+      - MYSQL_DATABASE=keycloak
   tests-kubernetes:
     image: ${IMAGE_REPO:-lagoon}/tests:${IMAGE_REPO_TESTS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:

From 38b140e79c86b4ac903b5d5f174693fa6bfc1298 Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 03/12] chore: adjust migrations for more fields

---
 .../20241231000000_mysql8_compatibility.js    | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/services/api/database/migrations/20241231000000_mysql8_compatibility.js b/services/api/database/migrations/20241231000000_mysql8_compatibility.js
index a5d71954c0..1c054cf778 100644
--- a/services/api/database/migrations/20241231000000_mysql8_compatibility.js
+++ b/services/api/database/migrations/20241231000000_mysql8_compatibility.js
@@ -8,6 +8,10 @@ exports.up = async function(knex) {
         table.specificType('description', 'text').notNullable().alter();
         table.string('group_name', 300).alter();
         table.specificType('command', 'text').alter();
+        table.timestamp('deleted').notNullable().alter();
+    })
+    .alterTable('environment', function (table) {
+        table.timestamp('deleted').notNullable().alter();
     })
     .alterTable('environment_fact', function (table) {
         table.specificType('description', 'text').alter();
@@ -18,12 +22,17 @@ exports.up = async function(knex) {
         table.string('lagoon_service', 100).defaultTo('').alter();
         table.specificType('description', 'text').alter();
         table.string('version', 100).defaultTo('').alter();
+        table.timestamp('deleted').notNullable().alter();
     })
     .alterTable('organization', function (table) {
         table.specificType('description', 'text').notNullable().alter();
     })
-    .alterTable('project', function (table) {
-        table.json('metadata').alter();
+    .alterTable('s3_file', function (table) {
+        table.timestamp('created').notNullable().defaultTo(knex.fn.now()).alter();
+        table.timestamp('deleted').notNullable().alter();
+    })
+    .alterTable('workflow', function (table) {
+        table.timestamp('deleted').notNullable().alter();
     })
 };
 
@@ -51,6 +60,13 @@ exports.down = async function(knex) {
         table.specificType('description', 'text').defaultTo('').alter();
         table.string('version', 100).defaultTo('').alter();
     })
+    .alterTable('project', function (table) {
+        table.json('metadata').defaultTo('{}').alter();
+    })
+    .alterTable('s3_file', function (table) {
+        table.datetime('created').notNullable().defaultTo(knex.fn.now()).alter();
+        table.datetime('deleted').notNullable().alter();
+    })
     .alterTable('organization', function (table) {
         table.specificType('description', 'text').notNullable().defaultTo('').alter();
     })

From 1483f8db1cdabd9d4e2c24abf46a045587d7afed Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 04/12] chore: use separate make targets for mysql with
 docker-compose override file

---
 Makefile                            | 22 ++++++++++++++++++++++
 docker-compose.local-dev-mysql.yaml | 23 +++++++++++++++++++++++
 docker-compose.yaml                 | 15 ++-------------
 3 files changed, 47 insertions(+), 13 deletions(-)
 create mode 100644 docker-compose.local-dev-mysql.yaml

diff --git a/Makefile b/Makefile
index a19f43d704..284380f2c3 100644
--- a/Makefile
+++ b/Makefile
@@ -352,6 +352,19 @@ else
 endif
 	$(MAKE) wait-for-keycloak
 
+# Start all Lagoon Services with mysql instead of mariadb
+up-mysql:
+ifeq ($(ARCH), darwin)
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
+else
+	# once this docker issue is fixed we may be able to do away with this
+	# linux-specific workaround: https://github.com/docker/cli/issues/2290
+	KEYCLOAK_URL=$$(docker network inspect -f '{{(index .IPAM.Config 0).Gateway}}' bridge):8088 \
+		IMAGE_REPO=$(CI_BUILD_TAG) \
+		docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
+endif
+	$(MAKE) wait-for-keycloak
+
 down:
 	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans
 
@@ -392,6 +405,15 @@ ui-logs-development: build-ui-logs-development
 api-logs-development: build-ui-logs-development
 	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
 
+# MySQL specific targets for local dev
+.PHONY: ui-logs-development-mysql
+ui-logs-development-mysql: build-ui-logs-development
+	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="ui" $(MAKE) compose-api-logs-development
+
+.PHONY: api-logs-development-mysql
+api-logs-development-mysql: build-ui-logs-development
+	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
+
 # compose-api-logs-development can be consumed by other repositories to start a local api
 # supported make variable passthrough are
 # IMAGE_REPO - the docker repostory to use (uselagoon, testlagoon, other)
diff --git a/docker-compose.local-dev-mysql.yaml b/docker-compose.local-dev-mysql.yaml
new file mode 100644
index 0000000000..d60c44cedc
--- /dev/null
+++ b/docker-compose.local-dev-mysql.yaml
@@ -0,0 +1,23 @@
+version: '3.2'
+
+volumes:
+  broker: {}
+
+services:
+  api-db:
+    image: uselagoon/mysql-8.0:latest
+    command: mysqld --sql_mode=""
+    environment:
+      - MYSQL_USER=api
+      - MYSQL_PASSWORD=api
+      - MYSQL_DATABASE=infrastructure
+  keycloak:
+    environment:
+      - DB_VENDOR=mysql
+      - LAGOON_DB_VENDOR=mysql
+  keycloak-db:
+    image: uselagoon/mysql-8.0:latest
+    environment:
+      - MYSQL_USER=keycloak
+      - MYSQL_PASSWORD=keycloak
+      - MYSQL_DATABASE=keycloak
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 16c1a4190f..8b0ac7b293 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -5,14 +5,9 @@ volumes:
 
 services:
   api-db:
-    image: uselagoon/mysql-8.0:latest
+    image: ${IMAGE_REPO:-lagoon}/api-db:${IMAGE_REPO_API_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '3366:3306'
-    command: mysqld --sql_mode=""
-    environment:
-      - MYSQL_USER=api
-      - MYSQL_PASSWORD=api
-      - MYSQL_DATABASE=infrastructure
   webhook-handler:
     image: ${IMAGE_REPO:-lagoon}/webhook-handler:${IMAGE_REPO_WEBHOOK_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
@@ -163,19 +158,13 @@ services:
     environment:
       - KEYCLOAK_FRONTEND_URL=http://localhost:8088/auth/
       - KEYCLOAK_ADMIN_EMAIL=admin@example.com
-      - DB_VENDOR=mysql
-      - LAGOON_DB_VENDOR=mysql
       # Uncomment for local new relic tracking
       # - NEW_RELIC_LICENSE_KEY=
       # - NEW_RELIC_APP_NAME=keycloak-local
   keycloak-db:
-    image: uselagoon/mysql-8.0:latest
+    image: ${IMAGE_REPO:-lagoon}/keycloak-db:${IMAGE_REPO_KEYCLOAK_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '3336:3306'
-    environment:
-      - MYSQL_USER=keycloak
-      - MYSQL_PASSWORD=keycloak
-      - MYSQL_DATABASE=keycloak
   tests-kubernetes:
     image: ${IMAGE_REPO:-lagoon}/tests:${IMAGE_REPO_TESTS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:

From 98ca4cab7f9db4e071fc12aa96379caf5b81592b Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 05/12] chore: fix stored enum for source_type for deployment
 and task

---
 .../20241231000000_source_type_enums.js       | 22 +++++++++++++++++++
 ...=> 20241231000100_mysql8_compatibility.js} |  0
 services/api/src/models/environment.ts        |  2 +-
 .../api/src/resources/deployment/resolvers.ts |  3 +++
 services/api/src/resources/task/helpers.ts    |  3 +++
 5 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 services/api/database/migrations/20241231000000_source_type_enums.js
 rename services/api/database/migrations/{20241231000000_mysql8_compatibility.js => 20241231000100_mysql8_compatibility.js} (100%)

diff --git a/services/api/database/migrations/20241231000000_source_type_enums.js b/services/api/database/migrations/20241231000000_source_type_enums.js
new file mode 100644
index 0000000000..ef2afea9bb
--- /dev/null
+++ b/services/api/database/migrations/20241231000000_source_type_enums.js
@@ -0,0 +1,22 @@
+exports.up = function(knex) {
+    // fix the way the enum values are stored to be lowercase for mysql strict
+    return Promise.all([
+        knex('deployment')
+            .where('source_type', '=', 'API')
+            .update('source_type', 'api'),
+        knex('deployment')
+            .where('source_type', '=', 'WEBHOOK')
+            .update('source_type', 'webhook'),
+        knex('task')
+            .where('source_type', '=', 'API')
+            .update('source_type', 'api'),
+    ]);
+  };
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function(knex) {
+return knex.schema;
+};
\ No newline at end of file
diff --git a/services/api/database/migrations/20241231000000_mysql8_compatibility.js b/services/api/database/migrations/20241231000100_mysql8_compatibility.js
similarity index 100%
rename from services/api/database/migrations/20241231000000_mysql8_compatibility.js
rename to services/api/database/migrations/20241231000100_mysql8_compatibility.js
diff --git a/services/api/src/models/environment.ts b/services/api/src/models/environment.ts
index ba2588e655..10de050f16 100644
--- a/services/api/src/models/environment.ts
+++ b/services/api/src/models/environment.ts
@@ -13,7 +13,7 @@ export interface Environment {
   openshiftProjectName?: string; // varchar(100) COLLATE utf8_bin DEFAULT NULL,
   updated?: string; // timestamp NOT NULL DEFAULT current_timestamp(),
   created?: string; //  timestamp NOT NULL DEFAULT current_timestamp(),
-  deleted?: string; // timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  deleted?: string;
   route?: string; // varchar(300) COLLATE utf8_bin DEFAULT NULL,
   routes?: string; // text COLLATE utf8_bin DEFAULT NULL,
   autoIdle?: Boolean; // int(1) NOT NULL DEFAULT 1,
diff --git a/services/api/src/resources/deployment/resolvers.ts b/services/api/src/resources/deployment/resolvers.ts
index 28908dbc9c..28d815518f 100644
--- a/services/api/src/resources/deployment/resolvers.ts
+++ b/services/api/src/resources/deployment/resolvers.ts
@@ -343,6 +343,9 @@ export const addDeployment: ResolverFn = async (
   if (!sourceType) {
     sourceType = "API"
   }
+  if (sourceType) {
+    sourceType.toLocaleLowerCase();
+  }
   const { insertId } = await query(
     sqlClientPool,
     Sql.insertDeployment({
diff --git a/services/api/src/resources/task/helpers.ts b/services/api/src/resources/task/helpers.ts
index db28a8c71e..62d8820efe 100644
--- a/services/api/src/resources/task/helpers.ts
+++ b/services/api/src/resources/task/helpers.ts
@@ -73,6 +73,9 @@ export const Helpers = (sqlClientPool: Pool, hasPermission, adminScopes) => {
       sourceUser: string;
       sourceType: string;
     }) => {
+      if (sourceType) {
+        sourceType.toLocaleLowerCase();
+      }
       const { insertId } = await query(
         sqlClientPool,
         Sql.insertTask({

From 5c8770ddc3fed527019ee4ece785d2f8aecbd50a Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 06/12] chore: update deploy base/head ref length

---
 .../database/migrations/20241231000100_mysql8_compatibility.js  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/services/api/database/migrations/20241231000100_mysql8_compatibility.js b/services/api/database/migrations/20241231000100_mysql8_compatibility.js
index 1c054cf778..ddf028ac5c 100644
--- a/services/api/database/migrations/20241231000100_mysql8_compatibility.js
+++ b/services/api/database/migrations/20241231000100_mysql8_compatibility.js
@@ -12,6 +12,8 @@ exports.up = async function(knex) {
     })
     .alterTable('environment', function (table) {
         table.timestamp('deleted').notNullable().alter();
+        table.string('deploy_base_ref', 250).alter();
+        table.string('deploy_head_ref', 250).alter();
     })
     .alterTable('environment_fact', function (table) {
         table.specificType('description', 'text').alter();

From 45c145c52ed431f81815b14a6fd667700226fc92 Mon Sep 17 00:00:00 2001
From: Toby Bellwood <toby.bellwood@amazee.io>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 07/12] chore: ensure migration down matches up

---
 .../20241231000100_mysql8_compatibility.js         | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/services/api/database/migrations/20241231000100_mysql8_compatibility.js b/services/api/database/migrations/20241231000100_mysql8_compatibility.js
index ddf028ac5c..da891923f4 100644
--- a/services/api/database/migrations/20241231000100_mysql8_compatibility.js
+++ b/services/api/database/migrations/20241231000100_mysql8_compatibility.js
@@ -49,25 +49,29 @@ exports.down = async function(knex) {
     return knex.schema
     .alterTable('advanced_task_definition', function (table) {
         table.specificType('description', 'text').defaultTo('').notNullable().alter();
-        table.string('group_name', 300).alter();
+        table.string('group_name', 2000).alter();
         table.specificType('command', 'text').defaultTo('').alter();
     })
+    .alterTable('environment', function (table) {
+        table.string('deploy_base_ref', 100).alter();
+        table.string('deploy_head_ref', 100).alter();
+    })
     .alterTable('environment_fact', function (table) {
         table.specificType('description', 'text').defaultTo('').alter();
         table.specificType('category', 'text').defaultTo('').alter();
     })
     .alterTable('environment_problem', function (table) {
-        table.string('identifier', 100).notNullable().alter();
-        table.string('lagoon_service', 100).defaultTo('').alter();
+        table.string('identifier', 300).notNullable().alter();
+        table.string('lagoon_service', 300).defaultTo('').alter();
         table.specificType('description', 'text').defaultTo('').alter();
-        table.string('version', 100).defaultTo('').alter();
+        table.string('version', 300).defaultTo('').alter();
     })
     .alterTable('project', function (table) {
         table.json('metadata').defaultTo('{}').alter();
     })
     .alterTable('s3_file', function (table) {
         table.datetime('created').notNullable().defaultTo(knex.fn.now()).alter();
-        table.datetime('deleted').notNullable().alter();
+        table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00').alter();
     })
     .alterTable('organization', function (table) {
         table.specificType('description', 'text').notNullable().defaultTo('').alter();

From 31e614d2646db695b901c87c181c08cc96c01077 Mon Sep 17 00:00:00 2001
From: Toby Bellwood <toby.bellwood@amazee.io>
Date: Thu, 5 Dec 2024 16:45:29 +1100
Subject: [PATCH 08/12] chore: rename migrations

---
 ...0_source_type_enums.js => 20241204000000_source_type_enums.js} | 0
 ...l8_compatibility.js => 20241204000100_mysql8_compatibility.js} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename services/api/database/migrations/{20241231000000_source_type_enums.js => 20241204000000_source_type_enums.js} (100%)
 rename services/api/database/migrations/{20241231000100_mysql8_compatibility.js => 20241204000100_mysql8_compatibility.js} (100%)

diff --git a/services/api/database/migrations/20241231000000_source_type_enums.js b/services/api/database/migrations/20241204000000_source_type_enums.js
similarity index 100%
rename from services/api/database/migrations/20241231000000_source_type_enums.js
rename to services/api/database/migrations/20241204000000_source_type_enums.js
diff --git a/services/api/database/migrations/20241231000100_mysql8_compatibility.js b/services/api/database/migrations/20241204000100_mysql8_compatibility.js
similarity index 100%
rename from services/api/database/migrations/20241231000100_mysql8_compatibility.js
rename to services/api/database/migrations/20241204000100_mysql8_compatibility.js

From d60014a95b18e71808970c823e911e4f64814514 Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Tue, 10 Dec 2024 11:30:06 +1100
Subject: [PATCH 09/12] refactor: use make for api/keycloak db overrides

---
 Makefile                              | 45 ++++++++++-----------------
 docker-bake.hcl                       | 11 +++++++
 docker-compose.local-dev-mysql.yaml   | 23 --------------
 services/api-db/Dockerfile.mysql      | 27 ++++++++++++++++
 services/keycloak-db/Dockerfile.mysql | 19 +++++++++++
 services/keycloak/Dockerfile          |  6 ++--
 6 files changed, 78 insertions(+), 53 deletions(-)
 delete mode 100644 docker-compose.local-dev-mysql.yaml
 create mode 100644 services/api-db/Dockerfile.mysql
 create mode 100644 services/keycloak-db/Dockerfile.mysql

diff --git a/Makefile b/Makefile
index 284380f2c3..4d518c3727 100644
--- a/Makefile
+++ b/Makefile
@@ -94,13 +94,21 @@ KUBECTL = $(realpath ./local-dev/kubectl)
 JQ = $(realpath ./local-dev/jq)
 K3D = $(realpath ./local-dev/k3d)
 
+# which database vendor type to use, can be mariadb (default) or mysql
+# DATABASE_VENDOR = mariadb
+DATABASE_VENDOR = mysql
+DATABASE_DOCKERFILE = Dockerfile
+ifeq ($(DATABASE_VENDOR), mysql)
+DATABASE_DOCKERFILE = Dockerfile.mysql
+endif
+
 #######
 ####### Functions
 #######
 
 # Builds a docker image. Expects as arguments: name of the image, location of Dockerfile, path of
 # Docker Build Context
-docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
+docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
 
 docker_buildx_create = 	docker buildx create --name $(CI_BUILD_TAG) || echo  -e '$(CI_BUILD_TAG) builder already present\n'
 
@@ -186,13 +194,13 @@ $(build-services):
 
 # Dependencies of Service Images
 build/auth-server build/webhook-handler build/webhooks2tasks build/api: build/yarn-workspace-builder
-build/api-db: services/api-db/Dockerfile
+build/api-db: services/api-db/$(DATABASE_DOCKERFILE)
 build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
 build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile
-build/keycloak-db: services/keycloak-db/Dockerfile
+build/keycloak-db: services/keycloak-db/$(DATABASE_DOCKERFILE)
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
 build/tests: tests/Dockerfile
@@ -305,14 +313,17 @@ webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test
 
 .PHONY: publish-testlagoon-images
 publish-testlagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 # tag and push all images
 
 .PHONY: publish-uselagoon-images
 publish-uselagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 .PHONY: clean
 clean:
@@ -352,19 +363,6 @@ else
 endif
 	$(MAKE) wait-for-keycloak
 
-# Start all Lagoon Services with mysql instead of mariadb
-up-mysql:
-ifeq ($(ARCH), darwin)
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
-else
-	# once this docker issue is fixed we may be able to do away with this
-	# linux-specific workaround: https://github.com/docker/cli/issues/2290
-	KEYCLOAK_URL=$$(docker network inspect -f '{{(index .IPAM.Config 0).Gateway}}' bridge):8088 \
-		IMAGE_REPO=$(CI_BUILD_TAG) \
-		docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
-endif
-	$(MAKE) wait-for-keycloak
-
 down:
 	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans
 
@@ -405,15 +403,6 @@ ui-logs-development: build-ui-logs-development
 api-logs-development: build-ui-logs-development
 	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
 
-# MySQL specific targets for local dev
-.PHONY: ui-logs-development-mysql
-ui-logs-development-mysql: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="ui" $(MAKE) compose-api-logs-development
-
-.PHONY: api-logs-development-mysql
-api-logs-development-mysql: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
-
 # compose-api-logs-development can be consumed by other repositories to start a local api
 # supported make variable passthrough are
 # IMAGE_REPO - the docker repostory to use (uselagoon, testlagoon, other)
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 30df12fd1e..996f9eb450 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -15,6 +15,14 @@ variable "UPSTREAM_REPO" {
   default = "uselagoon"
 }
 
+variable "DATABASE_DOCKERFILE" {
+  default = "Dockerfile"
+}
+
+variable "DATABASE_VENDOR" {
+  default = "mariadb"
+}
+
 variable "UPSTREAM_TAG" {
   default = "latest"
 }
@@ -39,6 +47,7 @@ target "default"{
     LAGOON_VERSION = "${LAGOON_VERSION}"
     UPSTREAM_REPO = "${UPSTREAM_REPO}"
     UPSTREAM_TAG = "${UPSTREAM_TAG}"
+    DATABASE_VENDOR = "${DATABASE_VENDOR}"
   }
 }
 
@@ -131,6 +140,7 @@ target "api" {
 target "api-db" {
   inherits = ["default"]
   context = "services/api-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/api-db - the MariaDB database service for Lagoon API"
   }
@@ -206,6 +216,7 @@ target "keycloak" {
 target "keycloak-db" {
   inherits = ["default"]
   context = "services/keycloak-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/keycloak-db - the MariaDB database service for Lagoon Keycloak"
   }
diff --git a/docker-compose.local-dev-mysql.yaml b/docker-compose.local-dev-mysql.yaml
deleted file mode 100644
index d60c44cedc..0000000000
--- a/docker-compose.local-dev-mysql.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-version: '3.2'
-
-volumes:
-  broker: {}
-
-services:
-  api-db:
-    image: uselagoon/mysql-8.0:latest
-    command: mysqld --sql_mode=""
-    environment:
-      - MYSQL_USER=api
-      - MYSQL_PASSWORD=api
-      - MYSQL_DATABASE=infrastructure
-  keycloak:
-    environment:
-      - DB_VENDOR=mysql
-      - LAGOON_DB_VENDOR=mysql
-  keycloak-db:
-    image: uselagoon/mysql-8.0:latest
-    environment:
-      - MYSQL_USER=keycloak
-      - MYSQL_PASSWORD=keycloak
-      - MYSQL_DATABASE=keycloak
\ No newline at end of file
diff --git a/services/api-db/Dockerfile.mysql b/services/api-db/Dockerfile.mysql
new file mode 100644
index 0000000000..bfd1dbe973
--- /dev/null
+++ b/services/api-db/Dockerfile.mysql
@@ -0,0 +1,27 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+USER root
+
+COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
+RUN chown -R mysql /legacy-migration-scripts/ \
+    && /bin/fix-permissions /legacy-migration-scripts/
+
+USER mysql
+
+ENV MYSQL_DATABASE=infrastructure \
+    MYSQL_USER=api \
+    MYSQL_PASSWORD=api
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# do we need to keep these any more? v2.10.0 was long ago...
+# COPY ./rerun_initdb.sh /rerun_initdb.sh
+# COPY ./legacy_rerun_initdb.sh /legacy_rerun_initdb.sh
+
+CMD ["mysqld", "--sql_mode", ""]
\ No newline at end of file
diff --git a/services/keycloak-db/Dockerfile.mysql b/services/keycloak-db/Dockerfile.mysql
new file mode 100644
index 0000000000..53e98010c5
--- /dev/null
+++ b/services/keycloak-db/Dockerfile.mysql
@@ -0,0 +1,19 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+ENV MYSQL_DATABASE=keycloak \
+    MYSQL_USER=keycloak \
+    MYSQL_PASSWORD=keycloak
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# not used in mysql8
+# COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
+# USER root
+# RUN fix-permissions /etc/mysql/conf.d/
+# USER mysql
diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile
index 1aac568ab2..b86461eeb0 100644
--- a/services/keycloak/Dockerfile
+++ b/services/keycloak/Dockerfile
@@ -22,6 +22,8 @@ COPY --from=ubi-micro-build /mnt/rootfs /
 
 ARG LAGOON_VERSION
 ENV LAGOON_VERSION=$LAGOON_VERSION
+ARG DATABASE_VENDOR
+ENV DATABASE_VENDOR=$DATABASE_VENDOR
 
 USER root
 
@@ -58,7 +60,7 @@ ENV TMPDIR=/tmp \
     KEYCLOAK_ADMIN_PASSWORD=admin \
     KEYCLOAK_REALM=lagoon \
     KEYCLOAK_REALM_ROLES=admin \
-    DB_VENDOR=mariadb \
+    DB_VENDOR=${DATABASE_VENDOR} \
     DB_ADDR=keycloak-db \
     DB_USER=keycloak \
     DB_PASSWORD=keycloak \
@@ -70,7 +72,7 @@ ENV TMPDIR=/tmp \
     KEYCLOAK_AUTH_SERVER_CLIENT_SECRET=f605b150-7636-4447-abd3-70988786b330 \
     KEYCLOAK_SERVICE_API_CLIENT_SECRET=d3724d52-34d1-4967-a802-4d178678564b \
     KEYCLOAK_LAGOON_UI_OIDC_CLIENT_SECRET=20580a56-6fbc-11ef-9a5b-3b4da292aa54 \
-    LAGOON_DB_VENDOR=mariadb \
+    LAGOON_DB_VENDOR=${DATABASE_VENDOR} \
     LAGOON_DB_DATABASE=infrastructure \
     LAGOON_DB_USER=api \
     LAGOON_DB_PASSWORD=api \

From e99f499b3f866c61445ec5d67aa432e395f19101 Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Tue, 10 Dec 2024 14:27:58 +1100
Subject: [PATCH 10/12] chore: adjust image startup scripts, set chart branch

---
 Makefile                              |   6 +-
 services/api-db/Dockerfile.mysql      |   2 +
 services/api-db/mysql-init.bash       | 156 ++++++++++++++++++++++++++
 services/keycloak-db/Dockerfile.mysql |   4 +
 services/keycloak-db/mysql-init.bash  | 156 ++++++++++++++++++++++++++
 5 files changed, 323 insertions(+), 1 deletion(-)
 create mode 100644 services/api-db/mysql-init.bash
 create mode 100644 services/keycloak-db/mysql-init.bash

diff --git a/Makefile b/Makefile
index 4d518c3727..eafac67161 100644
--- a/Makefile
+++ b/Makefile
@@ -424,7 +424,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.11.0
 K3D_IMAGE = docker.io/rancher/k3s:v1.31.0-k3s1
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = main
+CHARTS_TREEISH = mysql-image-support
 TASK_IMAGES = task-activestandby
 
 # the name of the docker network to create
@@ -580,6 +580,7 @@ k3d/test: k3d/setup
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_ALL_DEPS=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
@@ -614,6 +615,7 @@ k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_INSTALL_REGISTRY=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
@@ -673,6 +675,7 @@ k3d/dev: build
 		&& $(MAKE) install-lagoon-core DOCKER_NETWORK=$(DOCKER_NETWORK) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \
 			HELM=$(HELM) KUBECTL=$(KUBECTL) \
 			JQ=$(JQ) \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \
 			$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \
 			$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \
@@ -797,6 +800,7 @@ k3d/retest:
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_ALL_DEPS=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
diff --git a/services/api-db/Dockerfile.mysql b/services/api-db/Dockerfile.mysql
index bfd1dbe973..372e9ea41c 100644
--- a/services/api-db/Dockerfile.mysql
+++ b/services/api-db/Dockerfile.mysql
@@ -11,6 +11,8 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
+
 USER mysql
 
 ENV MYSQL_DATABASE=infrastructure \
diff --git a/services/api-db/mysql-init.bash b/services/api-db/mysql-init.bash
new file mode 100644
index 0000000000..38ae24a854
--- /dev/null
+++ b/services/api-db/mysql-init.bash
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ "$(ls -A /etc/mysql/conf.d/)" ]; then
+   ep /etc/mysql/conf.d/*
+fi
+
+if [ "${1:0:1}" = '-' ]; then
+  set -- mysqld "$@"
+fi
+
+wantHelp=
+for arg; do
+  case "$arg" in
+    -'?'|--help|--print-defaults|-V|--version)
+      wantHelp=1
+      break
+      ;;
+  esac
+done
+
+# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder
+# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container).
+# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting
+# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system
+# the data will be prefilled and persistent across container restarts.
+if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying"
+  else
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'"
+    CUR_DIR=${PWD}
+    cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -)
+    cd $CUR_DIR
+  fi
+fi
+
+ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf
+
+if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
+  if [ ! -d "/run/mysqld" ]; then
+    mkdir -p /var/run/mysqld
+    chown -R mysql:mysql /var/run/mysqld
+  fi
+
+  MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30}
+  MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1}
+
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MySQL directory already present, skipping creation"
+
+    echo "starting mysql for mysql upgrade."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+  else
+    echo "MySQL data directory not found, creating initial DBs"
+
+    /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr
+
+    echo "starting mysql for initdb.d import."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then
+      MYSQL_ROOT_PASSWORD=`pwgen 16 1`
+      echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD"
+    fi
+
+    MYSQL_DATABASE=${MYSQL_DATABASE:-""}
+    MYSQL_USER=${MYSQL_USER:-""}
+    MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}
+
+    tfile=`mktemp`
+    if [ ! -f "$tfile" ]; then
+        return 1
+    fi
+
+    cat << EOF > $tfile
+DROP DATABASE IF EXISTS test;
+USE mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
+DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
+FLUSH PRIVILEGES;
+
+EOF
+
+    if [ "$MYSQL_DATABASE" != "" ]; then
+      echo "[i] Creating database: $MYSQL_DATABASE"
+      echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile
+      if [ "$MYSQL_USER" != "" ]; then
+        echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD"
+        echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
+        echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile
+      fi
+    fi
+
+
+    cat $tfile
+    cat $tfile | mysql -v -u root
+    rm -v -f $tfile
+
+    echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "password=${MYSQL_ROOT_PASSWORD}"  >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+
+    for f in /docker-entrypoint-initdb.d/*; do
+      if [ -e "$f" ]; then
+        case "$f" in
+          *.sh)     echo "$0: running $f"; . "$f" ;;
+          *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;;
+          *)        echo "$0: ignoring $f" ;;
+        esac
+      fi
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+
+  fi
+
+  echo "done, now starting daemon"
+  touch /tmp/startup-init-complete
+  touch /tmp/mysql-init-complete
+
+fi
\ No newline at end of file
diff --git a/services/keycloak-db/Dockerfile.mysql b/services/keycloak-db/Dockerfile.mysql
index 53e98010c5..5a51cd3b08 100644
--- a/services/keycloak-db/Dockerfile.mysql
+++ b/services/keycloak-db/Dockerfile.mysql
@@ -12,6 +12,10 @@ ENV MYSQL_DATABASE=keycloak \
     # MYSQL_CHARSET=utf8 \
     # MYSQL_COLLATION=utf8_general_ci
 
+USER root
+COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
+USER mysql
+
 # not used in mysql8
 # COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
 # USER root
diff --git a/services/keycloak-db/mysql-init.bash b/services/keycloak-db/mysql-init.bash
new file mode 100644
index 0000000000..38ae24a854
--- /dev/null
+++ b/services/keycloak-db/mysql-init.bash
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ "$(ls -A /etc/mysql/conf.d/)" ]; then
+   ep /etc/mysql/conf.d/*
+fi
+
+if [ "${1:0:1}" = '-' ]; then
+  set -- mysqld "$@"
+fi
+
+wantHelp=
+for arg; do
+  case "$arg" in
+    -'?'|--help|--print-defaults|-V|--version)
+      wantHelp=1
+      break
+      ;;
+  esac
+done
+
+# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder
+# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container).
+# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting
+# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system
+# the data will be prefilled and persistent across container restarts.
+if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying"
+  else
+    echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'"
+    CUR_DIR=${PWD}
+    cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -)
+    cd $CUR_DIR
+  fi
+fi
+
+ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf
+
+if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
+  if [ ! -d "/run/mysqld" ]; then
+    mkdir -p /var/run/mysqld
+    chown -R mysql:mysql /var/run/mysqld
+  fi
+
+  MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30}
+  MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1}
+
+  if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
+    echo "MySQL directory already present, skipping creation"
+
+    echo "starting mysql for mysql upgrade."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+  else
+    echo "MySQL data directory not found, creating initial DBs"
+
+    /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr
+
+    echo "starting mysql for initdb.d import."
+    /usr/sbin/mysqld --skip-networking &
+    pid="$!"
+    echo "pid is $pid"
+
+    for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
+      if echo 'SELECT 1' | mysql -u root; then
+        break
+      fi
+      echo 'MySQL init process in progress...'
+      sleep $MYSQL_INIT_PERIOD_SECONDS
+    done
+
+    if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then
+      MYSQL_ROOT_PASSWORD=`pwgen 16 1`
+      echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD"
+    fi
+
+    MYSQL_DATABASE=${MYSQL_DATABASE:-""}
+    MYSQL_USER=${MYSQL_USER:-""}
+    MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}
+
+    tfile=`mktemp`
+    if [ ! -f "$tfile" ]; then
+        return 1
+    fi
+
+    cat << EOF > $tfile
+DROP DATABASE IF EXISTS test;
+USE mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
+DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
+FLUSH PRIVILEGES;
+
+EOF
+
+    if [ "$MYSQL_DATABASE" != "" ]; then
+      echo "[i] Creating database: $MYSQL_DATABASE"
+      echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile
+      if [ "$MYSQL_USER" != "" ]; then
+        echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD"
+        echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
+        echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile
+      fi
+    fi
+
+
+    cat $tfile
+    cat $tfile | mysql -v -u root
+    rm -v -f $tfile
+
+    echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "password=${MYSQL_ROOT_PASSWORD}"  >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+    echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
+
+    for f in /docker-entrypoint-initdb.d/*; do
+      if [ -e "$f" ]; then
+        case "$f" in
+          *.sh)     echo "$0: running $f"; . "$f" ;;
+          *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;;
+          *)        echo "$0: ignoring $f" ;;
+        esac
+      fi
+    done
+
+    if ! kill -s TERM "$pid" || ! wait "$pid"; then
+      echo >&2 'MySQL init process failed.'
+      exit 1
+    fi
+
+  fi
+
+  echo "done, now starting daemon"
+  touch /tmp/startup-init-complete
+  touch /tmp/mysql-init-complete
+
+fi
\ No newline at end of file

From f4167af8df2de7d925fab0a7c27e1b9370f642ff Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Tue, 10 Dec 2024 14:33:49 +1100
Subject: [PATCH 11/12] chore: fix mariadb entrypoint init filename

---
 services/api-db/Dockerfile      | 2 ++
 services/keycloak-db/Dockerfile | 1 +
 2 files changed, 3 insertions(+)

diff --git a/services/api-db/Dockerfile b/services/api-db/Dockerfile
index 93098a7ade..9f9a4e0788 100644
--- a/services/api-db/Dockerfile
+++ b/services/api-db/Dockerfile
@@ -12,6 +12,8 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash
+
 USER mysql
 
 ENV MARIADB_DATABASE=infrastructure \
diff --git a/services/keycloak-db/Dockerfile b/services/keycloak-db/Dockerfile
index 6f26eb61a3..da3035388a 100644
--- a/services/keycloak-db/Dockerfile
+++ b/services/keycloak-db/Dockerfile
@@ -13,5 +13,6 @@ ENV MARIADB_DATABASE=keycloak \
 
 COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
 USER root
+RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash
 RUN fix-permissions /etc/mysql/conf.d/
 USER mysql

From 17c7afc51db15b3ebfc5f01a14a4a40f4ab1c899 Mon Sep 17 00:00:00 2001
From: shreddedbacon <b@benjackson.email>
Date: Tue, 10 Dec 2024 15:20:01 +1100
Subject: [PATCH 12/12] chore: add password entrypoint to keycloak and api db

---
 services/api-db/Dockerfile                    |  2 ++
 services/api-db/Dockerfile.mysql              |  2 ++
 services/api-db/password-entrypoint.bash      | 11 +++++++++++
 services/keycloak-db/Dockerfile               |  2 ++
 services/keycloak-db/Dockerfile.mysql         |  2 ++
 services/keycloak-db/password-entrypoint.bash | 11 +++++++++++
 6 files changed, 30 insertions(+)
 create mode 100644 services/api-db/password-entrypoint.bash
 create mode 100644 services/keycloak-db/password-entrypoint.bash

diff --git a/services/api-db/Dockerfile b/services/api-db/Dockerfile
index 9f9a4e0788..57ecf32b8f 100644
--- a/services/api-db/Dockerfile
+++ b/services/api-db/Dockerfile
@@ -12,6 +12,8 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
 RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash
 
 USER mysql
diff --git a/services/api-db/Dockerfile.mysql b/services/api-db/Dockerfile.mysql
index 372e9ea41c..e77db81364 100644
--- a/services/api-db/Dockerfile.mysql
+++ b/services/api-db/Dockerfile.mysql
@@ -11,6 +11,8 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
 COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
 
 USER mysql
diff --git a/services/api-db/password-entrypoint.bash b/services/api-db/password-entrypoint.bash
new file mode 100644
index 0000000000..1064d2c36f
--- /dev/null
+++ b/services/api-db/password-entrypoint.bash
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ ${API_DB_PASSWORD+x} ]; then
+    if [ "${LAGOON}" == "mysql" ]; then
+        export MYSQL_PASSWORD=${API_DB_PASSWORD}
+    else
+        export MARIADB_PASSWORD=${API_DB_PASSWORD}
+    fi
+fi
\ No newline at end of file
diff --git a/services/keycloak-db/Dockerfile b/services/keycloak-db/Dockerfile
index da3035388a..81bb03972c 100644
--- a/services/keycloak-db/Dockerfile
+++ b/services/keycloak-db/Dockerfile
@@ -13,6 +13,8 @@ ENV MARIADB_DATABASE=keycloak \
 
 COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
 USER root
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
 RUN sed -i 's/mariadb-init-complete/startup-init-complete/g' /lagoon/entrypoints/9999-mariadb-init.bash
 RUN fix-permissions /etc/mysql/conf.d/
 USER mysql
diff --git a/services/keycloak-db/Dockerfile.mysql b/services/keycloak-db/Dockerfile.mysql
index 5a51cd3b08..c941c0439f 100644
--- a/services/keycloak-db/Dockerfile.mysql
+++ b/services/keycloak-db/Dockerfile.mysql
@@ -13,6 +13,8 @@ ENV MYSQL_DATABASE=keycloak \
     # MYSQL_COLLATION=utf8_general_ci
 
 USER root
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
 COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
 USER mysql
 
diff --git a/services/keycloak-db/password-entrypoint.bash b/services/keycloak-db/password-entrypoint.bash
new file mode 100644
index 0000000000..2ada8a28b2
--- /dev/null
+++ b/services/keycloak-db/password-entrypoint.bash
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ ${KEYCLOAK_DB_PASSWORD+x} ]; then
+    if [ "${LAGOON}" == "mysql" ]; then
+        export MYSQL_PASSWORD=${KEYCLOAK_DB_PASSWORD}
+    else
+        export MARIADB_PASSWORD=${KEYCLOAK_DB_PASSWORD}
+    fi
+fi
\ No newline at end of file