From d0c834344fd3963ff124b7cdd861c7baf50a9933 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 22 Dec 2023 12:24:16 +1300 Subject: [PATCH 1/8] Adds code to clean up insights on deleting environments --- ...20231222000000_delete_orphaned_insights.js | 20 +++++++++++++++++++ .../api/src/resources/environment/helpers.ts | 15 ++++++++++++++ services/api/src/resources/fact/sql.ts | 7 +++++++ services/api/src/resources/problem/sql.ts | 7 +++++++ 4 files changed, 49 insertions(+) create mode 100644 services/api/database/migrations/20231222000000_delete_orphaned_insights.js diff --git a/services/api/database/migrations/20231222000000_delete_orphaned_insights.js b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js new file mode 100644 index 0000000000..fa4c2860e5 --- /dev/null +++ b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js @@ -0,0 +1,20 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = function(knex) { + return knex.schema.raw(` + DELETE ef + FROM environment_fact ef + LEFT JOIN environment e ON ef.environment = e.id + WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' + `); + }; + + /** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ + exports.down = function(knex) { + return knex.schema; + }; diff --git a/services/api/src/resources/environment/helpers.ts b/services/api/src/resources/environment/helpers.ts index 3f1d6d65e3..a55f2bc38c 100644 --- a/services/api/src/resources/environment/helpers.ts +++ b/services/api/src/resources/environment/helpers.ts @@ -3,6 +3,8 @@ import { Pool } from 'mariadb'; import { asyncPipe } from '@lagoon/commons/dist/util/func'; import { query } from '../../util/db'; import { Sql } from './sql'; +import { Sql as problemSql } from '../problem/sql'; +import { Sql as factSql } from '../fact/sql'; import { Helpers as projectHelpers } from '../project/helpers'; // import { logger } from '../../loggers/logger'; @@ -48,6 +50,19 @@ export const Helpers = (sqlClientPool: Pool) => { sqlClientPool, Sql.deleteEnvironment(name, pid) ); + + // Here we clean up insights attached to the environment + + await query( + sqlClientPool, + factSql.deleteFactsForEnvironment(eid) + ); + + await query( + sqlClientPool, + problemSql.deleteProblemsForEnvironment(eid) + ); + }, getEnvironmentsDeploytarget: async (eid) => { const rows = await query( diff --git a/services/api/src/resources/fact/sql.ts b/services/api/src/resources/fact/sql.ts index 1908719edb..5b7b4b052d 100644 --- a/services/api/src/resources/fact/sql.ts +++ b/services/api/src/resources/fact/sql.ts @@ -48,6 +48,13 @@ export const Sql = { }) .del() .toString(), + deleteFactsForEnvironment: (environment) => // Used when removing environments. + knex('environment_fact') + .where({ + environment + }) + .del() + .toString(), deleteFactsFromSource: (environment, source) => knex('environment_fact') .where({ environment, source }) diff --git a/services/api/src/resources/problem/sql.ts b/services/api/src/resources/problem/sql.ts index afa9afd34f..0607b7705a 100644 --- a/services/api/src/resources/problem/sql.ts +++ b/services/api/src/resources/problem/sql.ts @@ -139,6 +139,13 @@ export const Sql = { } return q.del().toString(); }, + deleteProblemsForEnvironment: (environment) => { // This should be used primarily to remove problems when deleting an environment + let q = knex('environment_problem') + .where({ + environment: environment + }); + return q.del().toString(); + }, deleteProblemsFromSource: (environment, source, service) => knex('environment_problem') .where({ From 262fb54a8e866213d1ade2da83aa7ec5a555a8b9 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 22 Dec 2023 12:55:07 +1300 Subject: [PATCH 2/8] Adds problems to migration --- ...20231222000000_delete_orphaned_insights.js | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/services/api/database/migrations/20231222000000_delete_orphaned_insights.js b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js index fa4c2860e5..7ca1d243f9 100644 --- a/services/api/database/migrations/20231222000000_delete_orphaned_insights.js +++ b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js @@ -3,12 +3,20 @@ * @returns { Promise } */ exports.up = function(knex) { - return knex.schema.raw(` - DELETE ef - FROM environment_fact ef - LEFT JOIN environment e ON ef.environment = e.id - WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' - `); + return Promise.all([ + knex.schema.raw(` + DELETE ef + FROM environment_fact ef + LEFT JOIN environment e ON ef.environment = e.id + WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' + `), + knex.schema.raw(` + DELETE ep + FROM environment_problem ep + LEFT JOIN environment e ON ep.environment = e.id + WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' + `), +]); }; /** From 2e87efc81c49a9c6c1f12614a2b80629f64c7b09 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 29 Dec 2023 08:35:45 +1100 Subject: [PATCH 3/8] fix: use Float for restore size --- services/api/src/typeDefs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api/src/typeDefs.js b/services/api/src/typeDefs.js index 12f577e019..8a0ba7ed91 100644 --- a/services/api/src/typeDefs.js +++ b/services/api/src/typeDefs.js @@ -975,7 +975,7 @@ const typeDefs = gql` """ The size of the restored file in bytes """ - restoreSize: Int + restoreSize: Float created: String } From 915f3818290ca790fa541ffd4e6576ca0b227acc Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 8 Jan 2024 08:09:24 +1300 Subject: [PATCH 4/8] Adds insights cleanup for missing projects --- .../migrations/20231222000000_delete_orphaned_insights.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api/database/migrations/20231222000000_delete_orphaned_insights.js b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js index 7ca1d243f9..1f3a279b73 100644 --- a/services/api/database/migrations/20231222000000_delete_orphaned_insights.js +++ b/services/api/database/migrations/20231222000000_delete_orphaned_insights.js @@ -8,15 +8,15 @@ exports.up = function(knex) { DELETE ef FROM environment_fact ef LEFT JOIN environment e ON ef.environment = e.id - WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' + WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' OR e.project not in (select id from project) `), knex.schema.raw(` DELETE ep FROM environment_problem ep LEFT JOIN environment e ON ep.environment = e.id - WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' + WHERE e.id IS NULL OR e.deleted != '0000-00-00 00:00:00' OR e.project not in (select id from project) `), -]); + ]); }; /** From 57d0fa9cc9829d43c89b4fc70b8867261072e43c Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 9 Jan 2024 08:43:39 +1300 Subject: [PATCH 5/8] Adds migration to delete orphaned environments --- .../20240108000000_fix_env_orphans.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 services/api/database/migrations/20240108000000_fix_env_orphans.js diff --git a/services/api/database/migrations/20240108000000_fix_env_orphans.js b/services/api/database/migrations/20240108000000_fix_env_orphans.js new file mode 100644 index 0000000000..edf1576202 --- /dev/null +++ b/services/api/database/migrations/20240108000000_fix_env_orphans.js @@ -0,0 +1,19 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + return knex('environment') + .update('deleted', knex.raw('NOW()')) + .whereNotIn('project', function() { + this.select('id').from('project'); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + return knex.schema +}; From 89a61e51c963817b3020079be1f3a5ca8cc06954 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 9 Jan 2024 08:54:32 +1300 Subject: [PATCH 6/8] changes logic of update for orphaned envs --- .../database/migrations/20240108000000_fix_env_orphans.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/api/database/migrations/20240108000000_fix_env_orphans.js b/services/api/database/migrations/20240108000000_fix_env_orphans.js index edf1576202..7313e1527e 100644 --- a/services/api/database/migrations/20240108000000_fix_env_orphans.js +++ b/services/api/database/migrations/20240108000000_fix_env_orphans.js @@ -4,10 +4,9 @@ */ exports.up = async function(knex) { return knex('environment') - .update('deleted', knex.raw('NOW()')) - .whereNotIn('project', function() { - this.select('id').from('project'); - }); + .leftJoin('project', 'environment.project', 'project.id') + .whereNull('project.id') + .update('deleted', knex.raw('NOW()')); }; /** From 56f75a76f7391646a8cddbd85d1783b8299c084d Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 11 Jan 2024 12:40:22 +1100 Subject: [PATCH 7/8] fix: return error when no matching event to prevent empty email notifications --- .../logs2notifications/internal/handler/email_events.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/logs2notifications/internal/handler/email_events.go b/services/logs2notifications/internal/handler/email_events.go index 3e16e85fd9..f26c19e463 100644 --- a/services/logs2notifications/internal/handler/email_events.go +++ b/services/logs2notifications/internal/handler/email_events.go @@ -60,7 +60,7 @@ func (h *Messaging) processEmailTemplates(notification *Notification) (string, s if err != nil { eventSplit := strings.Split(notification.Event, ":") if eventSplit[0] != "problem" { - return "", "", "", "", "", nil + return "", "", "", "", "", fmt.Errorf("no matching event") } if eventSplit[1] == "insert" { tpl = "problemNotification" @@ -127,7 +127,7 @@ func (h *Messaging) processEmailTemplates(notification *Notification) (string, s case "problemNotification": eventSplit := strings.Split(notification.Event, ":") if eventSplit[0] != "problem" && eventSplit[1] == "insert" { - return "", "", "", "", "", nil + return "", "", "", "", "", fmt.Errorf("no matching event") } mainHTMLTpl = `[{{.ProjectName}}] New problem found for {{.EnvironmentName}}
  • * Service: {{.ServiceName}}
    • {{ if ne .Severity "" }} @@ -142,7 +142,7 @@ func (h *Messaging) processEmailTemplates(notification *Notification) (string, s notification.Meta.EnvironmentName, ) default: - return "", "", "", "", "", nil + return "", "", "", "", "", fmt.Errorf("no matching event") } var body bytes.Buffer From 8189e19f422e3caa5036ea87436d478149e29dc7 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 15 Jan 2024 13:03:02 +1300 Subject: [PATCH 8/8] Limiting migration to only non-deleted orphaned envs --- .../api/database/migrations/20240108000000_fix_env_orphans.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/api/database/migrations/20240108000000_fix_env_orphans.js b/services/api/database/migrations/20240108000000_fix_env_orphans.js index 7313e1527e..7bb0072a31 100644 --- a/services/api/database/migrations/20240108000000_fix_env_orphans.js +++ b/services/api/database/migrations/20240108000000_fix_env_orphans.js @@ -6,6 +6,7 @@ exports.up = async function(knex) { return knex('environment') .leftJoin('project', 'environment.project', 'project.id') .whereNull('project.id') + .andWhere('environment.deleted','0000-00-00 00:00:00') .update('deleted', knex.raw('NOW()')); };