From b76d3062594a7707d8b2b6bad78259e45f55dc33 Mon Sep 17 00:00:00 2001 From: vinny Date: Mon, 3 Feb 2025 12:43:00 -0500 Subject: [PATCH 1/9] HARMONY-1998: Show warning status + sub_status --- services/harmony/app/frontends/workflow-ui.ts | 2 ++ .../app/views/workflow-ui/job/work-item-table-row.mustache.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/services/harmony/app/frontends/workflow-ui.ts b/services/harmony/app/frontends/workflow-ui.ts index 01880fa10..a3be8234d 100644 --- a/services/harmony/app/frontends/workflow-ui.ts +++ b/services/harmony/app/frontends/workflow-ui.ts @@ -481,8 +481,10 @@ function workItemRenderingFunctions(job: Job, isAdmin: boolean, isLogViewer: boo badgeClasses[WorkItemStatus.SUCCESSFUL] = 'success'; badgeClasses[WorkItemStatus.RUNNING] = 'info'; badgeClasses[WorkItemStatus.QUEUED] = 'warning'; + badgeClasses[WorkItemStatus.WARNING] = 'warning'; return { workflowItemBadge(): string { return badgeClasses[this.status]; }, + workflowItemStatus(): string { return this.sub_status ? `${this.status}: ${this.sub_status}` : this.status; }, workflowItemStep(): string { return sanitizeImage(this.serviceID); }, workflowItemCreatedAt(): string { return this.createdAt.getTime(); }, workflowItemUpdatedAt(): string { return this.updatedAt.getTime(); }, diff --git a/services/harmony/app/views/workflow-ui/job/work-item-table-row.mustache.html b/services/harmony/app/views/workflow-ui/job/work-item-table-row.mustache.html index d25e01e21..7b35c4692 100644 --- a/services/harmony/app/views/workflow-ui/job/work-item-table-row.mustache.html +++ b/services/harmony/app/views/workflow-ui/job/work-item-table-row.mustache.html @@ -2,7 +2,7 @@ {{workflowStepIndex}} {{workflowItemStep}} {{id}} - {{status}} + {{workflowItemStatus}} {{#isAdminOrLogViewer}} {{{workflowItemLogsButton}}} {{/isAdminOrLogViewer}} From e581ad83d283c3272088fb4f2afe4d4ecb65bc38 Mon Sep 17 00:00:00 2001 From: vinny Date: Mon, 3 Feb 2025 13:11:08 -0500 Subject: [PATCH 2/9] HARMONY-1998: Test message_category display --- services/harmony/app/frontends/workflow-ui.ts | 2 +- services/harmony/test/workflow-ui/work-items-table-row.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/harmony/app/frontends/workflow-ui.ts b/services/harmony/app/frontends/workflow-ui.ts index a3be8234d..92b7c938e 100644 --- a/services/harmony/app/frontends/workflow-ui.ts +++ b/services/harmony/app/frontends/workflow-ui.ts @@ -484,7 +484,7 @@ function workItemRenderingFunctions(job: Job, isAdmin: boolean, isLogViewer: boo badgeClasses[WorkItemStatus.WARNING] = 'warning'; return { workflowItemBadge(): string { return badgeClasses[this.status]; }, - workflowItemStatus(): string { return this.sub_status ? `${this.status}: ${this.sub_status}` : this.status; }, + workflowItemStatus(): string { return this.message_category ? `${this.status}: ${this.message_category}` : this.status; }, workflowItemStep(): string { return sanitizeImage(this.serviceID); }, workflowItemCreatedAt(): string { return this.createdAt.getTime(); }, workflowItemUpdatedAt(): string { return this.updatedAt.getTime(); }, diff --git a/services/harmony/test/workflow-ui/work-items-table-row.ts b/services/harmony/test/workflow-ui/work-items-table-row.ts index f461d296c..87e4b5412 100644 --- a/services/harmony/test/workflow-ui/work-items-table-row.ts +++ b/services/harmony/test/workflow-ui/work-items-table-row.ts @@ -37,7 +37,7 @@ const step2 = buildWorkflowStep( // build the items // give them an id so we know what id to request in the tests const item1 = buildWorkItem( - { jobID: targetJob.jobID, workflowStepIndex: 1, serviceID: step1ServiceId, status: WorkItemStatus.RUNNING, id: 1 }, + { jobID: targetJob.jobID, workflowStepIndex: 1, serviceID: step1ServiceId, status: WorkItemStatus.RUNNING, id: 1, message_category: 'smoothly' }, ); const item2 = buildWorkItem( { jobID: targetJob.jobID, workflowStepIndex: 1, serviceID: step1ServiceId, status: WorkItemStatus.SUCCESSFUL, id: 2 }, @@ -190,6 +190,10 @@ describe('Workflow UI work items table row route', function () { const listing = this.res.text; expect((listing.match(/retry-button/g) || []).length).to.equal(1); }); + it('returns the message_category along with the main status', async function () { + const listing = this.res.text; + expect(listing).to.contain(`${WorkItemStatus.RUNNING.valueOf()}: smoothly`); + }); }); describe('who requests a queued work item for their job', function () { @@ -288,7 +292,7 @@ describe('Workflow UI work items table row route', function () { expect(listing).to.not.contain(`${WorkItemStatus.SUCCESSFUL.valueOf()}`); expect(listing).to.not.contain(`${WorkItemStatus.CANCELED.valueOf()}`); expect(listing).to.not.contain(`${WorkItemStatus.READY.valueOf()}`); - expect(listing).to.contain(`${WorkItemStatus.RUNNING.valueOf()}`); + expect(listing).to.contain(`${WorkItemStatus.RUNNING.valueOf()}: smoothly`); }); }); }); From a2e9c73ed93dc7dc2248a5dc3a293f223e791895 Mon Sep 17 00:00:00 2001 From: vinny Date: Wed, 5 Feb 2025 09:10:53 -0500 Subject: [PATCH 3/9] HARMONY-1998: Filter by work item warning status --- .../js/workflow-ui/job/work-items-table.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/services/harmony/public/js/workflow-ui/job/work-items-table.js b/services/harmony/public/js/workflow-ui/job/work-items-table.js index a44d7f8b6..2f8d7d311 100644 --- a/services/harmony/public/js/workflow-ui/job/work-items-table.js +++ b/services/harmony/public/js/workflow-ui/job/work-items-table.js @@ -1,4 +1,4 @@ -import { formatDates, initCopyHandler } from '../table.js'; +import { formatDates, initCopyHandler, trimForDisplay } from '../table.js'; import toasts from '../toasts.js'; import PubSub from '../../pub-sub.js'; @@ -97,11 +97,13 @@ function initFilter(tableFilter) { { value: 'status: running', dbValue: 'running', field: 'status' }, { value: 'status: failed', dbValue: 'failed', field: 'status' }, { value: 'status: queued', dbValue: 'queued', field: 'status' }, + { value: 'status: warning', dbValue: 'warning', field: 'status' }, ]; const allowedValues = allowedList.map((t) => t.value); // eslint-disable-next-line no-new const tagInput = new Tagify(filterInput, { whitelist: allowedList, + delimiters: null, validate(tag) { if (allowedValues.includes(tag.value)) { return true; @@ -115,6 +117,21 @@ function initFilter(tableFilter) { enabled: 0, closeOnSelect: true, }, + templates: { + tag(tagData) { + return ` + +
+ ${trimForDisplay(tagData.value.split(': ')[1], 20)} +
+ `; + }, + }, }); const initialTags = JSON.parse(tableFilter); tagInput.addTags(initialTags); From 7061500256249cd5f4eb8ec692796cf4393fe648 Mon Sep 17 00:00:00 2001 From: vinny Date: Wed, 5 Feb 2025 09:35:58 -0500 Subject: [PATCH 4/9] HARMONY-1998: Allow filtering by message category --- services/harmony/app/frontends/workflow-ui.ts | 19 ++++++++++++++++++- .../harmony/app/models/work-item-interface.ts | 1 + .../js/workflow-ui/job/work-items-table.js | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/services/harmony/app/frontends/workflow-ui.ts b/services/harmony/app/frontends/workflow-ui.ts index 92b7c938e..d9d89cd58 100644 --- a/services/harmony/app/frontends/workflow-ui.ts +++ b/services/harmony/app/frontends/workflow-ui.ts @@ -53,6 +53,8 @@ interface TableQuery { userValues: string[], providerValues: string[], labelValues: string[], + messageCategoryValues: string[], + allowMessageCategoryValues: boolean, from: Date, to: Date, dateKind: 'createdAt' | 'updatedAt', @@ -85,10 +87,12 @@ function parseQuery( /* eslint-disable @typescript-eslint/no-explicit-any */ userValues: [], providerValues: [], labelValues: [], + messageCategoryValues: [], allowStatuses: true, allowServices: true, allowUsers: true, allowProviders: true, + allowMessageCategoryValues: true, // date controls from: undefined, to: undefined, @@ -101,6 +105,7 @@ function parseQuery( /* eslint-disable @typescript-eslint/no-explicit-any */ tableQuery.allowServices = !(requestQuery.disallowservice === 'on'); tableQuery.allowUsers = !(requestQuery.disallowuser === 'on'); tableQuery.allowProviders = !(requestQuery.disallowprovider === 'on'); + tableQuery.allowMessageCategoryValues = !(requestQuery.disallowmessagecategory === 'on'); const selectedOptions: { field: string, dbValue: string, value: string }[] = JSON.parse(requestQuery.tablefilter); const validStatusSelections = selectedOptions @@ -123,6 +128,10 @@ function parseQuery( /* eslint-disable @typescript-eslint/no-explicit-any */ .filter(option => /^provider: [A-Za-z0-9_]{1,100}$/.test(option.value)); const providerValues = validProviderSelections.map(option => option.value.split('provider: ')[1].toLowerCase()); + const validMessageCategorySelections = selectedOptions + .filter(option => /^message category: .{1,100}$/.test(option.value)); + const messageCategoryValues = validMessageCategorySelections.map(option => option.dbValue || option.value.split('message category: ')[1].toLowerCase()); + if ((statusValues.length + serviceValues.length + userValues.length + providerValues.length) > maxFilters) { throw new RequestValidationError(`Maximum amount of filters (${maxFilters}) was exceeded.`); } @@ -130,12 +139,14 @@ function parseQuery( /* eslint-disable @typescript-eslint/no-explicit-any */ .concat(validServiceSelections) .concat(validUserSelections) .concat(validProviderSelections) - .concat(validLabelSelections)); + .concat(validLabelSelections) + .concat(validMessageCategorySelections)); tableQuery.statusValues = statusValues; tableQuery.serviceValues = serviceValues; tableQuery.userValues = userValues; tableQuery.providerValues = providerValues; tableQuery.labelValues = labelValues; + tableQuery.messageCategoryValues = messageCategoryValues; } // everything in the Workflow UI uses the browser timezone, so we need a timezone offset const offSetMs = parseInt(requestQuery.tzoffsetminutes || 0) * 60 * 1000; @@ -538,6 +549,12 @@ function tableQueryToWorkItemQuery(tableFilter: TableQuery, jobID: string, id?: in: tableFilter.allowStatuses, }; } + if (tableFilter.messageCategoryValues.length) { + itemQuery.whereIn.message_category = { + values: tableFilter.messageCategoryValues, + in: tableFilter.allowMessageCategoryValues, + }; + } if (tableFilter.from || tableFilter.to) { itemQuery.dates = { field: tableFilter.dateKind }; itemQuery.dates.from = tableFilter.from; diff --git a/services/harmony/app/models/work-item-interface.ts b/services/harmony/app/models/work-item-interface.ts index 72986ec0b..c3db41fc5 100644 --- a/services/harmony/app/models/work-item-interface.ts +++ b/services/harmony/app/models/work-item-interface.ts @@ -91,6 +91,7 @@ export interface WorkItemQuery { }; whereIn?: { status?: { in: boolean, values: string[] }; + message_category?: { in: boolean, values: string[] }; }; dates?: { from?: Date; diff --git a/services/harmony/public/js/workflow-ui/job/work-items-table.js b/services/harmony/public/js/workflow-ui/job/work-items-table.js index 2f8d7d311..2fc2c417b 100644 --- a/services/harmony/public/js/workflow-ui/job/work-items-table.js +++ b/services/harmony/public/js/workflow-ui/job/work-items-table.js @@ -100,12 +100,14 @@ function initFilter(tableFilter) { { value: 'status: warning', dbValue: 'warning', field: 'status' }, ]; const allowedValues = allowedList.map((t) => t.value); + allowedList.push({ value: 'message category: no-data', dbValue: 'no-data', field: 'message_category' }); // eslint-disable-next-line no-new const tagInput = new Tagify(filterInput, { whitelist: allowedList, delimiters: null, validate(tag) { - if (allowedValues.includes(tag.value)) { + if (allowedValues.includes(tag.value) + || /^message category: .{1,100}$/.test(tag.value)) { return true; } return false; From 952064f3c67b9fa1d5fae9290fa1812ef5e4f1dc Mon Sep 17 00:00:00 2001 From: vinny Date: Wed, 5 Feb 2025 10:03:19 -0500 Subject: [PATCH 5/9] HARMONY-1998: Integrate disallowMessageCategoryChecked --- services/harmony/app/frontends/workflow-ui.ts | 1 + .../harmony/app/views/workflow-ui/job/index.mustache.html | 6 ++++++ services/harmony/public/js/workflow-ui/job/index.js | 1 + .../harmony/public/js/workflow-ui/job/work-items-table.js | 2 +- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/services/harmony/app/frontends/workflow-ui.ts b/services/harmony/app/frontends/workflow-ui.ts index d9d89cd58..c62c8f7b5 100644 --- a/services/harmony/app/frontends/workflow-ui.ts +++ b/services/harmony/app/frontends/workflow-ui.ts @@ -437,6 +437,7 @@ export async function getJob( updatedAtChecked: dateKind == 'updatedAt' ? 'checked' : '', createdAtChecked: dateKind != 'updatedAt' ? 'checked' : '', disallowStatusChecked: requestQuery.disallowstatus === 'on' ? 'checked' : '', + disallowMessageCategoryChecked: requestQuery.disallowmessagecategory === 'on' ? 'checked' : '', selectedFilters: originalValues, version, isAdminRoute: req.context.isAdminAccess, diff --git a/services/harmony/app/views/workflow-ui/job/index.mustache.html b/services/harmony/app/views/workflow-ui/job/index.mustache.html index 852be7034..288347e6b 100644 --- a/services/harmony/app/views/workflow-ui/job/index.mustache.html +++ b/services/harmony/app/views/workflow-ui/job/index.mustache.html @@ -63,6 +63,12 @@ negate statuses +
+ + +
page size diff --git a/services/harmony/public/js/workflow-ui/job/index.js b/services/harmony/public/js/workflow-ui/job/index.js index f22616653..0b1d5c93f 100644 --- a/services/harmony/public/js/workflow-ui/job/index.js +++ b/services/harmony/public/js/workflow-ui/job/index.js @@ -14,6 +14,7 @@ async function init() { }); params.tableFilter = document.getElementsByName('tableFilter')[0].getAttribute('data-value'); params.disallowStatus = document.getElementsByName('disallowStatus')[0].checked ? 'on' : ''; + params.disallowMessageCategory = document.getElementsByName('disallowMessageCategory')[0].checked ? 'on' : ''; params.dateKind = document.getElementById('dateKindUpdated').checked ? 'updatedAt' : 'createdAt'; // kick off job state change links logic if this user is allowed to change the job state diff --git a/services/harmony/public/js/workflow-ui/job/work-items-table.js b/services/harmony/public/js/workflow-ui/job/work-items-table.js index 2fc2c417b..d661eb1db 100644 --- a/services/harmony/public/js/workflow-ui/job/work-items-table.js +++ b/services/harmony/public/js/workflow-ui/job/work-items-table.js @@ -36,7 +36,7 @@ import PubSub from '../../pub-sub.js'; */ async function load(params, checkJobStatus) { let tableUrl = `./${params.jobID}/work-items?page=${params.currentPage}&limit=${params.limit}&checkJobStatus=${checkJobStatus}`; - tableUrl += `&tableFilter=${encodeURIComponent(params.tableFilter)}&disallowStatus=${params.disallowStatus}`; + tableUrl += `&tableFilter=${encodeURIComponent(params.tableFilter)}&disallowStatus=${params.disallowStatus}&disallowMessageCategory=${params.disallowMessageCategory}`; tableUrl += `&fromDateTime=${encodeURIComponent(params.fromDateTime)}&toDateTime=${encodeURIComponent(params.toDateTime)}`; tableUrl += `&tzOffsetMinutes=${params.tzOffsetMinutes}&dateKind=${params.dateKind}`; const res = await fetch(tableUrl); From 2cde0818223c59c072eb95a04f523a51c3f2730e Mon Sep 17 00:00:00 2001 From: vinny Date: Wed, 5 Feb 2025 14:09:58 -0500 Subject: [PATCH 6/9] HARMONY-1998: Test message category filter --- .../test/workflow-ui/work-items-table.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/services/harmony/test/workflow-ui/work-items-table.ts b/services/harmony/test/workflow-ui/work-items-table.ts index bbfe11e3d..b420f5a36 100644 --- a/services/harmony/test/workflow-ui/work-items-table.ts +++ b/services/harmony/test/workflow-ui/work-items-table.ts @@ -131,6 +131,8 @@ describe('Workflow UI work items table route', function () { await otherItem3.save(this.trx); const otherItem4 = buildWorkItem({ jobID: otherJob.jobID, status: WorkItemStatus.READY }); await otherItem4.save(this.trx); + const otherItem5 = buildWorkItem({ jobID: otherJob.jobID, status: WorkItemStatus.WARNING, message_category: 'no-data' }); + await otherItem5.save(this.trx); const otherStep1 = buildWorkflowStep({ jobID: otherJob.jobID, stepIndex: 1 }); await otherStep1.save(this.trx); const otherStep2 = buildWorkflowStep({ jobID: otherJob.jobID, stepIndex: 2 }); @@ -572,7 +574,7 @@ describe('Workflow UI work items table route', function () { hookWorkflowUIWorkItems({ username: 'adam', jobID: otherJob.jobID }); it('returns metrics logs links for each each work item', function () { const listing = this.res.text; - expect((listing.match(/logs-metrics/g) || []).length).to.equal(4); + expect((listing.match(/logs-metrics/g) || []).length).to.equal(5); }); }); @@ -585,6 +587,24 @@ describe('Workflow UI work items table route', function () { }); }); + describe('when the admin filters otherJob\'s items by status IN [WARNING]', function () { + hookWorkflowUIWorkItems({ username: 'adam', jobID: otherJob.jobID, + query: { tableFilter: '[{"value":"status: warning","dbValue":"warning","field":"status"}]' } }); + it('contains the WARNING work item', async function () { + expect((this.res.text.match(/work-item-table-row/g) || []).length).to.equal(1); + expect(this.res.text).to.contain(`${WorkItemStatus.WARNING.valueOf()}: no-data`); + }); + }); + + describe('when the admin filters otherJob\'s items by message_category IN [no-data]', function () { + hookWorkflowUIWorkItems({ username: 'adam', jobID: otherJob.jobID, + query: { tableFilter: '[{"value":"message category: no-data","dbValue":"no-data","field":"message_category"}]' } }); + it('contains the no-data work item', async function () { + expect((this.res.text.match(/work-item-table-row/g) || []).length).to.equal(1); + expect(this.res.text).to.contain(`${WorkItemStatus.WARNING.valueOf()}: no-data`); + }); + }); + describe('when the user is not part of the admin group', function () { hookAdminWorkflowUIWorkItems({ username: 'eve', jobID: targetJob.jobID }); it('returns an error', function () { From c6599c45e4f5d8b5825043fa4fb702425c49f02e Mon Sep 17 00:00:00 2001 From: vinny Date: Thu, 6 Feb 2025 13:31:31 -0500 Subject: [PATCH 7/9] HARMONY-1998: Fix enum value --- services/harmony/public/js/workflow-ui/job/work-items-table.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/harmony/public/js/workflow-ui/job/work-items-table.js b/services/harmony/public/js/workflow-ui/job/work-items-table.js index d661eb1db..269d642fe 100644 --- a/services/harmony/public/js/workflow-ui/job/work-items-table.js +++ b/services/harmony/public/js/workflow-ui/job/work-items-table.js @@ -100,7 +100,7 @@ function initFilter(tableFilter) { { value: 'status: warning', dbValue: 'warning', field: 'status' }, ]; const allowedValues = allowedList.map((t) => t.value); - allowedList.push({ value: 'message category: no-data', dbValue: 'no-data', field: 'message_category' }); + allowedList.push({ value: 'message category: nodata', dbValue: 'nodata', field: 'message_category' }); // eslint-disable-next-line no-new const tagInput = new Tagify(filterInput, { whitelist: allowedList, From b4ab3ccec25ca0821ca24a262f1adc5944c62baf Mon Sep 17 00:00:00 2001 From: vinny Date: Thu, 6 Feb 2025 15:54:00 -0500 Subject: [PATCH 8/9] HARMONY-1998: Fix null case --- services/harmony/app/models/work-item.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/harmony/app/models/work-item.ts b/services/harmony/app/models/work-item.ts index 93eb233be..bf4136b35 100644 --- a/services/harmony/app/models/work-item.ts +++ b/services/harmony/app/models/work-item.ts @@ -431,7 +431,7 @@ export async function queryAll( if (constraint.in) { void queryBuilder.whereIn(field, constraint.values); } else { - void queryBuilder.whereNotIn(field, constraint.values); + void queryBuilder.where(builder => builder.whereNotIn(field, constraint.values).orWhereNull(field)); } } } From a3af3620de3f126a4e0f19e3ac9d60a144ec9584 Mon Sep 17 00:00:00 2001 From: vinny Date: Thu, 6 Feb 2025 16:04:25 -0500 Subject: [PATCH 9/9] HARMONY-1998: Test null case --- .../harmony/test/workflow-ui/work-items-table.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/harmony/test/workflow-ui/work-items-table.ts b/services/harmony/test/workflow-ui/work-items-table.ts index b420f5a36..ee325a550 100644 --- a/services/harmony/test/workflow-ui/work-items-table.ts +++ b/services/harmony/test/workflow-ui/work-items-table.ts @@ -124,7 +124,7 @@ describe('Workflow UI work items table route', function () { await step2.save(this.trx); await otherJob.save(this.trx); - const otherItem1 = buildWorkItem({ jobID: otherJob.jobID, status: WorkItemStatus.CANCELED }); + const otherItem1 = buildWorkItem({ jobID: otherJob.jobID, status: WorkItemStatus.CANCELED, message_category: 'jeepers' }); await otherItem1.save(this.trx); const otherItem2 = buildWorkItem({ jobID: otherJob.jobID, status: WorkItemStatus.FAILED }); await otherItem2.save(this.trx); @@ -605,6 +605,16 @@ describe('Workflow UI work items table route', function () { }); }); + describe('when the admin filters otherJob\'s items by message_category NOT IN [no-data]', function () { + hookWorkflowUIWorkItems({ username: 'adam', jobID: otherJob.jobID, + query: { disallowMessageCategory: 'on', tableFilter: '[{"value":"message category: no-data","dbValue":"no-data","field":"message_category"}]' } }); + it('contains all but the no-data work item', async function () { + expect((this.res.text.match(/work-item-table-row/g) || []).length).to.equal(4); + expect(this.res.text).to.not.contain(`${WorkItemStatus.WARNING.valueOf()}: no-data`); + expect(this.res.text).to.contain(`${WorkItemStatus.CANCELED.valueOf()}: jeepers`); + }); + }); + describe('when the user is not part of the admin group', function () { hookAdminWorkflowUIWorkItems({ username: 'eve', jobID: targetJob.jobID }); it('returns an error', function () {