From 00c4847176ea3d0c1752b04f04ed4bab935550fb Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 12:46:43 +0200 Subject: [PATCH 01/67] chore: improve status logging --- .../src/expectationManager.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index edc3cf22..32e9d408 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -19,6 +19,7 @@ import PromisePool from '@supercharge/promise-pool' /** * The Expectation Manager is responsible for tracking the state of the Expectations, * and communicate with the Workers to progress them. + * @see FOR_DEVELOPERS.md */ export class ExpectationManager { @@ -1078,7 +1079,10 @@ export class ExpectationManager { trackedPackageContainer.packageContainer ) if (!dispose.disposed) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, dispose.reason) + this.updateTrackedPackageContainerStatus( + trackedPackageContainer, + 'Unable to dispose: ' + dispose.reason + ) continue // Break further execution for this PackageContainer } trackedPackageContainer.currentWorker = null @@ -1115,7 +1119,10 @@ export class ExpectationManager { notSupportReason = 'Found no worker that supports this packageContainer' } if (notSupportReason) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, notSupportReason) + this.updateTrackedPackageContainerStatus( + trackedPackageContainer, + 'Not supported: ' + notSupportReason + ) continue // Break further execution for this PackageContainer } } @@ -1140,7 +1147,11 @@ export class ExpectationManager { trackedPackageContainer.packageContainer ) if (!cronJobStatus.completed) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, cronJobStatus.reason) + console.log(trackedPackageContainer.packageContainer.cronjobs) + this.updateTrackedPackageContainerStatus( + trackedPackageContainer, + 'Cron job not completed: ' + cronJobStatus.reason + ) continue } } @@ -1161,7 +1172,7 @@ export class ExpectationManager { if (updatedReason) { this.logger.info( - `${trackedPackageContainer.packageContainer.label}: Reason: "${trackedPackageContainer.status.reason}"` + `PackageContainerStatus "${trackedPackageContainer.packageContainer.label}": Reason: "${trackedPackageContainer.status.reason}"` ) } From cb9147789f440441ed999bf252d8d1089c6a677f Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 12:47:36 +0200 Subject: [PATCH 02/67] fix: add fast-path for when no cronjobs are defined --- .../windowsWorker/packageContainerExpectationHandler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts index 58561d7b..fabc47bf 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts @@ -21,6 +21,11 @@ export async function runPackageContainerCronJob( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { + // Quick-check: If there are no cronjobs at all, no need to check: + if (!Object.keys(packageContainer.cronjobs).length) { + return { completed: true } // all good + } + const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'cronjob') if (!lookup.ready) return { completed: lookup.ready, reason: lookup.reason } From 7ecc4e5669b753f09772139ad4dc055bc06e91e1 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 12:48:12 +0200 Subject: [PATCH 03/67] feat: add hack for a nrk-workflow called "smartbull" --- .../generic/src/expectationGenerator.ts | 215 +++++++++++------- 1 file changed, 139 insertions(+), 76 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index e049f508..93fd35f5 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -6,7 +6,7 @@ import { PackageContainers, PackageManagerSettings, } from './packageManager' -import { Expectation, hashObj, PackageContainerExpectation } from '@shared/api' +import { Expectation, hashObj, PackageContainerExpectation, literal } from '@shared/api' export interface ExpectedPackageWrapMediaFile extends ExpectedPackageWrap { expectedPackage: ExpectedPackage.ExpectedPackageMediaFile @@ -55,63 +55,121 @@ export function generateExpectations( activeRundownMap.set(activeRundown._id, activeRundown) } - for (const expWrap of expectedPackages) { - let exp: Expectation.Any | undefined = undefined - - if (expWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { - exp = generateMediaFileCopy(managerId, expWrap, settings) - } else if (expWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { - exp = generateQuantelCopy(managerId, expWrap) + function prioritizeExpectation(packageWrap: ExpectedPackageWrap, exp: Expectation.Any): void { + // Prioritize + /* + 0: Things that are to be played out like RIGHT NOW + 10: Things that are to be played out pretty soon (things that could be cued anytime now) + 100: Other things that affect users (GUI things) + 1000+: Other things that can be played out + */ + + let prioAdd = 1000 + const activeRundown: ActiveRundown | undefined = packageWrap.expectedPackage.rundownId + ? activeRundownMap.get(packageWrap.expectedPackage.rundownId) + : undefined + + if (activeRundown) { + // The expected package is in an active rundown + prioAdd = 0 + activeRundown._rank // Earlier rundowns should have higher priority } - if (exp) { - // Prioritize - /* - 0: Things that are to be played out like RIGHT NOW - 10: Things that are to be played out pretty soon (things that could be cued anytime now) - 100: Other things that affect users (GUI things) - 1000+: Other things that can be played out - */ - - let prioAdd = 1000 - const activeRundown: ActiveRundown | undefined = expWrap.expectedPackage.rundownId - ? activeRundownMap.get(expWrap.expectedPackage.rundownId) - : undefined - - if (activeRundown) { - // The expected package is in an active rundown - prioAdd = 0 + activeRundown._rank // Earlier rundowns should have higher priority + exp.priority += prioAdd + } + function addExpectation(packageWrap: ExpectedPackageWrap, exp: Expectation.Any) { + const existingExp = expectations[exp.id] + if (existingExp) { + // There is already an expectation pointing at the same place. + + existingExp.priority = Math.min(existingExp.priority, exp.priority) + + const existingPackage = existingExp.fromPackages[0] + const newPackage = exp.fromPackages[0] + + if (existingPackage.expectedContentVersionHash !== newPackage.expectedContentVersionHash) { + // log warning: + console.log(`WARNING: 2 expectedPackages have the same content, but have different contentVersions!`) + console.log(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) + console.log(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) + console.log(`${JSON.stringify(exp.startRequirement)}`) + + // TODO: log better warnings! + } else { + existingExp.fromPackages.push(exp.fromPackages[0]) } - exp.priority += prioAdd + } else { + expectations[exp.id] = { + ...exp, + sideEffect: packageWrap.expectedPackage.sideEffect, + external: packageWrap.external, + } + } + } - const existingExp = expectations[exp.id] - if (existingExp) { - // There is already an expectation pointing at the same place. + const smartbullExpectations: ExpectedPackageWrap[] = [] // Hack, Smartbull + let orgSmartbullExpectation: ExpectedPackageWrap | undefined = undefined // Hack, Smartbull - existingExp.priority = Math.min(existingExp.priority, exp.priority) + for (const packageWrap of expectedPackages) { + let exp: Expectation.Any | undefined = undefined - const existingPackage = existingExp.fromPackages[0] - const newPackage = exp.fromPackages[0] + // Temporary hacks: handle smartbull: + if (packageWrap.expectedPackage._id.match(/smartbull_auto_clip/)) { + // hack + orgSmartbullExpectation = packageWrap + continue + } + if (packageWrap.sources.find((source) => source.containerId === 'source-smartbull')) { + smartbullExpectations.push(packageWrap) + continue + } - if (existingPackage.expectedContentVersionHash !== newPackage.expectedContentVersionHash) { - // log warning: - console.log( - `WARNING: 2 expectedPackages have the same content, but have different contentVersions!` - ) - console.log(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) - console.log(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) - console.log(`${JSON.stringify(exp.startRequirement)}`) + if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { + exp = generateMediaFileCopy(managerId, packageWrap, settings) + } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { + exp = generateQuantelCopy(managerId, packageWrap) + } + if (exp) { + prioritizeExpectation(packageWrap, exp) + addExpectation(packageWrap, exp) + } + } - // TODO: log better warnings! - } else { - existingExp.fromPackages.push(exp.fromPackages[0]) + // hack: handle Smartbull: + if (orgSmartbullExpectation) { + // Sort alphabetically on filePath: + smartbullExpectations.sort((a, b) => { + const expA = a.expectedPackage as ExpectedPackage.ExpectedPackageMediaFile + const expB = b.expectedPackage as ExpectedPackage.ExpectedPackageMediaFile + + // lowest first: + if (expA.content.filePath > expB.content.filePath) return 1 + if (expA.content.filePath < expB.content.filePath) return -1 + + return 0 + }) + // Pick the last one: + const bestSmartbull = smartbullExpectations[smartbullExpectations.length - 1] as + | ExpectedPackageWrapMediaFile + | undefined + if (bestSmartbull) { + if (orgSmartbullExpectation.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { + const org = orgSmartbullExpectation as ExpectedPackageWrapMediaFile + + const newPackage: ExpectedPackageWrapMediaFile = { + ...org, + expectedPackage: { + ...org.expectedPackage, + // Take these from bestSmartbull: + content: bestSmartbull.expectedPackage.content, + version: bestSmartbull.expectedPackage.version, + sources: bestSmartbull.expectedPackage.sources, + }, } - } else { - expectations[exp.id] = { - ...exp, - sideEffect: expWrap.expectedPackage.sideEffect, - external: expWrap.external, + const exp = generateMediaFileCopy(managerId, newPackage, settings) + if (exp) { + prioritizeExpectation(newPackage, exp) + addExpectation(newPackage, exp) } - } + } else console.log('orgSmartbullExpectation is not a MEDIA_FILE') } } @@ -296,7 +354,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E return exp } function generatePackageScan(expectation: Expectation.FileCopy | Expectation.QuantelClipCopy): Expectation.PackageScan { - const scan: Expectation.PackageScan = { + return literal({ id: expectation.id + '_scan', priority: expectation.priority + 100, managerId: expectation.managerId, @@ -337,14 +395,12 @@ function generatePackageScan(expectation: Expectation.FileCopy | Expectation.Qua }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return scan + }) } function generatePackageDeepScan( expectation: Expectation.FileCopy | Expectation.QuantelClipCopy ): Expectation.PackageDeepScan { - const deepScan: Expectation.PackageDeepScan = { + return literal({ id: expectation.id + '_deepscan', priority: expectation.priority + 1001, managerId: expectation.managerId, @@ -390,9 +446,7 @@ function generatePackageDeepScan( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return deepScan + }) } function generateMediaFileThumbnail( @@ -401,7 +455,7 @@ function generateMediaFileThumbnail( settings: ExpectedPackage.SideEffectThumbnailSettings, packageContainer: PackageContainer ): Expectation.MediaFileThumbnail { - const thumbnail: Expectation.MediaFileThumbnail = { + return literal({ id: expectation.id + '_thumbnail', priority: expectation.priority + 1002, managerId: expectation.managerId, @@ -444,9 +498,7 @@ function generateMediaFileThumbnail( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return thumbnail + }) } function generateMediaFilePreview( expectation: Expectation.FileCopy, @@ -454,7 +506,7 @@ function generateMediaFilePreview( settings: ExpectedPackage.SideEffectPreviewSettings, packageContainer: PackageContainer ): Expectation.MediaFilePreview { - const preview: Expectation.MediaFilePreview = { + return literal({ id: expectation.id + '_preview', priority: expectation.priority + 1003, managerId: expectation.managerId, @@ -496,8 +548,7 @@ function generateMediaFilePreview( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - return preview + }) } function generateQuantelClipThumbnail( @@ -506,7 +557,7 @@ function generateQuantelClipThumbnail( settings: ExpectedPackage.SideEffectThumbnailSettings, packageContainer: PackageContainer ): Expectation.QuantelClipThumbnail { - const thumbnail: Expectation.QuantelClipThumbnail = { + return literal({ id: expectation.id + '_thumbnail', priority: expectation.priority + 1002, managerId: expectation.managerId, @@ -548,9 +599,7 @@ function generateQuantelClipThumbnail( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return thumbnail + }) } function generateQuantelClipPreview( expectation: Expectation.QuantelClipCopy, @@ -558,7 +607,7 @@ function generateQuantelClipPreview( settings: ExpectedPackage.SideEffectPreviewSettings, packageContainer: PackageContainer ): Expectation.QuantelClipPreview { - const preview: Expectation.QuantelClipPreview = { + return literal({ id: expectation.id + '_preview', priority: expectation.priority + 1003, managerId: expectation.managerId, @@ -594,9 +643,6 @@ function generateQuantelClipPreview( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP_PREVIEW, - // bitrate: string // default: '40k' - // width: number - // height: number }, }, workOptions: { @@ -605,8 +651,7 @@ function generateQuantelClipPreview( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - return preview + }) } // function generateMediaFileHTTPCopy(expectation: Expectation.FileCopy): Expectation.FileCopy { @@ -663,9 +708,9 @@ export function generatePackageContainerExpectations( ): { [id: string]: PackageContainerExpectation } { const o: { [id: string]: PackageContainerExpectation } = {} - // This is temporary, to test/show how the for (const [containerId, packageContainer] of Object.entries(packageContainers)) { - if (containerId === 'source0') { + // This is temporary, to test/show how a monitor would work: + if (containerId === 'source_monitor') { o[containerId] = { ...packageContainer, id: containerId, @@ -680,6 +725,24 @@ export function generatePackageContainerExpectations( }, } } + + // This is a hard-coded hack for the "smartbull" feature, + // to be replaced or moved out later: + if (containerId === 'source-smartbull') { + o[containerId] = { + ...packageContainer, + id: containerId, + managerId: managerId, + cronjobs: {}, + monitors: { + packages: { + targetLayers: ['target0'], + ignore: '.bat', + }, + }, + } + } } + return o } From a6811663460ad6e30bf50d0845b753fc49ea2a34 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 13:14:26 +0200 Subject: [PATCH 04/67] fix: smartbull: filter filenames --- .../packages/generic/src/expectationGenerator.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 93fd35f5..4c459539 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -117,7 +117,11 @@ export function generateExpectations( orgSmartbullExpectation = packageWrap continue } - if (packageWrap.sources.find((source) => source.containerId === 'source-smartbull')) { + if ( + packageWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE && + packageWrap.sources.find((source) => source.containerId === 'source-smartbull') && + (packageWrap as ExpectedPackageWrapMediaFile).expectedPackage.content.filePath.match(/^smartbull/) // the files are on the form "smartbull_TIMESTAMP.mxf/mp4" + ) { smartbullExpectations.push(packageWrap) continue } @@ -736,8 +740,7 @@ export function generatePackageContainerExpectations( cronjobs: {}, monitors: { packages: { - targetLayers: ['target0'], - ignore: '.bat', + targetLayers: [], // not used, since the layers of the original smartbull-package are used }, }, } From a80b991ab1b165e9c6bd34c17f48b93c9032701d Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 13:35:05 +0200 Subject: [PATCH 05/67] chore: release32 deps, and a few hacks to quicklt fix type issues --- .../generic/src/expectationGenerator.ts | 2 +- package.json | 4 +- .../src/worker/accessorHandlers/quantel.ts | 43 +++++++------------ .../quantelClipThumbnail.ts | 24 +---------- yarn.lock | 26 +++++------ 5 files changed, 34 insertions(+), 65 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 4c459539..d894fdd3 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -346,7 +346,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E content: content, version: { type: Expectation.Version.Type.QUANTEL_CLIP, - ...expWrapQuantelClip.expectedPackage.version, + ...(expWrapQuantelClip.expectedPackage.version as any), // hack, for release 32 }, }, workOptions: { diff --git a/package.json b/package.json index a0245cd8..8b440a59 100644 --- a/package.json +++ b/package.json @@ -55,8 +55,8 @@ "node": ">=12.3.0" }, "dependencies": { - "@sofie-automation/blueprints-integration": "1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0", - "@sofie-automation/server-core-integration": "1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0" + "@sofie-automation/blueprints-integration": "1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0", + "@sofie-automation/server-core-integration": "1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "husky": { diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index 30a47824..458c73ee 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -68,9 +68,13 @@ export class QuantelAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { - if (!this.accessor.transformerURL) return undefined - - const clip = await this.getClip() - if (clip) { - const baseURL = this.accessor.transformerURL - const url = `/quantel/homezone/clips/streams/${clip.ClipID}/stream.mpd` - return { - baseURL, - url, - fullURL: [ - baseURL.replace(/\/$/, ''), // trim trailing slash - url.replace(/^\//, ''), // trim leading slash - ].join('/'), - } - } + // Not supported in Release 32: return undefined } @@ -294,10 +284,14 @@ export class QuantelAccessorHandle extends GenericAccessorHandle('gateways', {}) + const ISAUrls: string[] = [] + if (this.accessor.ISAUrlMaster) ISAUrls.push(this.accessor.ISAUrlMaster) + if (this.accessor.ISAUrlBackup) ISAUrls.push(this.accessor.ISAUrlBackup) + // These errors are just for types. User-facing checks are done in this.checkAccessor() if (!this.accessor.quantelGatewayUrl) throw new Error('accessor.quantelGatewayUrl is not set') - if (!this.accessor.ISAUrls) throw new Error('accessor.ISAUrls is not set') - if (!this.accessor.ISAUrls.length) throw new Error('accessor.ISAUrls array is empty') + if (!ISAUrls) throw new Error('accessor.ISAUrls is not set') + if (!ISAUrls.length) throw new Error('accessor.ISAUrls array is empty') // if (!this.accessor.serverId) throw new Error('accessor.serverId is not set') const id = `${this.accessor.quantelGatewayUrl}` @@ -306,12 +300,7 @@ export class QuantelAccessorHandle extends GenericAccessorHandle console.log(`Quantel.QuantelGateway`, e)) @@ -329,9 +318,9 @@ export class QuantelAccessorHandle extends GenericAccessorHandle ): Promise<{ baseURL: string; url: string } | undefined> { if (!lookupSource.accessor) throw new Error(`Source accessor not set!`) @@ -239,27 +239,7 @@ async function getThumbnailURL( throw new Error(`Source accessor should have been a Quantel ("${lookupSource.accessor.type}")`) if (!isQuantelClipAccessorHandle(lookupSource.handle)) throw new Error(`Source AccessHandler type is wrong`) - if (!lookupSource.accessor.transformerURL) return undefined - - const clip = await lookupSource.handle.getClip() - if (clip) { - const width = exp.endRequirement.version.width - let frame: number = exp.endRequirement.version.frame || 0 - if (frame > 0 && frame < 1) { - // If between 0 and 1, will be treated as % of the source duration: - const totalFrames = parseInt(clip.Frames, 10) - - if (totalFrames) { - frame = Math.floor(totalFrames * frame) - } - } - - return { - baseURL: lookupSource.accessor.transformerURL, - url: `/quantel/homezone/clips/stills/${clip.ClipID}/${frame}.${width ? width + '.' : ''}jpg`, - } - } - return undefined + return undefined // not supported in Release 32 } export function getSourceHTTPHandle( worker: GenericWorker, diff --git a/yarn.lock b/yarn.lock index ae2df9eb..07aa4306 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1478,13 +1478,13 @@ tslib "^2.0.3" underscore "1.12.0" -"@sofie-automation/blueprints-integration@1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0": - version "1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0.tgz#9a4a528559e4dad175f9bb4f369d3bb1887aadbe" - integrity sha512-CMhfQiEMMIZRTymHdkqfVgabA0ElXfvJvhApyEAFZ7LQbf7+EzT248hyxi2u1iT5ZeFoa3LASWQIY4yBvcIK8w== +"@sofie-automation/blueprints-integration@1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0": + version "1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0.tgz#508defbd4f83679f3c2dd843b1bf0c9ff9578c20" + integrity sha512-SApaMtAZdJ/bnE+KCW/nmGu7iqf1WhoFBv+1l6gq7ySPvuVSbHrVaRR8tqL7XbKdDorpzQI8x97fjWWauvGdhg== dependencies: moment "2.29.1" - timeline-state-resolver-types "5.7.0-nightly-release33-20210421-102534-d3150d7e3.0" + timeline-state-resolver-types "5.7.0" tslib "^2.1.0" underscore "1.12.1" @@ -1507,10 +1507,10 @@ read-pkg-up "^7.0.1" shelljs "^0.8.4" -"@sofie-automation/server-core-integration@1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0": - version "1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.34.0-nightly-test-mserk5-20210507-123656-f937ba8.0.tgz#7e8cf63085ad2265d24d657f89596f2350d2b3fa" - integrity sha512-1VWmOlAHcXJ5K3MX/s1UFK1rPWzka/WcFHI0RF5imcu3ypN1bRvbYjRfmY2PCBvGbO7AJ8q5cbRs2Fd84fDZEw== +"@sofie-automation/server-core-integration@1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0": + version "1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.32.2-nightly-feat-listen-to-packageInfo-20210526-110955-fb89361.0.tgz#63fccaae028ca44603e1ad9dbd09d1c8c718b093" + integrity sha512-ietJTOKBQIEy/mNQaXpByJov8nR8DPDv7SQ8uHmmjOAtLQJJKQaL6oZ+9fKF0W1wta+XaSq5upYcypx7B62eyQ== dependencies: data-store "3.1.0" ejson "^2.2.0" @@ -9648,10 +9648,10 @@ timeline-state-resolver-types@5.5.1: dependencies: tslib "^1.13.0" -timeline-state-resolver-types@5.7.0-nightly-release33-20210421-102534-d3150d7e3.0: - version "5.7.0-nightly-release33-20210421-102534-d3150d7e3.0" - resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-5.7.0-nightly-release33-20210421-102534-d3150d7e3.0.tgz#569d8436cb89996c32d354acf63ade5e91a6140c" - integrity sha512-0duh41MtYAnDaFoQ2JibTzGtmz+c/ydoAECmpdnpo0GVMb96viz7eLnTkMTWMMlwJBYJbuzPtvmkgh/ZWaZRvw== +timeline-state-resolver-types@5.7.0: + version "5.7.0" + resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-5.7.0.tgz#3f481f90bf060a6920a6fd4892fec103b1de8b0c" + integrity sha512-whIgGPLp+ZPeseE8UC8hULi6DWQ/QF8jn/upsYCAYnZnoCj6eW27i+cwDwBjTWO4Y4decIxrQ1LOkDigCg8Wog== dependencies: tslib "^1.13.0" From ed20d965500b3b0e112587412e0cc8a93562ec61 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 26 May 2021 13:43:40 +0200 Subject: [PATCH 06/67] fix: add filePath to ffprobe result --- .../packages/worker/src/worker/accessorHandlers/fileShare.ts | 2 +- shared/packages/worker/src/worker/accessorHandlers/http.ts | 2 +- .../worker/src/worker/accessorHandlers/localFolder.ts | 2 +- .../workers/windowsWorker/expectationHandlers/lib/scan.ts | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 8710adc2..8418abbd 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -268,7 +268,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return undefined // all good } /** Local path to the Package, ie the File */ - private get filePath(): string { + get filePath(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') const filePath = this.accessor.filePath || this.content.filePath diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index d11a93e7..6642ebe8 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -195,7 +195,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericFileAccessorHand return this.accessor.folderPath } /** Local path to the Package, ie the File */ - private get filePath(): string { + get filePath(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') const filePath = this.accessor.filePath || this.content.filePath if (!filePath) throw new Error(`LocalFolderAccessor: filePath not set!`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts index 15688abd..711fbd00 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts @@ -35,13 +35,17 @@ export function scanWithFFProbe( isHTTPAccessorHandle(sourceHandle) ) { let inputPath: string + let filePath: string if (isLocalFolderAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullPath + filePath = sourceHandle.filePath } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() inputPath = sourceHandle.fullPath + filePath = sourceHandle.filePath } else if (isHTTPAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullUrl + filePath = sourceHandle.filePath } else { assertNever(sourceHandle) throw new Error('Unknown handle') @@ -73,6 +77,7 @@ export function scanWithFFProbe( reject(new Error(`File doesn't seem to be a media file`)) return } + json.filePath = filePath resolve(json) }) } else if (isQuantelClipAccessorHandle(sourceHandle)) { From 839cb3ed5ec5eef8b8536aa0abc47b12453c7489 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:21:04 +0200 Subject: [PATCH 07/67] fix: handle robocopy exit code 0 (all is synchronized) --- .../worker/src/worker/workers/windowsWorker/lib/robocopy.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts index b642f886..ff97420c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts @@ -53,7 +53,10 @@ export function roboCopyFile(src: string, dst: string, progress?: (progress: num rbcpy.on('close', (code) => { rbcpy = undefined - if (code && (code & 1) === 1) { + if ( + code === 0 || // No errors occurred, and no copying was done. + (code && (code & 1) === 1) // One or more files were copied successfully (that is, new files have arrived). + ) { // Robocopy's code for succesfully copying files is 1 at LSB: https://ss64.com/nt/robocopy-exit.html if (srcFileName !== dstFileName) { fs.rename(path.join(dstFolder, srcFileName), path.join(dstFolder, dstFileName), (err) => { From c1c3df811867f8be21240776725f71a32846569b Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:25:34 +0200 Subject: [PATCH 08/67] fix: smartbull fixes --- .../packages/generic/src/expectationGenerator.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index d894fdd3..4b17d278 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -119,10 +119,13 @@ export function generateExpectations( } if ( packageWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE && - packageWrap.sources.find((source) => source.containerId === 'source-smartbull') && - (packageWrap as ExpectedPackageWrapMediaFile).expectedPackage.content.filePath.match(/^smartbull/) // the files are on the form "smartbull_TIMESTAMP.mxf/mp4" + packageWrap.sources.find((source) => source.containerId === 'source-smartbull') ) { - smartbullExpectations.push(packageWrap) + if ((packageWrap as ExpectedPackageWrapMediaFile).expectedPackage.content.filePath.match(/^smartbull/)) { + // the files are on the form "smartbull_TIMESTAMP.mxf/mp4" + smartbullExpectations.push(packageWrap) + } + // (any other files in the "source-smartbull"-container are to be ignored) continue } @@ -164,7 +167,7 @@ export function generateExpectations( ...org.expectedPackage, // Take these from bestSmartbull: content: bestSmartbull.expectedPackage.content, - version: bestSmartbull.expectedPackage.version, + version: {}, // Don't even use bestSmartbull.expectedPackage.version, sources: bestSmartbull.expectedPackage.sources, }, } @@ -740,7 +743,7 @@ export function generatePackageContainerExpectations( cronjobs: {}, monitors: { packages: { - targetLayers: [], // not used, since the layers of the original smartbull-package are used + targetLayers: ['source-smartbull'], // not used, since the layers of the original smartbull-package are used }, }, } From 6c0627f910cbd8ba0752723fc89fd77483f8f4f8 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:26:06 +0200 Subject: [PATCH 09/67] fix: swallow http.body errors when fetching header --- shared/packages/worker/src/worker/accessorHandlers/http.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index 6642ebe8..d826c439 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -215,6 +215,10 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + // Swallow the error. Since we're aborting the request, we're not interested in the body anyway. + }) + const headers: HTTPHeaders = { contentType: res.headers.get('content-type'), contentLength: res.headers.get('content-length'), From 73578b5fdb3df77f639b1a117104b629e56d2548 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:26:37 +0200 Subject: [PATCH 10/67] fix: handle when workOnExpectation throws --- shared/packages/worker/src/workerAgent.ts | 49 +++++++++++++---------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index f51d0546..12cdae91 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -172,37 +172,42 @@ export class WorkerAgent { progress: 0, // callbacksOnDone: [], } + const wipId = this.wipI++ this.currentJobs.push(currentjob) - const wipId = this.wipI++ + try { + const workInProgress = await this._worker.workOnExpectation(exp) - const workInProgress = await this._worker.workOnExpectation(exp) + this.worksInProgress[`${wipId}`] = workInProgress - this.worksInProgress[`${wipId}`] = workInProgress + workInProgress.on('progress', (actualVersionHash, progress: number) => { + currentjob.progress = progress + expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch(console.error) + }) + workInProgress.on('error', (error) => { + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - workInProgress.on('progress', (actualVersionHash, progress: number) => { - currentjob.progress = progress - expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch(console.error) - }) - workInProgress.on('error', (error) => { - this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + expectedManager.api.wipEventError(wipId, error).catch(console.error) + delete this.worksInProgress[`${wipId}`] + }) + workInProgress.on('done', (actualVersionHash, reason, result) => { + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - expectedManager.api.wipEventError(wipId, error).catch(console.error) - delete this.worksInProgress[`${wipId}`] - }) - workInProgress.on('done', (actualVersionHash, reason, result) => { - this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch(console.error) + delete this.worksInProgress[`${wipId}`] + }) - expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch(console.error) - delete this.worksInProgress[`${wipId}`] - }) + return { + wipId: wipId, + properties: workInProgress.properties, + } + } catch (err) { + // The workOnExpectation failed. - return { - wipId: wipId, - properties: workInProgress.properties, - } + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - // return workInProgress + throw err + } }, removeExpectation: async (exp: Expectation.Any): Promise => { return this._worker.removeExpectation(exp) From a795b7bd33d82fb699b9867ee07be67dbe51718f Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:28:05 +0200 Subject: [PATCH 11/67] chore: doc & trace --- apps/package-manager/packages/generic/src/packageManager.ts | 4 ++++ shared/packages/api/src/expectationApi.ts | 2 +- shared/packages/expectationManager/src/expectationManager.ts | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index f89f492b..ab57bebc 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -619,6 +619,10 @@ export class PackageManagerHandler { } } + this.logger.info( + `reportMonitoredPackages: ${expectedPackages.length} packages, ${expectedPackagesWraps.length} wraps` + ) + this.monitoredPackages[monitorId] = expectedPackagesWraps this._triggerUpdatedExpectedPackages() diff --git a/shared/packages/api/src/expectationApi.ts b/shared/packages/api/src/expectationApi.ts index 1fa0b08b..00f4138c 100644 --- a/shared/packages/api/src/expectationApi.ts +++ b/shared/packages/api/src/expectationApi.ts @@ -48,7 +48,7 @@ export namespace Expectation { /** Id of the ExpectationManager the expectation was created from */ managerId: string - /** Expectation priority. Lower will be handled first */ + /** Expectation priority. Lower will be handled first. Note: This is not absolute, the actual execution order might vary. */ priority: number /** A list of which expectedPackages that resultet in this expectation */ diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 32e9d408..ca2378aa 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -683,6 +683,8 @@ export class ExpectationManager { if (trackedExp.session.assignedWorker) { const assignedWorker = trackedExp.session.assignedWorker + this.logger.info(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) + // Start working on the Expectation: const wipInfo = await assignedWorker.worker.workOnExpectation(trackedExp.exp, assignedWorker.cost) @@ -814,6 +816,7 @@ export class ExpectationManager { assertNever(trackedExp.state) } } catch (err) { + this.logger.error('Error thrown in evaluateExpectationState') this.logger.error(err) this.updateTrackedExpStatus(trackedExp, undefined, err.toString()) } @@ -1147,7 +1150,6 @@ export class ExpectationManager { trackedPackageContainer.packageContainer ) if (!cronJobStatus.completed) { - console.log(trackedPackageContainer.packageContainer.cronjobs) this.updateTrackedPackageContainerStatus( trackedPackageContainer, 'Cron job not completed: ' + cronJobStatus.reason From 3861dbd3ee367bc82899970e8e2f56074e374c09 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 10:29:10 +0200 Subject: [PATCH 12/67] fix: make expectationManager not get stuck on the same expectation if it fails --- .../expectationManager/src/expectationManager.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index ca2378aa..b3fdb466 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -311,6 +311,7 @@ export class ExpectationManager { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + wip.trackedExp.errorCount++ this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, error) this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { status: wip.trackedExp.state, @@ -383,6 +384,7 @@ export class ExpectationManager { state: TrackedExpectationState.NEW, availableWorkers: [], lastEvaluationTime: 0, + errorCount: 0, reason: '', status: {}, session: null, @@ -398,6 +400,8 @@ export class ExpectationManager { // Removed: for (const id of Object.keys(this.trackedExpectations)) { + this.trackedExpectations[id].errorCount = 0 // Also reset the errorCount, to start fresh. + if (!this.receivedUpdates.expectations[id]) { // This expectation has been removed // TODO: handled removed expectations! @@ -476,6 +480,10 @@ export class ExpectationManager { private getTrackedExpectations(): TrackedExpectation[] { const tracked: TrackedExpectation[] = Object.values(this.trackedExpectations) tracked.sort((a, b) => { + // Lowest errorCount first, this is to make it so that if one expectation fails, it'll not block all the others + if (a.errorCount > b.errorCount) return 1 + if (a.errorCount < b.errorCount) return -1 + // Lowest priority first if (a.exp.priority > b.exp.priority) return 1 if (a.exp.priority < b.exp.priority) return -1 @@ -1226,6 +1234,8 @@ interface TrackedExpectation { availableWorkers: string[] /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number + /** The number of times the expectation has failed */ + errorCount: number /** These statuses are sent from the workers */ status: { From d173af0b4dd34125c2abc0dcca3c66547b0176c3 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 11:29:50 +0200 Subject: [PATCH 13/67] fix: fixes to the file monitor: chokidar reported "unlink" on files that wheren't unlinked.. Also added more monitor options. --- .../generic/src/expectationGenerator.ts | 2 + .../packages/api/src/packageContainerApi.ts | 5 +++ .../accessorHandlers/lib/FileHandler.ts | 40 ++++++++++++++++--- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 4b17d278..37dc499c 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -744,6 +744,8 @@ export function generatePackageContainerExpectations( monitors: { packages: { targetLayers: ['source-smartbull'], // not used, since the layers of the original smartbull-package are used + usePolling: 2000, + awaitWriteFinishStabilityThreshold: 2000, }, }, } diff --git a/shared/packages/api/src/packageContainerApi.ts b/shared/packages/api/src/packageContainerApi.ts index d3194c48..31f25a87 100644 --- a/shared/packages/api/src/packageContainerApi.ts +++ b/shared/packages/api/src/packageContainerApi.ts @@ -23,6 +23,11 @@ export interface PackageContainerExpectation extends PackageContainer { /** If set, ignore any files matching this. (Regular expression). */ ignore?: string + /** If set, the monitoring will be using polling */ + usePolling?: number | null + /** If set, will set the awaitWriteFinish.StabilityThreshold of chokidar */ + awaitWriteFinishStabilityThreshold?: number | null + /** What layers to set on the resulting ExpectedPackage */ targetLayers: string[] diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index 712aa47e..019aaf26 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -132,11 +132,22 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso const options = packageContainerExp.monitors.packages if (!options) throw new Error('Options not set (this should never happen)') - const watcher = chokidar.watch(this.folderPath, { + const chokidarOptions: chokidar.WatchOptions = { ignored: options.ignore ? new RegExp(options.ignore) : undefined, - persistent: true, - }) + } + if (options.usePolling) { + chokidarOptions.usePolling = true + chokidarOptions.interval = 2000 + chokidarOptions.binaryInterval = 2000 + } + if (options.awaitWriteFinishStabilityThreshold) { + chokidarOptions.awaitWriteFinish = { + stabilityThreshold: options.awaitWriteFinishStabilityThreshold, + pollInterval: options.awaitWriteFinishStabilityThreshold, + } + } + const watcher = chokidar.watch(this.folderPath, chokidarOptions) const monitorId = `${this.worker.genericConfig.workerId}_${this.worker.uniqueId}_${Date.now()}` const seenFiles = new Map() @@ -170,6 +181,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso seenFiles.set(filePath, version) } catch (err) { + version = null console.log('error', err) } } @@ -242,13 +254,31 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso triggerSendUpdate() } }) - .on('unlink', (fullPath) => { + .on('change', (fullPath) => { const localPath = getFilePath(fullPath) if (localPath) { - seenFiles.delete(localPath) + seenFiles.set(localPath, null) // This will cause triggerSendUpdate() to update the version triggerSendUpdate() } }) + .on('unlink', (fullPath) => { + // We don't trust chokidar, so we'll check it ourselves first.. + // (We've seen an issue where removing a single file from a folder causes chokidar to emit unlink for ALL the files) + fsAccess(fullPath, fs.constants.R_OK) + .then(() => { + // The file seems to exist, even though chokidar says it doesn't. + // Ignore the event, then + }) + .catch(() => { + // The file truly doesn't exist + + const localPath = getFilePath(fullPath) + if (localPath) { + seenFiles.delete(localPath) + triggerSendUpdate() + } + }) + }) .on('error', (error) => { console.log('error', error) }) From a189114292afc04e5dafda4c019672ee7e02ba97 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 15:21:06 +0200 Subject: [PATCH 14/67] fix: fileCopy to support from fileShare to fileShare --- .../packages/worker/src/worker/accessorHandlers/fileShare.ts | 4 ++++ .../workers/windowsWorker/expectationHandlers/fileCopy.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 8418abbd..97eb68c5 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -267,6 +267,10 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } return undefined // all good } + /** Called when the package is supposed to be in place */ + async packageIsInPlace(): Promise { + await this.clearPackageRemoval(this.filePath) + } /** Local path to the Package, ie the File */ get filePath(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index 7ed57e22..936cfc4e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -172,7 +172,8 @@ export const FileCopy: ExpectationWindowsHandler = { // We can do RoboCopy if (!isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) - if (!isLocalFolderAccessorHandle(targetHandle)) throw new Error(`Source AccessHandler type is wrong`) + if (!isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle)) + throw new Error(`Source AccessHandler type is wrong`) if (sourceHandle.fullPath === targetHandle.fullPath) { throw new Error('Unable to copy: source and Target file paths are the same!') From a8437ce10940ebd94918a85201c29809d2c85c54 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 28 May 2021 15:22:14 +0200 Subject: [PATCH 15/67] chore: add temporary worker logging --- shared/packages/worker/src/workerAgent.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 12cdae91..9bda6a7f 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -173,6 +173,7 @@ export class WorkerAgent { // callbacksOnDone: [], } const wipId = this.wipI++ + console.log(`Worker "${this.id}" starting job ${wipId}, (${exp.id}). (${this.currentJobs.length})`) this.currentJobs.push(currentjob) try { @@ -186,12 +187,18 @@ export class WorkerAgent { }) workInProgress.on('error', (error) => { this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + console.log( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to error. (${this.currentJobs.length})` + ) expectedManager.api.wipEventError(wipId, error).catch(console.error) delete this.worksInProgress[`${wipId}`] }) workInProgress.on('done', (actualVersionHash, reason, result) => { this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + console.log( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), done. (${this.currentJobs.length})` + ) expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch(console.error) delete this.worksInProgress[`${wipId}`] @@ -205,6 +212,9 @@ export class WorkerAgent { // The workOnExpectation failed. this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + console.log( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to initial error. (${this.currentJobs.length})` + ) throw err } From 289330b591cd9b1e2a6bcd20a59e09fd507b487b Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 2 Jun 2021 09:49:07 +0200 Subject: [PATCH 16/67] chore: move internal-tests package to /tests/ --- README.md | 24 ++++++++++--------- lerna.json | 6 ++++- package.json | 3 ++- .../tests => tests}/internal-tests/README.md | 0 .../internal-tests/jest.config.js | 2 +- .../internal-tests/package.json | 2 +- .../src/__mocks__/child_process.ts | 0 .../internal-tests/src/__mocks__/fs.ts | 0 .../tv-automation-quantel-gateway-client.ts | 0 .../src/__mocks__/windows-network-drive.ts | 0 .../src/__tests__/basic.spec.ts | 0 .../src/__tests__/issues.spec.ts | 0 .../src/__tests__/lib/containers.ts | 0 .../src/__tests__/lib/coreMockAPI.ts | 0 .../internal-tests/src/__tests__/lib/lib.ts | 0 .../src/__tests__/lib/setupEnv.ts | 0 .../internal-tests/src/index.ts | 0 .../internal-tests/tsconfig.json | 2 +- 18 files changed, 23 insertions(+), 16 deletions(-) rename {apps/tests => tests}/internal-tests/README.md (100%) rename {apps/tests => tests}/internal-tests/jest.config.js (73%) rename {apps/tests => tests}/internal-tests/package.json (93%) rename {apps/tests => tests}/internal-tests/src/__mocks__/child_process.ts (100%) rename {apps/tests => tests}/internal-tests/src/__mocks__/fs.ts (100%) rename {apps/tests => tests}/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts (100%) rename {apps/tests => tests}/internal-tests/src/__mocks__/windows-network-drive.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/basic.spec.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/issues.spec.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/lib/containers.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/lib/coreMockAPI.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/lib/lib.ts (100%) rename {apps/tests => tests}/internal-tests/src/__tests__/lib/setupEnv.ts (100%) rename {apps/tests => tests}/internal-tests/src/index.ts (100%) rename {apps/tests => tests}/internal-tests/tsconfig.json (60%) diff --git a/README.md b/README.md index ed896df6..89df24ff 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,26 @@ The packages in [shared/packages](shared/packages) are helper libraries, used by The packages in [apps/](apps/) can be run as individual applications. +The packages in [tests/](tests/) contain unit/integration tests. + ### Applications -| Name | Location | Description | -| ----- | -------- | ----------- | -| **Workforce** | [apps/workforce/app](apps/workforce/app) | Mediates connections between the Workers and the Package Managers. _(Later: Will handle spin-up/down of workers according to the current need.)_ | +| Name | Location | Description | +| ------------------- | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Workforce** | [apps/workforce/app](apps/workforce/app) | Mediates connections between the Workers and the Package Managers. _(Later: Will handle spin-up/down of workers according to the current need.)_ | | **Package Manager** | [apps/package-manager/app](apps/package-manager/app) | The Package Manager receives `expectedPackages` from a [Sofie Core](https://github.com/nrkno/tv-automation-server-core), converts them into `Expectations`. Keeps track of work statues and distributes the work to the Workers. | -| **Worker** | [apps/worker/app](apps/worker/app) | Executes work orders from the Package Manager | -| **HTTP-server** | [apps/http-server/app](apps/http-server/app) | A simple HTTP server, where files can be uploaded to and served from. (Often used for thumbnails & previews) | -| **Single-app** | [apps/single-app/app](apps/single-app/app) | Runs one of each of the above in a single application. | +| **Worker** | [apps/worker/app](apps/worker/app) | Executes work orders from the Package Manager | +| **HTTP-server** | [apps/http-server/app](apps/http-server/app) | A simple HTTP server, where files can be uploaded to and served from. (Often used for thumbnails & previews) | +| **Single-app** | [apps/single-app/app](apps/single-app/app) | Runs one of each of the above in a single application. | ### Packages (Libraries) -| Name | Location | Description | -| -- | -- | -- | -| **API** | [shared/packages/api](shared/packages/api) | Various interfaces used by the other libraries | +| Name | Location | Description | +| ---------------------- | ------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| **API** | [shared/packages/api](shared/packages/api) | Various interfaces used by the other libraries | | **ExpectationManager** | [shared/packages/expectationManager](shared/packages/expectationManager) | The ExpectationManager class is used by the Package Manager application | -| **Worker** | [shared/packages/worker](shared/packages/worker) | The Worker class is used by the Worker application | -| **Workforce** | [shared/packages/Workforce](shared/packages/Workforce) | The Workforce class is used by the Worker application | +| **Worker** | [shared/packages/worker](shared/packages/worker) | The Worker class is used by the Worker application | +| **Workforce** | [shared/packages/Workforce](shared/packages/Workforce) | The Workforce class is used by the Worker application | ## For Developers diff --git a/lerna.json b/lerna.json index 0a250d68..18048872 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,9 @@ { - "packages": ["shared/**", "apps/**"], + "packages": [ + "shared/**", + "apps/**", + "tests/**" + ], "version": "independent", "npmClient": "yarn", "useWorkspaces": true diff --git a/package.json b/package.json index b8a8f987..fb85873c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "private": true, "workspaces": [ "shared/**", - "apps/**" + "apps/**", + "tests/**" ], "scripts": { "ci": "yarn install && yarn build && yarn lint && yarn test", diff --git a/apps/tests/internal-tests/README.md b/tests/internal-tests/README.md similarity index 100% rename from apps/tests/internal-tests/README.md rename to tests/internal-tests/README.md diff --git a/apps/tests/internal-tests/jest.config.js b/tests/internal-tests/jest.config.js similarity index 73% rename from apps/tests/internal-tests/jest.config.js rename to tests/internal-tests/jest.config.js index ba142d81..9fe8df7f 100644 --- a/apps/tests/internal-tests/jest.config.js +++ b/tests/internal-tests/jest.config.js @@ -1,4 +1,4 @@ -const base = require('../../../jest.config.base'); +const base = require('../../jest.config.base'); const packageJson = require('./package'); module.exports = { diff --git a/apps/tests/internal-tests/package.json b/tests/internal-tests/package.json similarity index 93% rename from apps/tests/internal-tests/package.json rename to tests/internal-tests/package.json index 3e5f16a4..bf8e050d 100644 --- a/apps/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -1,7 +1,7 @@ { "name": "@tests/internal-tests", "version": "1.0.0", - "description": "Package Manager, http-proxy etc.. all in one application", + "description": "Internal tests", "private": true, "scripts": { "build": "rimraf dist && yarn build:main", diff --git a/apps/tests/internal-tests/src/__mocks__/child_process.ts b/tests/internal-tests/src/__mocks__/child_process.ts similarity index 100% rename from apps/tests/internal-tests/src/__mocks__/child_process.ts rename to tests/internal-tests/src/__mocks__/child_process.ts diff --git a/apps/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts similarity index 100% rename from apps/tests/internal-tests/src/__mocks__/fs.ts rename to tests/internal-tests/src/__mocks__/fs.ts diff --git a/apps/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts b/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts similarity index 100% rename from apps/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts rename to tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts diff --git a/apps/tests/internal-tests/src/__mocks__/windows-network-drive.ts b/tests/internal-tests/src/__mocks__/windows-network-drive.ts similarity index 100% rename from apps/tests/internal-tests/src/__mocks__/windows-network-drive.ts rename to tests/internal-tests/src/__mocks__/windows-network-drive.ts diff --git a/apps/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/basic.spec.ts rename to tests/internal-tests/src/__tests__/basic.spec.ts diff --git a/apps/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/issues.spec.ts rename to tests/internal-tests/src/__tests__/issues.spec.ts diff --git a/apps/tests/internal-tests/src/__tests__/lib/containers.ts b/tests/internal-tests/src/__tests__/lib/containers.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/containers.ts rename to tests/internal-tests/src/__tests__/lib/containers.ts diff --git a/apps/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts b/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts rename to tests/internal-tests/src/__tests__/lib/coreMockAPI.ts diff --git a/apps/tests/internal-tests/src/__tests__/lib/lib.ts b/tests/internal-tests/src/__tests__/lib/lib.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/lib.ts rename to tests/internal-tests/src/__tests__/lib/lib.ts diff --git a/apps/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/setupEnv.ts rename to tests/internal-tests/src/__tests__/lib/setupEnv.ts diff --git a/apps/tests/internal-tests/src/index.ts b/tests/internal-tests/src/index.ts similarity index 100% rename from apps/tests/internal-tests/src/index.ts rename to tests/internal-tests/src/index.ts diff --git a/apps/tests/internal-tests/tsconfig.json b/tests/internal-tests/tsconfig.json similarity index 60% rename from apps/tests/internal-tests/tsconfig.json rename to tests/internal-tests/tsconfig.json index 4e2fdaf5..47857232 100644 --- a/apps/tests/internal-tests/tsconfig.json +++ b/tests/internal-tests/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../../tsconfig.json", + "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "./dist" } From 813042969149d2c9c75006575885c1f3218dcf01 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 2 Jun 2021 11:46:58 +0200 Subject: [PATCH 17/67] fix: bug in workStatus --- apps/package-manager/packages/generic/src/packageManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 8a3ec8fc..cf7b348e 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -393,7 +393,9 @@ export class PackageManagerHandler { statusReason: '', }, // Previous properties: - ...(((this.toReportExpectationStatus[expectationId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in const packageStatus more strict + ...((this.toReportExpectationStatus[expectationId]?.workStatus || + {}) as Partial), // Intentionally cast to Partial<>, to make typings in const workStatus more strict + // Updated porperties: ...expectaction.statusReport, ...statusInfo, From bd8e67f77518228ccacea8a52c58ab521d29c27d Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 3 Jun 2021 10:05:41 +0200 Subject: [PATCH 18/67] feat: implement useTemporaryFilePath, so that copying is done to a temporary file first, then renamed to the final one. --- .../packages/generic/src/configManifest.ts | 5 +++ .../packages/generic/src/coreHandler.ts | 4 ++ .../generic/src/expectationGenerator.ts | 1 + .../packages/generic/src/packageManager.ts | 4 ++ shared/packages/api/src/expectationApi.ts | 14 ++++--- .../accessorHandlers/corePackageInfo.ts | 3 ++ .../src/worker/accessorHandlers/fileShare.ts | 31 +++++++++++++- .../worker/accessorHandlers/genericHandle.ts | 3 ++ .../src/worker/accessorHandlers/http.ts | 4 ++ .../worker/accessorHandlers/localFolder.ts | 34 ++++++++++++++-- .../src/worker/accessorHandlers/quantel.ts | 8 ++-- .../expectationHandlers/fileCopy.ts | 40 ++++++++++--------- .../expectationHandlers/lib/ffmpeg.ts | 1 + .../expectationHandlers/mediaFilePreview.ts | 1 + .../expectationHandlers/mediaFileThumbnail.ts | 1 + .../expectationHandlers/quantelClipCopy.ts | 27 ++++++------- .../expectationHandlers/quantelClipPreview.ts | 1 + .../quantelClipThumbnail.ts | 26 ++++++------ 18 files changed, 149 insertions(+), 59 deletions(-) diff --git a/apps/package-manager/packages/generic/src/configManifest.ts b/apps/package-manager/packages/generic/src/configManifest.ts index b9d93477..cdd4192b 100644 --- a/apps/package-manager/packages/generic/src/configManifest.ts +++ b/apps/package-manager/packages/generic/src/configManifest.ts @@ -12,5 +12,10 @@ export const PACKAGE_MANAGER_DEVICE_CONFIG: DeviceConfigManifest = { name: 'Delay removal of packages (milliseconds)', type: ConfigManifestEntryType.INT, }, + { + id: 'useTemporaryFilePath', + name: 'Use temporary file paths when copying', + type: ConfigManifestEntryType.BOOLEAN, + }, ], } diff --git a/apps/package-manager/packages/generic/src/coreHandler.ts b/apps/package-manager/packages/generic/src/coreHandler.ts index bcacad87..ae6d50ad 100644 --- a/apps/package-manager/packages/generic/src/coreHandler.ts +++ b/apps/package-manager/packages/generic/src/coreHandler.ts @@ -44,6 +44,7 @@ export class CoreHandler { public deviceSettings: { [key: string]: any } = {} public delayRemoval = 0 + public useTemporaryFilePath = false private _deviceOptions: DeviceConfig private _onConnected?: () => any @@ -231,6 +232,9 @@ export class CoreHandler { if (this.deviceSettings['delayRemoval'] !== this.delayRemoval) { this.delayRemoval = this.deviceSettings['delayRemoval'] } + if (this.deviceSettings['useTemporaryFilePath'] !== this.useTemporaryFilePath) { + this.useTemporaryFilePath = this.deviceSettings['useTemporaryFilePath'] + } if (this._packageManagerHandler) { this._packageManagerHandler.onSettingsChanged() diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 0eb01872..6ff76307 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -309,6 +309,7 @@ function generateMediaFileCopy( }, workOptions: { removeDelay: settings.delayRemoval, + useTemporaryFilePath: settings.useTemporaryFilePath, }, } exp.id = hashObj(exp.endRequirement) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index cf7b348e..98b60f3c 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -64,6 +64,7 @@ export class PackageManagerHandler { } = {} settings: PackageManagerSettings = { delayRemoval: 0, + useTemporaryFilePath: false, } constructor( @@ -131,7 +132,9 @@ export class PackageManagerHandler { onSettingsChanged(): void { this.settings = { delayRemoval: this._coreHandler.delayRemoval, + useTemporaryFilePath: this._coreHandler.useTemporaryFilePath, } + this._triggerUpdatedExpectedPackages() } getExpectationManager(): ExpectationManager { return this._expectationManager @@ -713,6 +716,7 @@ export interface ActiveRundown { } export interface PackageManagerSettings { delayRemoval: number + useTemporaryFilePath: boolean } /** Note: This is based on the Core method updateExpectedPackageWorkStatuses. */ diff --git a/shared/packages/api/src/expectationApi.ts b/shared/packages/api/src/expectationApi.ts index 00f4138c..46a12292 100644 --- a/shared/packages/api/src/expectationApi.ts +++ b/shared/packages/api/src/expectationApi.ts @@ -101,7 +101,7 @@ export namespace Expectation { } version: Version.ExpectedFileOnDisk } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Scan of a Media file. A Scan is to be performed on (one of) the sources and the scan result is to be stored on the target. */ export interface PackageScan extends Base { @@ -177,7 +177,7 @@ export namespace Expectation { } version: Version.ExpectedMediaFileThumbnail } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Preview of a Media file. A Preview is to be created from one of the the sources and the resulting file is to be stored on the target. */ export interface MediaFilePreview extends Base { @@ -195,7 +195,7 @@ export namespace Expectation { } version: Version.ExpectedMediaFilePreview } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Quantel clip. A Quantel clip is to be copied from one of the Sources, to the Target. */ @@ -231,7 +231,7 @@ export namespace Expectation { } version: Version.ExpectedQuantelClipThumbnail } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Preview of a Quantel Clip. A Preview is to be created from one of the the sources and the resulting file is to be stored on the target. */ export interface QuantelClipPreview extends Base { @@ -249,7 +249,7 @@ export namespace Expectation { } version: Version.ExpectedQuantelClipPreview } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Contains definitions of specific PackageContainer types, used in the Expectation-definitions */ @@ -284,6 +284,10 @@ export namespace Expectation { /** When removing, wait a duration of time before actually removing it (milliseconds). If not set, package is removed right away. */ removeDelay?: number } + export interface UseTemporaryFilePath { + /** When set, will work on a temporary package first, then move the package to the right place */ + useTemporaryFilePath?: boolean + } } /** Version defines properties to use for determining the version of a Package */ diff --git a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts index 51861c91..3272c823 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts @@ -81,6 +81,9 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand async putPackageInfo(_readInfo: PackageReadInfo): Promise { throw new Error('CorePackageInfo.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + // do nothing + } async fetchMetadata(): Promise { throw new Error('fetchMetadata not applicable for CorePackageInfo') diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 97eb68c5..033796da 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -16,6 +16,8 @@ const fsOpen = promisify(fs.open) const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) +const fsRename = promisify(fs.rename) +const fsUnlink = promisify(fs.unlink) const pExec = promisify(exec) /** Accessor handle for accessing files on a network share */ @@ -31,7 +33,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle onlyContainerAccess?: boolean filePath?: string } - private workOptions: Expectation.WorkOptions.RemoveDelay + private workOptions: Expectation.WorkOptions.RemoveDelay & Expectation.WorkOptions.UseTemporaryFilePath constructor( worker: GenericWorker, @@ -48,8 +50,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') } this.content = content + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') throw new Error('Bad input data: workOptions.removeDelay is not a number!') + if (workOptions.useTemporaryFilePath && typeof workOptions.useTemporaryFilePath !== 'boolean') + throw new Error('Bad input data: workOptions.useTemporaryFilePath is not a boolean!') this.workOptions = workOptions } /** Path to the PackageContainer, ie the folder on the share */ @@ -181,6 +186,19 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle await this.prepareFileAccess() await this.clearPackageRemoval(this.filePath) + const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath + + // Remove the file if it exists: + let exists = false + try { + await fsAccess(fullPath, fs.constants.R_OK) + // The file exists + exists = true + } catch (err) { + // Ignore + } + if (exists) await fsUnlink(fullPath) + const writeStream = sourceStream.pipe(fs.createWriteStream(this.fullPath)) const streamWrapper: PutPackageHandler = new PutPackageHandler(() => { @@ -201,6 +219,12 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle throw new Error('FileShare.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + if (this.workOptions.useTemporaryFilePath) { + await fsRename(this.temporaryFilePath, this.fullPath) + } + } + // Note: We handle metadata by storing a metadata json-file to the side of the file. async fetchMetadata(): Promise { @@ -279,7 +303,10 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle if (!filePath) throw new Error(`FileShareAccessor: filePath not set!`) return filePath } - + /** Full path to a temporary file */ + get temporaryFilePath(): string { + return this.fullPath + '.pmtemp' + } private get metadataPath() { return this.getMetadataPath(this.filePath) } diff --git a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts index 8593093c..5afc7ab2 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts @@ -84,6 +84,9 @@ export abstract class GenericAccessorHandle { /** For accessors that supports readInfo: Pipe info about a package source (obtained from getPackageReadInfo()) */ abstract putPackageInfo(readInfo: PackageReadInfo): Promise + /** Finalize the package. To be called after a .putPackageStream() or putPackageInfo() has completed. */ + abstract finalizePackage(): Promise + /** * Performs a cronjob on the Package container * @returns undefined if all is OK / string with error message diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index d826c439..f3d47737 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -29,6 +29,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { throw new Error('HTTP.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + // do nothing + } async fetchMetadata(): Promise { return this.fetchJSON(this.fullUrl + '_metadata.json') diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index 89efe7bd..6e2609e0 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -14,6 +14,8 @@ const fsOpen = promisify(fs.open) const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) +const fsRename = promisify(fs.rename) +const fsUnlink = promisify(fs.unlink) /** Accessor handle for accessing files in a local folder */ export class LocalFolderAccessorHandle extends GenericFileAccessorHandle { @@ -23,7 +25,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand onlyContainerAccess?: boolean filePath?: string } - private workOptions: Expectation.WorkOptions.RemoveDelay + private workOptions: Expectation.WorkOptions.RemoveDelay & Expectation.WorkOptions.UseTemporaryFilePath constructor( worker: GenericWorker, @@ -39,8 +41,11 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') } this.content = content + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') throw new Error('Bad input data: workOptions.removeDelay is not a number!') + if (workOptions.useTemporaryFilePath && typeof workOptions.useTemporaryFilePath !== 'boolean') + throw new Error('Bad input data: workOptions.useTemporaryFilePath is not a boolean!') this.workOptions = workOptions } static doYouSupportAccess(worker: GenericWorker, accessor0: AccessorOnPackage.Any): boolean { @@ -77,7 +82,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand async checkPackageReadAccess(): Promise { try { await fsAccess(this.fullPath, fs.constants.R_OK) - // The file exists + // The file exists and can be read } catch (err) { // File is not readable return `Not able to access file: ${err.toString()}` @@ -142,7 +147,20 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand async putPackageStream(sourceStream: NodeJS.ReadableStream): Promise { await this.clearPackageRemoval(this.filePath) - const writeStream = sourceStream.pipe(fs.createWriteStream(this.fullPath)) + const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath + + // Remove the file if it exists: + let exists = false + try { + await fsAccess(fullPath, fs.constants.R_OK) + // The file exists + exists = true + } catch (err) { + // Ignore + } + if (exists) await fsUnlink(fullPath) + + const writeStream = sourceStream.pipe(fs.createWriteStream(fullPath)) const streamWrapper: PutPackageHandler = new PutPackageHandler(() => { writeStream.destroy() @@ -162,6 +180,12 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand throw new Error('LocalFolder.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + if (this.workOptions.useTemporaryFilePath) { + await fsRename(this.temporaryFilePath, this.fullPath) + } + } + // Note: We handle metadata by storing a metadata json-file to the side of the file. async fetchMetadata(): Promise { @@ -246,6 +270,10 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand if (!filePath) throw new Error(`LocalFolderAccessor: filePath not set!`) return filePath } + /** Full path to a temporary file */ + get temporaryFilePath(): string { + return this.fullPath + '.pmtemp' + } /** Full path to the metadata file */ private get metadataPath() { return this.fullPath + '_metadata.json' diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index 30a47824..e4625567 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -23,7 +23,7 @@ export class QuantelAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { + // do nothing + } async fetchMetadata(): Promise { throw new Error('Quantel.fetchMetadata: Not supported') diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index 936cfc4e..f228bc44 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -176,7 +176,7 @@ export const FileCopy: ExpectationWindowsHandler = { throw new Error(`Source AccessHandler type is wrong`) if (sourceHandle.fullPath === targetHandle.fullPath) { - throw new Error('Unable to copy: source and Target file paths are the same!') + throw new Error('Unable to copy: Source and Target file paths are the same!') } let wasCancelled = false @@ -195,21 +195,24 @@ export const FileCopy: ExpectationWindowsHandler = { await targetHandle.packageIsInPlace() const sourcePath = sourceHandle.fullPath - const targetPath = targetHandle.fullPath + const targetPath = exp.workOptions.useTemporaryFilePath + ? targetHandle.temporaryFilePath + : targetHandle.fullPath copying = roboCopyFile(sourcePath, targetPath, (progress: number) => { workInProgress._reportProgress(actualSourceVersionHash, progress / 100) }) await copying - // The copy is done + // The copy is done at this point + copying = undefined if (wasCancelled) return // ignore - const duration = Date.now() - startTime - + await targetHandle.finalizePackage() await targetHandle.updateMetadata(actualSourceUVersion) + const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, `Copy completed in ${Math.round(duration / 100) / 10}s`, @@ -283,20 +286,19 @@ export const FileCopy: ExpectationWindowsHandler = { if (wasCancelled) return // ignore setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - - targetHandle - .updateMetadata(actualSourceUVersion) - .then(() => { - workInProgress._reportComplete( - actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await targetHandle.finalizePackage() + await targetHandle.updateMetadata(actualSourceUVersion) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + `Copy completed in ${Math.round(duration / 100) / 10}s`, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) }) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts index 5a730710..4c48e576 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts @@ -102,6 +102,7 @@ export async function runffMpeg( }) writeStream.once('close', () => { uploadIsDone = true + maybeDone() }) } else { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index 240e5ad7..2053149e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -173,6 +173,7 @@ export const MediaFilePreview: ExpectationWindowsHandler = { async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index 739a9aa4..234bd46e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -195,6 +195,7 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { ffMpegProcess = await runffMpeg(workInProgress, args, targetHandle, sourceVersionHash, async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts index 2a48c6d0..ad54c3d5 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts @@ -203,20 +203,19 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { wasCompleted = true setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - - targetHandle - .updateMetadata(actualSourceUVersion) - .then(() => { - workInProgress._reportComplete( - actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await targetHandle.finalizePackage() + await targetHandle.updateMetadata(actualSourceUVersion) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + `Copy completed in ${Math.round(duration / 100) / 10}s`, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index 26570157..9a948f39 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -185,6 +185,7 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index 58f7ad7d..f0a7c9c0 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -176,20 +176,20 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { if (wasCancelled) return // ignore setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - lookupTarget.handle - .updateMetadata(targetMetadata) - .then(() => { - workInProgress._reportComplete( - targetMetadata.sourceVersionHash, - `Thumbnail fetched in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await lookupTarget.handle.finalizePackage() + await lookupTarget.handle.updateMetadata(targetMetadata) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + targetMetadata.sourceVersionHash, + `Thumbnail fetched in ${Math.round(duration / 100) / 10}s`, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) }) From c1f139263b8776308a7937919b02459797d1129f Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 3 Jun 2021 11:55:50 +0200 Subject: [PATCH 19/67] fix: improve how logging is handled --- .eslintrc.js | 3 ++ apps/_boilerplate/app/src/index.ts | 1 + .../packages/generic/src/index.ts | 1 + apps/http-server/app/src/index.ts | 2 ++ apps/package-manager/app/src/index.ts | 3 ++ .../generic/src/expectationGenerator.ts | 13 ++++---- .../app/src/index.ts | 1 + apps/single-app/app/src/index.ts | 1 + apps/worker/app/src/index.ts | 2 ++ apps/workforce/app/src/index.ts | 2 ++ shared/packages/api/src/adapterClient.ts | 8 +++-- shared/packages/api/src/logger.ts | 1 + shared/packages/api/src/websocketClient.ts | 4 ++- .../expectationManager/src/workforceApi.ts | 6 ++-- .../worker/src/expectationManagerApi.ts | 6 ++-- .../accessorHandlers/lib/FileHandler.ts | 6 ++-- .../src/worker/accessorHandlers/quantel.ts | 6 +--- shared/packages/worker/src/worker/worker.ts | 2 ++ .../worker/workers/linuxWorker/linuxWorker.ts | 4 ++- .../workers/windowsWorker/windowsWorker.ts | 4 ++- shared/packages/worker/src/workerAgent.ts | 30 +++++++++++++------ shared/packages/worker/src/workforceApi.ts | 6 ++-- .../src/__mocks__/child_process.ts | 4 ++- tests/internal-tests/src/__mocks__/fs.ts | 1 + .../tv-automation-quantel-gateway-client.ts | 2 ++ .../src/__mocks__/windows-network-drive.ts | 2 ++ 26 files changed, 82 insertions(+), 39 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 28c1e159..82ab5170 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -8,4 +8,7 @@ module.exports = { "**/dist/**/*", "**/__tests__/**/*", ], + "rules": { + "no-console": "warn" + } }; diff --git a/apps/_boilerplate/app/src/index.ts b/apps/_boilerplate/app/src/index.ts index 18a80d0a..dffb2283 100644 --- a/apps/_boilerplate/app/src/index.ts +++ b/apps/_boilerplate/app/src/index.ts @@ -1,3 +1,4 @@ import { startProcess } from '@boilerplate/generic' +/* eslint-disable no-console */ startProcess().catch(console.error) diff --git a/apps/_boilerplate/packages/generic/src/index.ts b/apps/_boilerplate/packages/generic/src/index.ts index 7b7f3be2..9375a134 100644 --- a/apps/_boilerplate/packages/generic/src/index.ts +++ b/apps/_boilerplate/packages/generic/src/index.ts @@ -1,4 +1,5 @@ // boilerplate export async function startProcess(): Promise { + // eslint-disable-next-line no-console console.log('hello world!') } diff --git a/apps/http-server/app/src/index.ts b/apps/http-server/app/src/index.ts index 35f91ae3..34ec591e 100644 --- a/apps/http-server/app/src/index.ts +++ b/apps/http-server/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@http-server/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/package-manager/app/src/index.ts b/apps/package-manager/app/src/index.ts index c5d6b5c6..21120ec8 100644 --- a/apps/package-manager/app/src/index.ts +++ b/apps/package-manager/app/src/index.ts @@ -1,3 +1,6 @@ import { startProcess } from '@package-manager/generic' + +// eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup + startProcess() diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 6ff76307..0b92e0fc 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -6,7 +6,7 @@ import { PackageContainers, PackageManagerSettings, } from './packageManager' -import { Expectation, hashObj, PackageContainerExpectation, literal } from '@shared/api' +import { Expectation, hashObj, PackageContainerExpectation, literal, LoggerInstance } from '@shared/api' export interface ExpectedPackageWrapMediaFile extends ExpectedPackageWrap { expectedPackage: ExpectedPackage.ExpectedPackageMediaFile @@ -30,6 +30,7 @@ type GenerateExpectation = Expectation.Base & { external?: boolean } export function generateExpectations( + logger: LoggerInstance, managerId: string, packageContainers: PackageContainers, _activePlaylist: ActivePlaylist, @@ -87,10 +88,10 @@ export function generateExpectations( if (existingPackage.expectedContentVersionHash !== newPackage.expectedContentVersionHash) { // log warning: - console.log(`WARNING: 2 expectedPackages have the same content, but have different contentVersions!`) - console.log(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) - console.log(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) - console.log(`${JSON.stringify(exp.startRequirement)}`) + logger.warn(`WARNING: 2 expectedPackages have the same content, but have different contentVersions!`) + logger.warn(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) + logger.warn(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) + logger.warn(`${JSON.stringify(exp.startRequirement)}`) // TODO: log better warnings! } else { @@ -176,7 +177,7 @@ export function generateExpectations( prioritizeExpectation(newPackage, exp) addExpectation(newPackage, exp) } - } else console.log('orgSmartbullExpectation is not a MEDIA_FILE') + } else logger.warn('orgSmartbullExpectation is not a MEDIA_FILE') } } diff --git a/apps/quantel-http-transformer-proxy/app/src/index.ts b/apps/quantel-http-transformer-proxy/app/src/index.ts index f9dd7301..c86ac4ee 100644 --- a/apps/quantel-http-transformer-proxy/app/src/index.ts +++ b/apps/quantel-http-transformer-proxy/app/src/index.ts @@ -1,3 +1,4 @@ import { startProcess } from '@quantel-http-transformer-proxy/generic' +/* eslint-disable no-console */ console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/single-app/app/src/index.ts b/apps/single-app/app/src/index.ts index 2f606f77..2f3d488b 100644 --- a/apps/single-app/app/src/index.ts +++ b/apps/single-app/app/src/index.ts @@ -5,6 +5,7 @@ import * as Workforce from '@shared/workforce' import * as Worker from '@shared/worker' import { getSingleAppConfig, setupLogging } from '@shared/api' +// eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup const config = getSingleAppConfig() diff --git a/apps/worker/app/src/index.ts b/apps/worker/app/src/index.ts index 0cff45b5..d018de04 100644 --- a/apps/worker/app/src/index.ts +++ b/apps/worker/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@worker/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/workforce/app/src/index.ts b/apps/workforce/app/src/index.ts index a6f38469..a54aa429 100644 --- a/apps/workforce/app/src/index.ts +++ b/apps/workforce/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@workforce/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/shared/packages/api/src/adapterClient.ts b/shared/packages/api/src/adapterClient.ts index d95ed6b3..fcf977ae 100644 --- a/shared/packages/api/src/adapterClient.ts +++ b/shared/packages/api/src/adapterClient.ts @@ -1,3 +1,4 @@ +import { LoggerInstance } from './logger' import { WebsocketClient } from './websocketClient' import { Hook, MessageBase, MessageIdentifyClient } from './websocketConnection' @@ -14,13 +15,14 @@ export abstract class AdapterClient { throw new Error('.init() must be called first!') } - constructor(private clientType: MessageIdentifyClient['clientType']) {} + constructor(protected logger: LoggerInstance, private clientType: MessageIdentifyClient['clientType']) {} private conn?: WebsocketClient async init(id: string, connectionOptions: ClientConnectionOptions, clientMethods: ME): Promise { if (connectionOptions.type === 'websocket') { const conn = new WebsocketClient( + this.logger, id, connectionOptions.url, this.clientType, @@ -37,10 +39,10 @@ export abstract class AdapterClient { this.conn = conn conn.on('connected', () => { - console.log('Websocket client connected') + this.logger.debug('Websocket client connected') }) conn.on('disconnected', () => { - console.log('Websocket client disconnected') + this.logger.debug('Websocket client disconnected') }) this._sendMessage = ((type: string, ...args: any[]) => conn.send(type, ...args)) as any diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts index 083e131f..9244563a 100644 --- a/shared/packages/api/src/logger.ts +++ b/shared/packages/api/src/logger.ts @@ -49,6 +49,7 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance }) logger.info('Logging to Console') // Hijack console.log: + // eslint-disable-next-line no-console console.log = function (...args: any[]) { // orgConsoleLog('a') if (args.length >= 1) { diff --git a/shared/packages/api/src/websocketClient.ts b/shared/packages/api/src/websocketClient.ts index 90aa9bcc..907e66aa 100644 --- a/shared/packages/api/src/websocketClient.ts +++ b/shared/packages/api/src/websocketClient.ts @@ -1,4 +1,5 @@ import WebSocket from 'ws' +import { LoggerInstance } from './logger' import { MessageBase, MessageIdentifyClient, PING_TIME, WebsocketConnection } from './websocketConnection' /** A Class which */ @@ -8,6 +9,7 @@ export class WebsocketClient extends WebsocketConnection { private closed = false constructor( + private logger: LoggerInstance, private readonly id: string, private readonly url: string, private readonly clientType: MessageIdentifyClient['clientType'], @@ -71,7 +73,7 @@ export class WebsocketClient extends WebsocketConnection { } private reconnect() { this.connect().catch((err) => { - console.error(err) + this.logger.error(err) this.onLostConnection() }) } diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index 9cdd2510..1195e170 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, WorkForceExpectationManager } from '@shared/api' +import { AdapterClient, LoggerInstance, WorkForceExpectationManager } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the ExpectationManager @@ -8,8 +8,8 @@ import { AdapterClient, WorkForceExpectationManager } from '@shared/api' export class WorkforceAPI extends AdapterClient implements WorkForceExpectationManager.WorkForce { - constructor() { - super('expectationManager') + constructor(logger: LoggerInstance) { + super(logger, 'expectationManager') } async registerExpectationManager(managerId: string, url: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts diff --git a/shared/packages/worker/src/expectationManagerApi.ts b/shared/packages/worker/src/expectationManagerApi.ts index fba70133..2a79f65d 100644 --- a/shared/packages/worker/src/expectationManagerApi.ts +++ b/shared/packages/worker/src/expectationManagerApi.ts @@ -1,4 +1,4 @@ -import { ExpectationManagerWorkerAgent, AdapterClient } from '@shared/api' +import { ExpectationManagerWorkerAgent, AdapterClient, LoggerInstance } from '@shared/api' /** * Exposes the API-methods of a ExpectationManager, to be called from the WorkerAgent @@ -8,8 +8,8 @@ import { ExpectationManagerWorkerAgent, AdapterClient } from '@shared/api' export class ExpectationManagerAPI extends AdapterClient implements ExpectationManagerWorkerAgent.ExpectationManager { - constructor() { - super('workerAgent') + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') } async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index 019aaf26..bd050210 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -182,7 +182,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso seenFiles.set(filePath, version) } catch (err) { version = null - console.log('error', err) + this.worker.logger.error(err) } } @@ -237,7 +237,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso triggerSendUpdateIsRunning = false })().catch((err) => { triggerSendUpdateIsRunning = false - console.log('error', err) + this.worker.logger.error(err) }) }, 1000) // Wait just a little bit, to avoid doing multiple updates } @@ -280,7 +280,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso }) }) .on('error', (error) => { - console.log('error', error) + this.worker.logger.error(error.toString()) }) /** Persistant store for Monitors */ diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index e4625567..d3b51f2d 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -300,7 +300,6 @@ export class QuantelAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle console.log(`Quantel.QuantelGateway`, e)) - // @todo: We should be able to emit statuses somehow: - // gateway.monitorServerStatus(() => {}) + gateway.on('error', (e) => this.worker.logger.error(`Quantel.QuantelGateway`, e)) cacheGateways[id] = gateway } diff --git a/shared/packages/worker/src/worker/worker.ts b/shared/packages/worker/src/worker/worker.ts index c2aa14d6..4ee73017 100644 --- a/shared/packages/worker/src/worker/worker.ts +++ b/shared/packages/worker/src/worker/worker.ts @@ -1,6 +1,7 @@ import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -24,6 +25,7 @@ export abstract class GenericWorker { private _uniqueId = 0 constructor( + public logger: LoggerInstance, public readonly genericConfig: WorkerAgentConfig, public readonly location: WorkerLocation, public sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, diff --git a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts index 200701d5..f76671a4 100644 --- a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts +++ b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts @@ -2,6 +2,7 @@ import { IWorkInProgress } from '../../lib/workInProgress' import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -21,11 +22,12 @@ import { GenericWorker, WorkerLocation } from '../../worker' export class LinuxWorker extends GenericWorker { static readonly type = 'linuxWorker' constructor( + logger: LoggerInstance, public readonly config: WorkerAgentConfig, sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, location: WorkerLocation ) { - super(config, location, sendMessageToManager, LinuxWorker.type) + super(logger, config, location, sendMessageToManager, LinuxWorker.type) } async doYouSupportExpectation(_exp: Expectation.Any): Promise { return { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts index cba7b30b..fe093075 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts @@ -1,6 +1,7 @@ import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -36,11 +37,12 @@ export class WindowsWorker extends GenericWorker { public hasFFProbe = false constructor( + logger: LoggerInstance, public readonly config: WorkerAgentConfig, sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, location: WorkerLocation ) { - super(config, location, sendMessageToManager, WindowsWorker.type) + super(logger, config, location, sendMessageToManager, WindowsWorker.type) } async doYouSupportExpectation(exp: Expectation.Any): Promise { try { diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 9bda6a7f..6a428e16 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -51,7 +51,7 @@ export class WorkerAgent { } = {} constructor(private logger: LoggerInstance, private config: WorkerConfig) { - this.workforceAPI = new WorkforceAPI() + this.workforceAPI = new WorkforceAPI(this.logger) this.id = config.worker.workerId this.connectionOptions = this.config.worker.workforceURL @@ -65,6 +65,7 @@ export class WorkerAgent { // Todo: Different types of workers: this._worker = new WindowsWorker( + this.logger, this.config.worker, async (managerId: string, message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { // Forward the message to the expectationManager: @@ -136,7 +137,7 @@ export class WorkerAgent { this.logger.info(`Connecting to Expectation Manager "${id}" at url "${url}"`) const expectedManager = (this.expectationManagers[id] = { url: url, - api: new ExpectationManagerAPI(), + api: new ExpectationManagerAPI(this.logger), }) const methods: ExpectationManagerWorkerAgent.WorkerAgent = literal({ doYouSupportExpectation: async (exp: Expectation.Any): Promise => { @@ -173,7 +174,9 @@ export class WorkerAgent { // callbacksOnDone: [], } const wipId = this.wipI++ - console.log(`Worker "${this.id}" starting job ${wipId}, (${exp.id}). (${this.currentJobs.length})`) + this.logger.debug( + `Worker "${this.id}" starting job ${wipId}, (${exp.id}). (${this.currentJobs.length})` + ) this.currentJobs.push(currentjob) try { @@ -183,24 +186,33 @@ export class WorkerAgent { workInProgress.on('progress', (actualVersionHash, progress: number) => { currentjob.progress = progress - expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch(console.error) + expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch((err) => { + this.logger.error('Error in wipEventProgress') + this.logger.error(err) + }) }) workInProgress.on('error', (error) => { this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - console.log( + this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to error. (${this.currentJobs.length})` ) - expectedManager.api.wipEventError(wipId, error).catch(console.error) + expectedManager.api.wipEventError(wipId, error).catch((err) => { + this.logger.error('Error in wipEventError') + this.logger.error(err) + }) delete this.worksInProgress[`${wipId}`] }) workInProgress.on('done', (actualVersionHash, reason, result) => { this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - console.log( + this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), done. (${this.currentJobs.length})` ) - expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch(console.error) + expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch((err) => { + this.logger.error('Error in wipEventDone') + this.logger.error(err) + }) delete this.worksInProgress[`${wipId}`] }) @@ -212,7 +224,7 @@ export class WorkerAgent { // The workOnExpectation failed. this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) - console.log( + this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to initial error. (${this.currentJobs.length})` ) diff --git a/shared/packages/worker/src/workforceApi.ts b/shared/packages/worker/src/workforceApi.ts index d7bd28c4..0591178d 100644 --- a/shared/packages/worker/src/workforceApi.ts +++ b/shared/packages/worker/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, WorkForceWorkerAgent } from '@shared/api' +import { AdapterClient, LoggerInstance, WorkForceWorkerAgent } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the WorkerAgent @@ -8,8 +8,8 @@ import { AdapterClient, WorkForceWorkerAgent } from '@shared/api' export class WorkforceAPI extends AdapterClient implements WorkForceWorkerAgent.WorkForce { - constructor() { - super('workerAgent') + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') } async getExpectationManagerList(): Promise<{ id: string; url: string }[]> { return await this._sendMessage('getExpectationManagerList', undefined) diff --git a/tests/internal-tests/src/__mocks__/child_process.ts b/tests/internal-tests/src/__mocks__/child_process.ts index 9bcfcd67..eec8fe6e 100644 --- a/tests/internal-tests/src/__mocks__/child_process.ts +++ b/tests/internal-tests/src/__mocks__/child_process.ts @@ -3,11 +3,13 @@ import EventEmitter from 'events' import { promisify } from 'util' import path from 'path' +/* eslint-disable no-console */ + const fsCopyFile = promisify(fs.copyFile) // @ts-expect-error mock const fs__mockSetDirectory = fs.__mockSetDirectory -const child_process = jest.createMockFromModule('child_process') as any +const child_process: any = jest.createMockFromModule('child_process') function exec(commandString: string) { if (commandString.match(/^wmic /)) { diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index 45518fe4..dca213c3 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -1,6 +1,7 @@ // eslint-disable-next-line node/no-unpublished-import import fsMockType from 'windows-network-drive' // Note: this is a mocked module +/* eslint-disable no-console */ const DEBUG_LOG = false enum fsConstants { diff --git a/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts b/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts index 3a419263..7b24f27f 100644 --- a/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts +++ b/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts @@ -2,6 +2,8 @@ import EventEmitter from 'events' // eslint-disable-next-line node/no-unpublished-import import { Q, ClipSearchQuery } from 'tv-automation-quantel-gateway-client' // note: this is a mocked module +/* eslint-disable no-console */ + const client = jest.createMockFromModule('tv-automation-quantel-gateway-client') as any const DEBUG_LOG = false diff --git a/tests/internal-tests/src/__mocks__/windows-network-drive.ts b/tests/internal-tests/src/__mocks__/windows-network-drive.ts index 4ceb41de..d6384d7d 100644 --- a/tests/internal-tests/src/__mocks__/windows-network-drive.ts +++ b/tests/internal-tests/src/__mocks__/windows-network-drive.ts @@ -1,5 +1,7 @@ const wnd = jest.createMockFromModule('windows-network-drive') as any +/* eslint-disable no-console */ + const DEBUG_LOG = false const mountedDrives: { [key: string]: string } = {} From 1811204be08706f538aea8e2e0434269793feff4 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 3 Jun 2021 12:01:48 +0200 Subject: [PATCH 20/67] chore: refactor how callbacks from ExpectationManager are defined and handled in the PackageManagerHandler class --- .../packages/generic/src/packageManager.ts | 531 +++++++++--------- .../src/expectationManager.ts | 94 ++-- .../src/__tests__/lib/setupEnv.ts | 20 +- 3 files changed, 319 insertions(+), 326 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 98b60f3c..cfd7f951 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -10,7 +10,11 @@ import { PackageContainerOnPackage, } from '@sofie-automation/blueprints-integration' import { generateExpectations, generatePackageContainerExpectations } from './expectationGenerator' -import { ExpectationManager, ExpectationManagerServerOptions } from '@shared/expectation-manager' +import { + ExpectationManager, + ExpectationManagerCallbacks, + ExpectationManagerServerOptions, +} from '@shared/expectation-manager' import { ClientConnectionOptions, Expectation, @@ -24,48 +28,27 @@ import deepExtend from 'deep-extend' import clone = require('fast-clone') export class PackageManagerHandler { - private _coreHandler!: CoreHandler + public coreHandler!: CoreHandler private _observers: Array = [] - private _expectationManager: ExpectationManager + public expectationManager: ExpectationManager private expectedPackageCache: { [id: string]: ExpectedPackageWrap } = {} - private packageContainersCache: PackageContainers = {} - - private reportedWorkStatuses: { [id: string]: ExpectedPackageStatusAPI.WorkStatus } = {} - private toReportExpectationStatus: { - [id: string]: { - workStatus: ExpectedPackageStatusAPI.WorkStatus | null - /** If the status is new and needs to be reported to Core */ - isUpdated: boolean - } - } = {} - private sendUpdateExpectationStatusTimeouts: NodeJS.Timeout | undefined - - private reportedPackageStatuses: { [id: string]: ExpectedPackageStatusAPI.PackageContainerPackageStatus } = {} - private toReportPackageStatus: { - [key: string]: { - containerId: string - packageId: string - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - /** If the status is new and needs to be reported to Core */ - isUpdated: boolean - } - } = {} - private sendUpdatePackageContainerPackageStatusTimeouts: NodeJS.Timeout | undefined + public packageContainersCache: PackageContainers = {} private externalData: { packageContainers: PackageContainers; expectedPackages: ExpectedPackageWrap[] } = { packageContainers: {}, expectedPackages: [], } private _triggerUpdatedExpectedPackagesTimeout: NodeJS.Timeout | null = null - private monitoredPackages: { + public monitoredPackages: { [monitorId: string]: ResultingExpectedPackage[] } = {} settings: PackageManagerSettings = { delayRemoval: 0, useTemporaryFilePath: false, } + callbacksHandler: ExpectationManagerCallbacksHandler constructor( public logger: LoggerInstance, @@ -74,42 +57,22 @@ export class PackageManagerHandler { private serverAccessUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions ) { - this._expectationManager = new ExpectationManager( + this.callbacksHandler = new ExpectationManagerCallbacksHandler(this) + + this.expectationManager = new ExpectationManager( this.logger, this.managerId, this.serverOptions, this.serverAccessUrl, this.workForceConnectionOptions, - ( - expectationId: string, - expectaction: Expectation.Any | null, - actualVersionHash: string | null, - statusInfo: { - status?: string - progress?: number - statusReason?: string - } - ) => this.updateExpectationStatus(expectationId, expectaction, actualVersionHash, statusInfo), - ( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - ) => this.updatePackageContainerPackageStatus(containerId, packageId, packageStatus), - ( - containerId: string, - packageContainer: PackageContainerExpectation | null, - statusInfo: { - statusReason?: string - } - ) => this.updatePackageContainerStatus(containerId, packageContainer, statusInfo), - (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => this.onMessageFromWorker(message) + this.callbacksHandler ) } async init(_config: PackageManagerConfig, coreHandler: CoreHandler): Promise { - this._coreHandler = coreHandler + this.coreHandler = coreHandler - this._coreHandler.setPackageManagerHandler(this) + this.coreHandler.setPackageManagerHandler(this) this.logger.info('PackageManagerHandler init') @@ -122,101 +85,29 @@ export class PackageManagerHandler { }) this.setupObservers() this.onSettingsChanged() - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() - await this.cleanReportedExpectations() - await this._expectationManager.init() + await this.callbacksHandler.cleanReportedExpectations() + await this.expectationManager.init() this.logger.info('PackageManagerHandler initialized') } onSettingsChanged(): void { this.settings = { - delayRemoval: this._coreHandler.delayRemoval, - useTemporaryFilePath: this._coreHandler.useTemporaryFilePath, + delayRemoval: this.coreHandler.delayRemoval, + useTemporaryFilePath: this.coreHandler.useTemporaryFilePath, } - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } getExpectationManager(): ExpectationManager { - return this._expectationManager + return this.expectationManager } - private wrapExpectedPackage( - packageContainers: PackageContainers, - expectedPackage: ExpectedPackage.Any - ): ExpectedPackageWrap | undefined { - const combinedSources: PackageContainerOnPackage[] = [] - for (const packageSource of expectedPackage.sources) { - const lookedUpSource: PackageContainer = packageContainers[packageSource.containerId] - if (lookedUpSource) { - // We're going to combine the accessor attributes set on the Package with the ones defined on the source: - const combinedSource: PackageContainerOnPackage = { - ...omit(clone(lookedUpSource), 'accessors'), - accessors: {}, - containerId: packageSource.containerId, - } - - const accessorIds = _.uniq( - Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors)) - ) - - for (const accessorId of accessorIds) { - const sourceAccessor = lookedUpSource.accessors[accessorId] as Accessor.Any | undefined - - const packageAccessor = packageSource.accessors[accessorId] as AccessorOnPackage.Any | undefined - - if (packageAccessor && sourceAccessor && packageAccessor.type === sourceAccessor.type) { - combinedSource.accessors[accessorId] = deepExtend({}, sourceAccessor, packageAccessor) - } else if (packageAccessor) { - combinedSource.accessors[accessorId] = clone(packageAccessor) - } else if (sourceAccessor) { - combinedSource.accessors[accessorId] = clone( - sourceAccessor - ) as AccessorOnPackage.Any - } - } - combinedSources.push(combinedSource) - } - } - // Lookup Package targets: - const combinedTargets: PackageContainerOnPackage[] = [] - - for (const layer of expectedPackage.layers) { - // Hack: we use the layer name as a 1-to-1 relation to a target containerId - const packageContainerId: string = layer - - if (packageContainerId) { - const lookedUpTarget = packageContainers[packageContainerId] - if (lookedUpTarget) { - // Todo: should the be any combination of properties here? - combinedTargets.push({ - ...omit(clone(lookedUpTarget), 'accessors'), - accessors: lookedUpTarget.accessors as { - [accessorId: string]: AccessorOnPackage.Any - }, - containerId: packageContainerId, - }) - } - } - } - if (combinedSources.length) { - if (combinedTargets.length) { - return { - expectedPackage: expectedPackage, - priority: 999, // lowest priority - sources: combinedSources, - targets: combinedTargets, - playoutDeviceId: '', - external: true, - } - } - } - return undefined - } setExternalData(packageContainers: PackageContainers, expectedPackages: ExpectedPackage.Any[]): void { const expectedPackagesWraps: ExpectedPackageWrap[] = [] for (const expectedPackage of expectedPackages) { - const wrap = this.wrapExpectedPackage(packageContainers, expectedPackage) + const wrap = wrapExpectedPackage(packageContainers, expectedPackage) if (wrap) { expectedPackagesWraps.push(wrap) } @@ -226,7 +117,7 @@ export class PackageManagerHandler { packageContainers: packageContainers, expectedPackages: expectedPackagesWraps, } - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } private setupObservers(): void { if (this._observers.length) { @@ -238,19 +129,19 @@ export class PackageManagerHandler { } this.logger.info('Renewing observers') - const expectedPackagesObserver = this._coreHandler.core.observe('deviceExpectedPackages') + const expectedPackagesObserver = this.coreHandler.core.observe('deviceExpectedPackages') expectedPackagesObserver.added = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } expectedPackagesObserver.changed = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } expectedPackagesObserver.removed = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } this._observers.push(expectedPackagesObserver) } - private _triggerUpdatedExpectedPackages() { + public triggerUpdatedExpectedPackages(): void { this.logger.info('_triggerUpdatedExpectedPackages') if (this._triggerUpdatedExpectedPackagesTimeout) { @@ -265,7 +156,7 @@ export class PackageManagerHandler { const expectedPackages: ExpectedPackageWrap[] = [] const packageContainers: PackageContainers = {} - const objs = this._coreHandler.core.getCollection('deviceExpectedPackages').find(() => true) + const objs = this.coreHandler.core.getCollection('deviceExpectedPackages').find(() => true) const activePlaylistObj = objs.find((o) => o.type === 'active_playlist') if (!activePlaylistObj) { @@ -349,7 +240,8 @@ export class PackageManagerHandler { // Step 1: Generate expectations: const expectations = generateExpectations( - this._expectationManager.managerId, + this.logger, + this.expectationManager.managerId, this.packageContainersCache, activePlaylist, activeRundowns, @@ -358,7 +250,7 @@ export class PackageManagerHandler { ) this.logger.info(`Has ${Object.keys(expectations).length} expectations`) const packageContainerExpectations = generatePackageContainerExpectations( - this._expectationManager.managerId, + this.expectationManager.managerId, this.packageContainersCache, activePlaylist ) @@ -367,11 +259,92 @@ export class PackageManagerHandler { // this.logger.info(JSON.stringify(expectations, null, 2)) // Step 2: Track and handle new expectations: - this._expectationManager.updatePackageContainerExpectations(packageContainerExpectations) + this.expectationManager.updatePackageContainerExpectations(packageContainerExpectations) - this._expectationManager.updateExpectations(expectations) + this.expectationManager.updateExpectations(expectations) + } + public restartExpectation(workId: string): void { + // This method can be called from core + this.expectationManager.restartExpectation(workId) + } + public restartAllExpectations(): void { + // This method can be called from core + this.expectationManager.restartAllExpectations() + } + public abortExpectation(workId: string): void { + // This method can be called from core + this.expectationManager.abortExpectation(workId) } - public updateExpectationStatus( + + /** Ensures that the packageContainerExpectations containes the mandatory expectations */ + private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { + [id: string]: PackageContainerExpectation + }): void { + for (const [containerId, packageContainer] of Object.entries(this.packageContainersCache)) { + /** Is the Container writeable */ + let isWriteable = false + for (const accessor of Object.values(packageContainer.accessors)) { + if (accessor.allowWrite) { + isWriteable = true + break + } + } + // All writeable packageContainers should have the clean up cronjob: + if (isWriteable) { + if (!packageContainerExpectations[containerId]) { + // todo: Maybe should not all package-managers monitor, + // this should perhaps be coordinated with the Workforce-manager, who should monitor who? + + // Add default packageContainerExpectation: + packageContainerExpectations[containerId] = literal({ + ...packageContainer, + id: containerId, + managerId: this.expectationManager.managerId, + cronjobs: { + interval: 0, + }, + monitors: {}, + }) + } + packageContainerExpectations[containerId].cronjobs.cleanup = {} // Add cronjob to clean up + } + } + } +} +export function omit(obj: T, ...props: P[]): Omit { + return _.omit(obj, ...(props as string[])) as any +} + +/** This class handles data and requests that comes from ExpectationManager. */ +class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks { + reportedWorkStatuses: { [id: string]: ExpectedPackageStatusAPI.WorkStatus } = {} + toReportExpectationStatus: { + [id: string]: { + workStatus: ExpectedPackageStatusAPI.WorkStatus | null + /** If the status is new and needs to be reported to Core */ + isUpdated: boolean + } + } = {} + sendUpdateExpectationStatusTimeouts: NodeJS.Timeout | undefined + + reportedPackageStatuses: { [id: string]: ExpectedPackageStatusAPI.PackageContainerPackageStatus } = {} + toReportPackageStatus: { + [key: string]: { + containerId: string + packageId: string + packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + /** If the status is new and needs to be reported to Core */ + isUpdated: boolean + } + } = {} + sendUpdatePackageContainerPackageStatusTimeouts: NodeJS.Timeout | undefined + + logger: LoggerInstance + constructor(private packageManager: PackageManagerHandler) { + this.logger = this.packageManager.logger + } + + public reportExpectationStatus( expectationId: string, expectaction: Expectation.Any | null, actualVersionHash: string | null, @@ -418,6 +391,75 @@ export class PackageManagerHandler { this.triggerSendUpdateExpectationStatus(expectationId, workStatus) } } + public reportPackageContainerPackageStatus( + containerId: string, + packageId: string, + packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + ): void { + const packageContainerPackageId = `${containerId}_${packageId}` + if (!packageStatus) { + this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, null) + } else { + const o: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { + // Default properties: + ...{ + status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, + progress: 0, + statusReason: '', + }, + // pre-existing properties: + ...(((this.toReportPackageStatus[packageContainerPackageId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in the outer spread-assignment more strict + // Updated properties: + ...packageStatus, + } + + this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, o) + } + } + public reportPackageContainerExpectationStatus( + containerId: string, + _packageContainer: PackageContainerExpectation | null, + statusInfo: { statusReason?: string | undefined } + ): void { + // This is not (yet) reported to Core. + // ...to be implemented... + this.logger.info(`PackageContainerStatus "${containerId}"`) + this.logger.info(statusInfo.statusReason || '>No reason<') + } + public async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { + switch (message.type) { + case 'fetchPackageInfoMetadata': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.fetchPackageInfoMetadata, + message.arguments + ) + case 'updatePackageInfo': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.updatePackageInfo, + message.arguments + ) + case 'removePackageInfo': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.removePackageInfo, + message.arguments + ) + case 'reportFromMonitorPackages': + this.reportMonitoredPackages(...message.arguments) + break + + default: + // @ts-expect-error message is never + throw new Error(`Unsupported message type "${message.type}"`) + } + } + public async cleanReportedExpectations() { + // Clean out all reported statuses, this is an easy way to sync a clean state with core + this.reportedWorkStatuses = {} + await this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.removeAllExpectedPackageWorkStatusOfDevice, + [] + ) + } private triggerSendUpdateExpectationStatus( expectationId: string, workStatus: ExpectedPackageStatusAPI.WorkStatus | null @@ -482,7 +524,7 @@ export class PackageManagerHandler { } if (changesTosend.length) { - this._coreHandler.core + this.packageManager.coreHandler.core .callMethod(PeripheralDeviceAPI.methods.updateExpectedPackageWorkStatuses, [changesTosend]) .catch((err) => { this.logger.error('Error when calling method updateExpectedPackageWorkStatuses:') @@ -490,39 +532,6 @@ export class PackageManagerHandler { }) } } - private async cleanReportedExpectations() { - // Clean out all reported statuses, this is an easy way to sync a clean state with core - this.reportedWorkStatuses = {} - await this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.removeAllExpectedPackageWorkStatusOfDevice, - [] - ) - } - public updatePackageContainerPackageStatus( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - ): void { - const packageContainerPackageId = `${containerId}_${packageId}` - if (!packageStatus) { - this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, null) - } else { - const o: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { - // Default properties: - ...{ - status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, - progress: 0, - statusReason: '', - }, - // pre-existing properties: - ...(((this.toReportPackageStatus[packageContainerPackageId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in the outer spread-assignment more strict - // Updated properties: - ...packageStatus, - } - - this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, o) - } - } private triggerSendUpdatePackageContainerPackageStatus( containerId: string, packageId: string, @@ -544,7 +553,7 @@ export class PackageManagerHandler { }, 300) } } - public sendUpdatePackageContainerPackageStatus(): void { + private sendUpdatePackageContainerPackageStatus(): void { const changesTosend: UpdatePackageContainerPackageStatusesChanges = [] for (const [key, o] of Object.entries(this.toReportPackageStatus)) { @@ -575,7 +584,7 @@ export class PackageManagerHandler { } if (changesTosend.length) { - this._coreHandler.core + this.packageManager.coreHandler.core .callMethod(PeripheralDeviceAPI.methods.updatePackageContainerPackageStatuses, [changesTosend]) .catch((err) => { this.logger.error('Error when calling method updatePackageContainerPackageStatuses:') @@ -583,93 +592,11 @@ export class PackageManagerHandler { }) } } - public updatePackageContainerStatus( - containerId: string, - _packageContainer: PackageContainerExpectation | null, - statusInfo: { statusReason?: string | undefined } - ): void { - this.logger.info(`PackageContainerStatus "${containerId}"`) - this.logger.info(statusInfo.statusReason || '>No reason<') - } - private async onMessageFromWorker( - message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any - ): Promise { - switch (message.type) { - case 'fetchPackageInfoMetadata': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.fetchPackageInfoMetadata, - message.arguments - ) - case 'updatePackageInfo': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.updatePackageInfo, - message.arguments - ) - case 'removePackageInfo': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.removePackageInfo, - message.arguments - ) - case 'reportFromMonitorPackages': - this.reportMonitoredPackages(...message.arguments) - break - - default: - // @ts-expect-error message is never - throw new Error(`Unsupported message type "${message.type}"`) - } - } - public restartExpectation(workId: string): void { - // This method can be called from core - this._expectationManager.restartExpectation(workId) - } - public restartAllExpectations(): void { - // This method can be called from core - this._expectationManager.restartAllExpectations() - } - public abortExpectation(workId: string): void { - // This method can be called from core - this._expectationManager.abortExpectation(workId) - } - /** Ensures that the packageContainerExpectations containes the mandatory expectations */ - private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { - [id: string]: PackageContainerExpectation - }): void { - for (const [containerId, packageContainer] of Object.entries(this.packageContainersCache)) { - /** Is the Container writeable */ - let isWriteable = false - for (const accessor of Object.values(packageContainer.accessors)) { - if (accessor.allowWrite) { - isWriteable = true - break - } - } - // All writeable packageContainers should have the clean up cronjob: - if (isWriteable) { - if (!packageContainerExpectations[containerId]) { - // todo: Maybe should not all package-managers monitor, - // this should perhaps be coordinated with the Workforce-manager, who should monitor who? - - // Add default packageContainerExpectation: - packageContainerExpectations[containerId] = literal({ - ...packageContainer, - id: containerId, - managerId: this._expectationManager.managerId, - cronjobs: { - interval: 0, - }, - monitors: {}, - }) - } - packageContainerExpectations[containerId].cronjobs.cleanup = {} // Add cronjob to clean up - } - } - } private reportMonitoredPackages(_containerId: string, monitorId: string, expectedPackages: ExpectedPackage.Any[]) { const expectedPackagesWraps: ExpectedPackageWrap[] = [] for (const expectedPackage of expectedPackages) { - const wrap = this.wrapExpectedPackage(this.packageContainersCache, expectedPackage) + const wrap = wrapExpectedPackage(this.packageManager.packageContainersCache, expectedPackage) if (wrap) { expectedPackagesWraps.push(wrap) } @@ -679,13 +606,81 @@ export class PackageManagerHandler { `reportMonitoredPackages: ${expectedPackages.length} packages, ${expectedPackagesWraps.length} wraps` ) - this.monitoredPackages[monitorId] = expectedPackagesWraps + this.packageManager.monitoredPackages[monitorId] = expectedPackagesWraps - this._triggerUpdatedExpectedPackages() + this.packageManager.triggerUpdatedExpectedPackages() } } -export function omit(obj: T, ...props: P[]): Omit { - return _.omit(obj, ...(props as string[])) as any +function wrapExpectedPackage( + packageContainers: PackageContainers, + expectedPackage: ExpectedPackage.Any +): ExpectedPackageWrap | undefined { + const combinedSources: PackageContainerOnPackage[] = [] + for (const packageSource of expectedPackage.sources) { + const lookedUpSource: PackageContainer = packageContainers[packageSource.containerId] + if (lookedUpSource) { + // We're going to combine the accessor attributes set on the Package with the ones defined on the source: + const combinedSource: PackageContainerOnPackage = { + ...omit(clone(lookedUpSource), 'accessors'), + accessors: {}, + containerId: packageSource.containerId, + } + + const accessorIds = _.uniq( + Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors)) + ) + + for (const accessorId of accessorIds) { + const sourceAccessor = lookedUpSource.accessors[accessorId] as Accessor.Any | undefined + + const packageAccessor = packageSource.accessors[accessorId] as AccessorOnPackage.Any | undefined + + if (packageAccessor && sourceAccessor && packageAccessor.type === sourceAccessor.type) { + combinedSource.accessors[accessorId] = deepExtend({}, sourceAccessor, packageAccessor) + } else if (packageAccessor) { + combinedSource.accessors[accessorId] = clone(packageAccessor) + } else if (sourceAccessor) { + combinedSource.accessors[accessorId] = clone(sourceAccessor) as AccessorOnPackage.Any + } + } + combinedSources.push(combinedSource) + } + } + // Lookup Package targets: + const combinedTargets: PackageContainerOnPackage[] = [] + + for (const layer of expectedPackage.layers) { + // Hack: we use the layer name as a 1-to-1 relation to a target containerId + const packageContainerId: string = layer + + if (packageContainerId) { + const lookedUpTarget = packageContainers[packageContainerId] + if (lookedUpTarget) { + // Todo: should the be any combination of properties here? + combinedTargets.push({ + ...omit(clone(lookedUpTarget), 'accessors'), + accessors: lookedUpTarget.accessors as { + [accessorId: string]: AccessorOnPackage.Any + }, + containerId: packageContainerId, + }) + } + } + } + + if (combinedSources.length) { + if (combinedTargets.length) { + return { + expectedPackage: expectedPackage, + priority: 999, // lowest priority + sources: combinedSources, + targets: combinedTargets, + playoutDeviceId: '', + external: true, + } + } + } + return undefined } interface ResultingExpectedPackage { diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index b3fdb466..25472b47 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -33,7 +33,7 @@ export class ExpectationManager { */ private readonly ALLOW_SKIPPING_QUEUE_TIME = 30 * 1000 // ms - private workforceAPI = new WorkforceAPI() + private workforceAPI: WorkforceAPI /** Store for various incoming data, to be processed on next iteration round */ private receivedUpdates: { @@ -98,11 +98,9 @@ export class ExpectationManager { /** At what url the ExpectationManager can be reached on */ private serverAccessUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, - private reportExpectationStatus: ReportExpectationStatus, - private reportPackageContainerPackageStatus: ReportPackageContainerPackageStatus, - private reportPackageContainerExpectationStatus: ReportPackageContainerExpectationStatus, - private onMessageFromWorker: MessageFromWorker + private callbacks: ExpectationManagerCallbacks ) { + this.workforceAPI = new WorkforceAPI(this.logger) if (this.serverOptions.type === 'websocket') { this.logger.info(`Expectation Manager on port ${this.serverOptions.port}`) this.websocketServer = new WebsocketServer(this.serverOptions.port, (client: ClientConnection) => { @@ -251,7 +249,7 @@ export class ExpectationManager { messageFromWorker: async ( message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any ): Promise => { - return this.onMessageFromWorker(message) + return this.callbacks.messageFromWorker(message) }, wipEventProgress: async ( @@ -271,9 +269,14 @@ export class ExpectationManager { )}" progress: ${progress}` ) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, actualVersionHash, { - progress: progress, - }) + this.callbacks.reportExpectationStatus( + wip.trackedExp.id, + wip.trackedExp.exp, + actualVersionHash, + { + progress: progress, + } + ) } else { // ignore } @@ -290,11 +293,16 @@ export class ExpectationManager { if (wip.trackedExp.state === TrackedExpectationState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.FULFILLED, reason) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, actualVersionHash, { - status: wip.trackedExp.state, - statusReason: wip.trackedExp.reason, - progress: 1, - }) + this.callbacks.reportExpectationStatus( + wip.trackedExp.id, + wip.trackedExp.exp, + actualVersionHash, + { + status: wip.trackedExp.state, + statusReason: wip.trackedExp.reason, + progress: 1, + } + ) if (this.handleTriggerByFullfilledIds(wip.trackedExp)) { // Something was triggered, run again asap. @@ -313,7 +321,7 @@ export class ExpectationManager { if (wip.trackedExp.state === TrackedExpectationState.WORKING) { wip.trackedExp.errorCount++ this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, error) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { + this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { status: wip.trackedExp.state, statusReason: wip.trackedExp.reason, }) @@ -767,7 +775,7 @@ export class ExpectationManager { if (removed.removed) { trackedExp.session.expectationCanBeRemoved = true - this.reportExpectationStatus(trackedExp.id, null, null, {}) + this.callbacks.reportExpectationStatus(trackedExp.id, null, null, {}) } else { this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.REMOVED, removed.reason) } @@ -880,7 +888,7 @@ export class ExpectationManager { } if (updatedState || updatedReason) { - this.reportExpectationStatus(trackedExp.id, trackedExp.exp, null, { + this.callbacks.reportExpectationStatus(trackedExp.id, trackedExp.exp, null, { status: trackedExp.state, statusReason: trackedExp.reason, }) @@ -894,7 +902,7 @@ export class ExpectationManager { for (const fromPackage of trackedExp.exp.fromPackages) { // TODO: this is probably not eh right thing to do: for (const packageContainer of trackedExp.exp.endRequirement.targets) { - this.reportPackageContainerPackageStatus(packageContainer.containerId, fromPackage.id, { + this.callbacks.reportPackageContainerPackageStatus(packageContainer.containerId, fromPackage.id, { contentVersionHash: trackedExp.status.actualVersionHash || '', progress: trackedExp.status.workProgress || 0, status: this.getPackageStatus(trackedExp), @@ -1187,7 +1195,7 @@ export class ExpectationManager { } if (updatedReason) { - this.reportPackageContainerExpectationStatus( + this.callbacks.reportPackageContainerExpectationStatus( trackedPackageContainer.id, trackedPackageContainer.packageContainer, { @@ -1275,28 +1283,32 @@ interface WorkerAgentAssignment { } export type MessageFromWorker = (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => Promise -export type ReportExpectationStatus = ( - expectationId: string, - expectaction: Expectation.Any | null, - actualVersionHash: string | null, - statusInfo: { - status?: string - progress?: number - statusReason?: string - } -) => void -export type ReportPackageContainerPackageStatus = ( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null -) => void -export type ReportPackageContainerExpectationStatus = ( - containerId: string, - packageContainer: PackageContainerExpectation | null, - statusInfo: { - statusReason?: string - } -) => void +export interface ExpectationManagerCallbacks { + reportExpectationStatus: ( + expectationId: string, + expectaction: Expectation.Any | null, + actualVersionHash: string | null, + statusInfo: { + status?: string + progress?: number + statusReason?: string + } + ) => void + reportPackageContainerPackageStatus: ( + containerId: string, + packageId: string, + packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + ) => void + reportPackageContainerExpectationStatus: ( + containerId: string, + packageContainer: PackageContainerExpectation | null, + statusInfo: { + statusReason?: string + } + ) => void + messageFromWorker: MessageFromWorker +} + interface TrackedPackageContainerExpectation { /** Unique ID of the tracked packageContainer */ id: string diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index 6fd95e9d..d8811464 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -5,13 +5,7 @@ import * as Worker from '@shared/worker' import * as Winston from 'winston' import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, SingleAppConfig } from '@shared/api' // import deepExtend from 'deep-extend' -import { - ExpectationManager, - ReportExpectationStatus, - ReportPackageContainerPackageStatus, - MessageFromWorker, - ReportPackageContainerExpectationStatus, -} from '@shared/expectation-manager' +import { ExpectationManager, ExpectationManagerCallbacks } from '@shared/expectation-manager' import { CoreMockAPI } from './coreMockAPI' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' @@ -61,12 +55,7 @@ const defaultTestConfig: SingleAppConfig = { export async function setupExpectationManager( debugLogging: boolean, workerCount: number = 1, - callbacks: { - reportExpectationStatus: ReportExpectationStatus - reportPackageContainerPackageStatus: ReportPackageContainerPackageStatus - reportPackageContainerExpectationStatus: ReportPackageContainerExpectationStatus - messageFromWorker: MessageFromWorker - } + callbacks: ExpectationManagerCallbacks ) { const logger = new Winston.Logger({}) as LoggerInstance logger.add(Winston.transports.Console, { @@ -79,10 +68,7 @@ export async function setupExpectationManager( { type: 'internal' }, undefined, { type: 'internal' }, - callbacks.reportExpectationStatus, - callbacks.reportPackageContainerPackageStatus, - callbacks.reportPackageContainerExpectationStatus, - callbacks.messageFromWorker + callbacks ) // Initializing HTTP proxy Server: From 973cb165fcc12d1d736421aee4e050f3bbd8c472 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 11 Jun 2021 06:57:47 +0200 Subject: [PATCH 21/67] chore: refactor: metadata path in http-accessorHandler --- .../worker/src/worker/accessorHandlers/http.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index f3d47737..6b41611a 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -134,13 +134,13 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { - return this.fetchJSON(this.fullUrl + '_metadata.json') + return this.fetchJSON(this.getMetadataPath(this.fullUrl)) } async updateMetadata(metadata: Metadata): Promise { - await this.storeJSON(this.fullUrl + '_metadata.json', metadata) + await this.storeJSON(this.getMetadataPath(this.fullUrl), metadata) } async removeMetadata(): Promise { - await this.deletePackageIfExists(this.fullUrl + '_metadata.json') + await this.deletePackageIfExists(this.getMetadataPath(this.fullUrl)) } async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { @@ -297,7 +297,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle Date: Fri, 11 Jun 2021 14:41:23 +0200 Subject: [PATCH 22/67] feat: improved how statuses are handled. Introduced a separation between technical and user-displayable status descriptions. --- .../packages/generic/src/packageManager.ts | 11 +- shared/packages/api/src/logger.ts | 16 +- shared/packages/api/src/methods.ts | 8 +- shared/packages/api/src/worker.ts | 103 ++++++---- .../src/expectationManager.ts | 192 +++++++++++------- .../worker/src/expectationManagerApi.ts | 14 +- .../accessorHandlers/corePackageInfo.ts | 79 ++++--- .../src/worker/accessorHandlers/fileShare.ts | 104 +++++++--- .../worker/accessorHandlers/genericHandle.ts | 31 ++- .../src/worker/accessorHandlers/http.ts | 92 ++++++--- .../worker/accessorHandlers/localFolder.ts | 84 +++++--- .../src/worker/accessorHandlers/quantel.ts | 149 ++++++++++---- .../src/worker/lib/expectationHandler.ts | 1 + .../worker/src/worker/lib/workInProgress.ts | 8 +- .../worker/workers/linuxWorker/linuxWorker.ts | 4 +- .../expectationHandlers/fileCopy.ts | 72 +++++-- .../windowsWorker/expectationHandlers/lib.ts | 135 +++++++----- .../expectationHandlers/lib/scan.ts | 4 +- .../expectationHandlers/mediaFilePreview.ts | 88 ++++++-- .../expectationHandlers/mediaFileThumbnail.ts | 90 ++++++-- .../expectationHandlers/packageDeepScan.ts | 65 ++++-- .../expectationHandlers/packageScan.ts | 64 ++++-- .../expectationHandlers/quantelClipCopy.ts | 68 +++++-- .../expectationHandlers/quantelClipPreview.ts | 92 +++++++-- .../quantelClipThumbnail.ts | 117 +++++++++-- .../worker/workers/windowsWorker/lib/lib.ts | 25 ++- .../packageContainerExpectationHandler.ts | 36 ++-- shared/packages/worker/src/workerAgent.ts | 15 +- .../src/__tests__/lib/setupEnv.ts | 6 +- 29 files changed, 1277 insertions(+), 496 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index cfd7f951..3465c6cb 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -23,6 +23,7 @@ import { LoggerInstance, PackageContainerExpectation, literal, + Reason, } from '@shared/api' import deepExtend from 'deep-extend' import clone = require('fast-clone') @@ -351,7 +352,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks statusInfo: { status?: string progress?: number - statusReason?: string + statusReason?: Reason } ): void { if (!expectaction) { @@ -366,7 +367,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks ...{ status: 'N/A', progress: 0, - statusReason: '', + statusReason: { user: '', tech: '' }, }, // Previous properties: ...((this.toReportExpectationStatus[expectationId]?.workStatus || @@ -405,7 +406,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks ...{ status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, progress: 0, - statusReason: '', + statusReason: { user: '', tech: '' }, }, // pre-existing properties: ...(((this.toReportPackageStatus[packageContainerPackageId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in the outer spread-assignment more strict @@ -419,12 +420,12 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks public reportPackageContainerExpectationStatus( containerId: string, _packageContainer: PackageContainerExpectation | null, - statusInfo: { statusReason?: string | undefined } + statusInfo: { statusReason?: Reason } ): void { // This is not (yet) reported to Core. // ...to be implemented... this.logger.info(`PackageContainerStatus "${containerId}"`) - this.logger.info(statusInfo.statusReason || '>No reason<') + this.logger.info(statusInfo.statusReason?.tech || '>No reason<') } public async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { switch (message.type) { diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts index 9244563a..7ea11cef 100644 --- a/shared/packages/api/src/logger.ts +++ b/shared/packages/api/src/logger.ts @@ -50,14 +50,14 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance logger.info('Logging to Console') // Hijack console.log: // eslint-disable-next-line no-console - console.log = function (...args: any[]) { - // orgConsoleLog('a') - if (args.length >= 1) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore one or more arguments - logger.debug(...args) - } - } + // console.log = function (...args: any[]) { + // // orgConsoleLog('a') + // if (args.length >= 1) { + // // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // // @ts-ignore one or more arguments + // logger.debug(...args) + // } + // } } function getCurrentTime() { const v = Date.now() diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 5069a347..5eca03bd 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-namespace */ -import { ExpectedPackage } from '@sofie-automation/blueprints-integration' +import { ExpectedPackage, ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { Expectation } from './expectationApi' import { PackageContainerExpectation } from './packageContainerApi' import { @@ -13,6 +13,8 @@ import { ReturnTypeSetupPackageContainerMonitors, } from './worker' +/** Contains textual descriptions for statuses. */ +export type Reason = ExpectedPackageStatusAPI.Reason /* * This file contains API definitions for the methods used to communicate between the Workforce, Worker and Expectation-Manager. */ @@ -79,8 +81,8 @@ export namespace ExpectationManagerWorkerAgent { // Events emitted from a workInProgress: wipEventProgress: (wipId: number, actualVersionHash: string | null, progress: number) => Promise - wipEventDone: (wipId: number, actualVersionHash: string, reason: string, result: any) => Promise - wipEventError: (wipId: number, error: string) => Promise + wipEventDone: (wipId: number, actualVersionHash: string, reason: Reason, result: any) => Promise + wipEventError: (wipId: number, reason: Reason) => Promise } export interface WorkInProgressInfo { wipId: number diff --git a/shared/packages/api/src/worker.ts b/shared/packages/api/src/worker.ts index ff941c22..5c5ab357 100644 --- a/shared/packages/api/src/worker.ts +++ b/shared/packages/api/src/worker.ts @@ -2,24 +2,43 @@ * This file contains API definitions for the Worker methods */ -export interface ReturnTypeDoYouSupportExpectation { - support: boolean - reason: string -} +import { Reason } from './methods' + +export type ReturnTypeDoYouSupportExpectation = + | { + support: true + } + | { + support: false + reason: Reason + } export type ReturnTypeGetCostFortExpectation = number -export interface ReturnTypeIsExpectationReadyToStartWorkingOn { - ready: boolean - sourceExists?: boolean - reason?: string -} -export interface ReturnTypeIsExpectationFullfilled { - fulfilled: boolean - reason?: string -} -export interface ReturnTypeRemoveExpectation { - removed: boolean - reason?: string -} +export type ReturnTypeIsExpectationReadyToStartWorkingOn = + | { + ready: true + sourceExists?: boolean + } + | { + ready: false + sourceExists?: boolean + reason: Reason + } +export type ReturnTypeIsExpectationFullfilled = + | { + fulfilled: true + } + | { + fulfilled: false + reason: Reason + } +export type ReturnTypeRemoveExpectation = + | { + removed: true + } + | { + removed: false + reason: Reason + } /** Configurations for any of the workers */ export interface WorkerAgentConfig { @@ -38,23 +57,39 @@ export interface WorkerAgentConfig { */ windowsDriveLetters?: string[] } -export interface ReturnTypeDoYouSupportPackageContainer { - support: boolean - reason: string -} -export interface ReturnTypeRunPackageContainerCronJob { - completed: boolean - reason?: string -} -export interface ReturnTypeDisposePackageContainerMonitors { - disposed: boolean - reason?: string -} -export interface ReturnTypeSetupPackageContainerMonitors { - setupOk: boolean - reason?: string - monitors?: { [monitorId: string]: MonitorProperties } -} +export type ReturnTypeDoYouSupportPackageContainer = + | { + support: true + } + | { + support: false + reason: Reason + } +export type ReturnTypeRunPackageContainerCronJob = + | { + success: true + } + | { + success: false + reason: Reason + } +export type ReturnTypeDisposePackageContainerMonitors = + | { + success: true + } + | { + success: false + reason: Reason + } +export type ReturnTypeSetupPackageContainerMonitors = + | { + success: true + monitors: { [monitorId: string]: MonitorProperties } + } + | { + success: false + reason: Reason + } export interface MonitorProperties { label: string } diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 25472b47..bf124aaf 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -10,6 +10,7 @@ import { Hook, LoggerInstance, PackageContainerExpectation, + Reason, } from '@shared/api' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { WorkforceAPI } from './workforceApi' @@ -285,7 +286,7 @@ export class ExpectationManager { wipEventDone: async ( wipId: number, actualVersionHash: string, - reason: string, + reason: Reason, _result: any ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] @@ -315,12 +316,12 @@ export class ExpectationManager { delete this.worksInProgress[`${clientId}_${wipId}`] } }, - wipEventError: async (wipId: number, error: string): Promise => { + wipEventError: async (wipId: number, reason: Reason): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { if (wip.trackedExp.state === TrackedExpectationState.WORKING) { wip.trackedExp.errorCount++ - this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, error) + this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, reason) this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { status: wip.trackedExp.state, statusReason: wip.trackedExp.reason, @@ -393,15 +394,24 @@ export class ExpectationManager { availableWorkers: [], lastEvaluationTime: 0, errorCount: 0, - reason: '', + reason: { + user: '', + tech: '', + }, status: {}, session: null, } this.trackedExpectations[id] = trackedExp if (isNew) { - this.updateTrackedExpStatus(trackedExp, undefined, 'Added just now') + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Added just now`, + tech: `Added ${Date.now()}`, + }) } else { - this.updateTrackedExpStatus(trackedExp, undefined, 'Updated just now') + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Updated just now`, + tech: `Updated ${Date.now()}`, + }) } } } @@ -612,7 +622,10 @@ export class ExpectationManager { trackedExp.availableWorkers = [] trackedExp.status = {} - let notSupportReason = 'No workers registered' + let notSupportReason: Reason = { + user: 'No workers registered (this is likely a configuration issue)', + tech: 'No workers registered', + } await Promise.all( Object.entries(this.workerAgents).map(async ([id, workerAgent]) => { const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) @@ -625,18 +638,16 @@ export class ExpectationManager { }) ) if (trackedExp.availableWorkers.length) { - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.WAITING, - `Found ${trackedExp.availableWorkers.length} workers who supports this Expectation` - ) + this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.WAITING, { + user: `${trackedExp.availableWorkers.length} workers available, about to start...`, + tech: `Found ${trackedExp.availableWorkers.length} workers who supports this Expectation`, + }) trackedExp.session.triggerExpectationAgain = true } else { - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.NEW, - `Found no workers who supports this Expectation: "${notSupportReason}"` - ) + this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, { + user: `Found no workers who supports this Expectation, due to: ${notSupportReason.user}`, + tech: `Found no workers who supports this Expectation: "${notSupportReason.tech}"`, + }) } } else if (trackedExp.state === TrackedExpectationState.WAITING) { // Check if the expectation is ready to start: @@ -651,7 +662,7 @@ export class ExpectationManager { ) if (fulfilled.fulfilled) { // The expectation is already fulfilled: - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.FULFILLED, fulfilled.reason) + this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.FULFILLED, undefined) if (this.handleTriggerByFullfilledIds(trackedExp)) { // Something was triggered, run again ASAP: trackedExp.session.triggerOtherExpectationsAgain = true @@ -669,7 +680,10 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, TrackedExpectationState.READY, - 'Ready to start', + { + user: 'About to start working..', + tech: 'About to start working..', + }, newStatus ) trackedExp.session.triggerExpectationAgain = true @@ -689,7 +703,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else if (trackedExp.state === TrackedExpectationState.READY) { @@ -728,7 +742,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else if (trackedExp.state === TrackedExpectationState.WORKING) { @@ -762,7 +776,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else { @@ -785,7 +799,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else if (trackedExp.state === TrackedExpectationState.RESTARTED) { @@ -794,7 +808,10 @@ export class ExpectationManager { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, 'Ready to start') + this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, { + user: 'Ready to start (after restart)', + tech: 'Ready to start (after restart)', + }) trackedExp.session.triggerExpectationAgain = true } else { this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.RESTARTED, removed.reason) @@ -805,7 +822,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else if (trackedExp.state === TrackedExpectationState.ABORTED) { @@ -815,7 +832,10 @@ export class ExpectationManager { const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { // This will cause the expectation to be intentionally stuck in the ABORTED state. - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, 'Aborted') + this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, { + user: 'Aborted', + tech: 'Aborted', + }) } else { this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, removed.reason) } @@ -825,7 +845,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else { @@ -850,7 +870,7 @@ export class ExpectationManager { private updateTrackedExpStatus( trackedExp: TrackedExpectation, state: TrackedExpectationState | undefined, - reason: string | undefined, + reason: Reason | undefined, newStatus?: Partial ) { trackedExp.lastEvaluationTime = Date.now() @@ -867,8 +887,8 @@ export class ExpectationManager { updatedState = true } - if (trackedExp.reason !== reason) { - trackedExp.reason = reason || '' + if (!_.isEqual(trackedExp.reason, reason)) { + trackedExp.reason = reason || { user: '', tech: '' } updatedReason = true } const status = Object.assign({}, trackedExp.status, newStatus) // extend with new values @@ -879,11 +899,11 @@ export class ExpectationManager { // Log and report new states an reasons: if (updatedState) { this.logger.info( - `${trackedExp.exp.statusReport.label}: New state: "${prevState}"->"${trackedExp.state}", reason: "${trackedExp.reason}"` + `${trackedExp.exp.statusReport.label}: New state: "${prevState}"->"${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } else if (updatedReason) { this.logger.info( - `${trackedExp.exp.statusReport.label}: State: "${trackedExp.state}", reason: "${trackedExp.reason}"` + `${trackedExp.exp.statusReport.label}: State: "${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } @@ -943,7 +963,7 @@ export class ExpectationManager { const minWorkerCount = batchSize / 2 if (!trackedExp.availableWorkers.length) { - session.noAssignedWorkerReason = 'No workers available' + session.noAssignedWorkerReason = { user: `No workers available`, tech: `No workers available` } } const workerCosts: WorkerAgentAssignment[] = [] @@ -991,7 +1011,10 @@ export class ExpectationManager { // Only allow starting if the job can start in a short while session.assignedWorker = bestWorker } else { - session.noAssignedWorkerReason = `Waiting for a free worker (${trackedExp.availableWorkers.length} busy)` + session.noAssignedWorkerReason = { + user: `Waiting for a free worker (${trackedExp.availableWorkers.length} workers are currently busy)`, + tech: `Waiting for a free worker (${trackedExp.availableWorkers.length} busy)`, + } } } /** @@ -1031,7 +1054,10 @@ export class ExpectationManager { if (waitingFor) { return { ready: false, - reason: `Waiting for "${waitingFor.exp.statusReport.label}"`, + reason: { + user: `Waiting for "${waitingFor.exp.statusReport.label}"`, + tech: `Waiting for "${waitingFor.exp.statusReport.label}"`, + }, } } } @@ -1061,6 +1087,7 @@ export class ExpectationManager { currentWorker: null, isUpdated: true, lastEvaluationTime: 0, + monitorIsSetup: false, status: { monitors: {}, }, @@ -1094,14 +1121,14 @@ export class ExpectationManager { // If the packageContainer was newly updated, reset and set up again: if (trackedPackageContainer.currentWorker) { const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - const dispose = await workerAgent.api.disposePackageContainerMonitors( + const disposeMonitorResult = await workerAgent.api.disposePackageContainerMonitors( trackedPackageContainer.packageContainer ) - if (!dispose.disposed) { - this.updateTrackedPackageContainerStatus( - trackedPackageContainer, - 'Unable to dispose: ' + dispose.reason - ) + if (!disposeMonitorResult.success) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to remove monitor, due to ${disposeMonitorResult.reason}`, + tech: `Unable to dispose monitor: ${disposeMonitorResult.reason}`, + }) continue // Break further execution for this PackageContainer } trackedPackageContainer.currentWorker = null @@ -1115,10 +1142,10 @@ export class ExpectationManager { trackedPackageContainer.currentWorker = null } } - let currentWorkerIsNew = false if (!trackedPackageContainer.currentWorker) { - // Find a worker - let notSupportReason: string | null = null + // Find a worker that supports this PackageContainer + + let notSupportReason: Reason | null = null await Promise.all( Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { const support = await workerAgent.api.doYouSupportPackageContainer( @@ -1127,7 +1154,6 @@ export class ExpectationManager { if (!trackedPackageContainer.currentWorker) { if (support.support) { trackedPackageContainer.currentWorker = workerId - currentWorkerIsNew = true } else { notSupportReason = support.reason } @@ -1135,13 +1161,16 @@ export class ExpectationManager { }) ) if (!trackedPackageContainer.currentWorker) { - notSupportReason = 'Found no worker that supports this packageContainer' + notSupportReason = { + user: 'Found no worker that supports this packageContainer', + tech: 'Found no worker that supports this packageContainer', + } } if (notSupportReason) { - this.updateTrackedPackageContainerStatus( - trackedPackageContainer, - 'Not supported: ' + notSupportReason - ) + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to handle PackageContainer, due to: ${notSupportReason.user}`, + tech: `Unable to handle PackageContainer, due to: ${notSupportReason.tech}`, + }) continue // Break further execution for this PackageContainer } } @@ -1149,27 +1178,38 @@ export class ExpectationManager { if (trackedPackageContainer.currentWorker) { const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - if (currentWorkerIsNew) { + if (!trackedPackageContainer.monitorIsSetup) { const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( trackedPackageContainer.packageContainer ) trackedPackageContainer.status.monitors = {} - for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors ?? {})) { - trackedPackageContainer.status.monitors[monitorId] = { - label: monitor.label, - reason: 'Starting up', + if (monitorSetup.success) { + trackedPackageContainer.monitorIsSetup = true + for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { + trackedPackageContainer.status.monitors[monitorId] = { + label: monitor.label, + reason: { + user: 'Starting up', + tech: 'Starting up', + }, + } } + } else { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, + tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + }) } } const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( trackedPackageContainer.packageContainer ) - if (!cronJobStatus.completed) { - this.updateTrackedPackageContainerStatus( - trackedPackageContainer, - 'Cron job not completed: ' + cronJobStatus.reason - ) + if (!cronJobStatus.success) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: 'Cron job not completed: ' + cronJobStatus.reason.user, + tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, + }) continue } } @@ -1177,7 +1217,7 @@ export class ExpectationManager { } private updateTrackedPackageContainerStatus( trackedPackageContainer: TrackedPackageContainerExpectation, - reason: string | undefined + reason: Reason ) { trackedPackageContainer.lastEvaluationTime = Date.now() @@ -1204,6 +1244,18 @@ export class ExpectationManager { ) } } + private getNoAssignedWorkerReason(session: ExpectationStateHandlerSession): ExpectedPackageStatusAPI.Reason { + if (!session.noAssignedWorkerReason) { + this.logger.error( + `trackedExp.session.noAssignedWorkerReason is undefined, although assignedWorker was set..` + ) + return { + user: 'Unknown reason (internal error)', + tech: 'Unknown reason', + } + } + return session.noAssignedWorkerReason + } } export type ExpectationManagerServerOptions = | { @@ -1235,8 +1287,8 @@ interface TrackedExpectation { /** The current State of the expectation. */ state: TrackedExpectationState - /** Human-readable reason for the current state. (To be used in GUIs) */ - reason: string + /** Reason for the current state. */ + reason: Reason /** List of worker ids that supports this Expectation */ availableWorkers: string[] @@ -1273,7 +1325,7 @@ interface ExpectationStateHandlerSession { /** The Worker assigned to the Expectation during this evaluation-session */ assignedWorker?: WorkerAgentAssignment - noAssignedWorkerReason?: string + noAssignedWorkerReason?: Reason } interface WorkerAgentAssignment { worker: WorkerAgentAPI @@ -1291,7 +1343,7 @@ export interface ExpectationManagerCallbacks { statusInfo: { status?: string progress?: number - statusReason?: string + statusReason?: Reason } ) => void reportPackageContainerPackageStatus: ( @@ -1303,7 +1355,7 @@ export interface ExpectationManagerCallbacks { containerId: string, packageContainer: PackageContainerExpectation | null, statusInfo: { - statusReason?: string + statusReason?: Reason } ) => void messageFromWorker: MessageFromWorker @@ -1322,13 +1374,17 @@ interface TrackedPackageContainerExpectation { /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number + /** If the monitor is set up okay */ + monitorIsSetup: boolean + /** These statuses are sent from the workers */ status: { - reason?: string + /** Reason for the status (used in GUIs) */ + reason?: Reason monitors: { [monitorId: string]: { label: string - reason: string + reason: Reason } } } diff --git a/shared/packages/worker/src/expectationManagerApi.ts b/shared/packages/worker/src/expectationManagerApi.ts index 2a79f65d..d9495d0c 100644 --- a/shared/packages/worker/src/expectationManagerApi.ts +++ b/shared/packages/worker/src/expectationManagerApi.ts @@ -1,4 +1,4 @@ -import { ExpectationManagerWorkerAgent, AdapterClient, LoggerInstance } from '@shared/api' +import { ExpectationManagerWorkerAgent, AdapterClient, LoggerInstance, Reason } from '@shared/api' /** * Exposes the API-methods of a ExpectationManager, to be called from the WorkerAgent @@ -13,18 +13,18 @@ export class ExpectationManagerAPI } async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('messageFromWorker', message) + return this._sendMessage('messageFromWorker', message) } async wipEventProgress(wipId: number, actualVersionHash: string | null, progress: number): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventProgress', wipId, actualVersionHash, progress) + return this._sendMessage('wipEventProgress', wipId, actualVersionHash, progress) } - async wipEventDone(wipId: number, actualVersionHash: string, reason: string, result: unknown): Promise { + async wipEventDone(wipId: number, actualVersionHash: string, reason: Reason, result: unknown): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventDone', wipId, actualVersionHash, reason, result) + return this._sendMessage('wipEventDone', wipId, actualVersionHash, reason, result) } - async wipEventError(wipId: number, error: string): Promise { + async wipEventError(wipId: number, reason: Reason): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventError', wipId, error) + return this._sendMessage('wipEventError', wipId, reason) } } diff --git a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts index 3272c823..916fdb13 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts @@ -1,6 +1,6 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { GenericAccessorHandle, PackageReadInfo, PutPackageHandler } from './genericHandle' -import { hashObj, Expectation } from '@shared/api' +import { GenericAccessorHandle, PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' +import { hashObj, Expectation, Reason } from '@shared/api' import { GenericWorker } from '../worker' /** Accessor handle for accessing data store in Core */ @@ -28,31 +28,37 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand static doYouSupportAccess(): boolean { return true // always has access } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { // Note: We assume that we always have write access here, no need to check this.accessor.allowRead return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { // Note: We assume that we always have write access here, no need to check this.accessor.allowWrite return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.CORE_PACKAGE_INFO) { - return `CorePackageInfo Accessor type is not CORE_PACKAGE_INFO ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `CorePackageInfo Accessor type is not CORE_PACKAGE_INFO ("${this.accessor.type}")!`, + }, + } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { // todo: add a check here? - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { // not needed - return undefined + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { // todo: add a check here? - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { throw new Error('getPackageActualVersion not applicable for CorePackageInfo') @@ -94,14 +100,29 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand async removeMetadata(): Promise { // Not applicable } - async runCronJob(): Promise { - return undefined // not applicable - } - async setupPackageContainerMonitors(): Promise { - return undefined // not applicable + async runCronJob(): Promise { + return { + success: false, + reason: { user: `There is an internal issue in Package Manager`, tech: 'runCronJob not supported' }, + } // not applicable } - async disposePackageContainerMonitors(): Promise { - return undefined // not applicable + async setupPackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'setupPackageContainerMonitors, not supported', + }, + } // not applicable + } + async disposePackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'disposePackageContainerMonitors, not supported', + }, + } // not applicable } public async findUnUpdatedPackageInfo( @@ -110,7 +131,7 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand content: unknown, actualSourceVersion: Expectation.Version.Any, expectTargetVersion: unknown - ): Promise<{ needsUpdate: boolean; reason: string }> { + ): Promise<{ needsUpdate: false } | { needsUpdate: true; reason: Reason }> { const actualContentVersionHash = this.getActualContentVersionHash( content, actualSourceVersion, @@ -131,24 +152,32 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand if (!packageInfo) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" not found in PackageInfo store`, + reason: { + user: 'Package info needs to be stored', + tech: `Package "${fromPackage.id}" not found in PackageInfo store`, + }, } } else if (packageInfo.expectedContentVersionHash !== fromPackage.expectedContentVersionHash) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" expected version differs in PackageInfo store`, + reason: { + user: 'Package info needs to be updated', + tech: `Package "${fromPackage.id}" expected version differs in PackageInfo store`, + }, } } else if (packageInfo.actualContentVersionHash !== actualContentVersionHash) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" actual version differs in PackageInfo store`, + reason: { + user: 'Package info needs to be re-synced', + tech: `Package "${fromPackage.id}" actual version differs in PackageInfo store`, + }, } } } return { needsUpdate: false, - reason: `All packages in PackageInfo store are in sync`, } } public async updatePackageInfo( diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 033796da..1170b135 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -1,7 +1,7 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { PackageReadInfo, PutPackageHandler } from './genericHandle' +import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' import { Expectation, PackageContainerExpectation } from '@shared/api' import { GenericWorker } from '../worker' import { WindowsWorker } from '../workers/windowsWorker/windowsWorker' @@ -70,32 +70,52 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle const accessor = accessor0 as AccessorOnPackage.FileShare return !accessor.networkId || worker.location.localNetworkIds.includes(accessor.networkId) } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { if (!this.accessor.allowRead) { - return `Not allowed to read` + return { + success: false, + reason: { + user: `Not allowed to read`, + tech: `Not allowed to read`, + }, + } } return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { if (!this.accessor.allowWrite) { - return `Not allowed to write` + return { + success: false, + reason: { + user: `Not allowed to write`, + tech: `Not allowed to write`, + }, + } } return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.FILE_SHARE) { - return `FileShare Accessor type is not FILE_SHARE ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `FileShare Accessor type is not FILE_SHARE ("${this.accessor.type}")!`, + }, + } } - if (!this.accessor.folderPath) return `Folder path not set` + if (!this.accessor.folderPath) + return { success: false, reason: { user: `Folder path not set`, tech: `Folder path not set` } } if (!this.content.onlyContainerAccess) { - if (!this.filePath) return `File path not set` + if (!this.filePath) + return { success: false, reason: { user: `File path not set`, tech: `File path not set` } } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { const readIssue = await this._checkPackageReadAccess() - if (readIssue) { - if (readIssue.match(/EPERM/)) { + if (!readIssue.success) { + if (readIssue.reason.tech.match(/EPERM/)) { // "EPERM: operation not permitted" if (this.accessor.userName) { // Try resetting the access permissions: @@ -108,9 +128,9 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return readIssue } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { try { // Check if we can open the file for reading: const fd = await fsOpen(this.fullPath, 'r+') @@ -119,16 +139,22 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle await fsClose(fd) } catch (err) { if (err && err.code === 'EBUSY') { - return `Not able to read file (busy)` + return { + success: false, + reason: { user: `Not able to read file (file is busy)`, tech: err.toString() }, + } } else if (err && err.code === 'ENOENT') { - return `File does not exist (ENOENT)` + return { success: false, reason: { user: `File does not exist`, tech: err.toString() } } } else { - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { user: `Not able to read file`, tech: err.toString() }, + } } } - return undefined // all good + return { success: true } } - private async _checkPackageReadAccess(): Promise { + private async _checkPackageReadAccess(): Promise { await this.prepareFileAccess() try { @@ -136,20 +162,32 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle // The file exists } catch (err) { // File is not readable - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { + user: `File doesn't exist`, + tech: `Not able to read file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { await this.prepareFileAccess() try { await fsAccess(this.folderPath, fs.constants.W_OK) // The file exists } catch (err) { - // File is not readable - return `Not able to write to file: ${err.toString()}` + // File is not writeable + return { + success: false, + reason: { + user: `Not able to write to file`, + tech: `Not able to write to file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { await this.prepareFileAccess() @@ -247,7 +285,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle async removeMetadata(): Promise { await this.unlinkIfExists(this.metadataPath) } - async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { @@ -260,9 +298,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } } - return undefined + return { success: true } } - async setupPackageContainerMonitors(packageContainerExp: PackageContainerExpectation): Promise { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -274,11 +314,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } } - return undefined // all good + return { success: true } } async disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise { + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -289,7 +329,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle assertNever(monitor) } } - return undefined // all good + return { success: true } } /** Called when the package is supposed to be in place */ async packageIsInPlace(): Promise { diff --git a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts index 5afc7ab2..259a1034 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts @@ -1,6 +1,6 @@ import { AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { EventEmitter } from 'events' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, Reason } from '@shared/api' import { GenericWorker } from '../worker' /** @@ -24,30 +24,30 @@ export abstract class GenericAccessorHandle { * Checks if there are any issues with the properties in the accessor or content for being able to read * @returns undefined if all is OK / string with error message */ - abstract checkHandleRead(): string | undefined + abstract checkHandleRead(): AccessorHandlerResult /** * Checks if there are any issues with the properties in the accessor or content for being able to write * @returns undefined if all is OK / string with error message */ - abstract checkHandleWrite(): string | undefined + abstract checkHandleWrite(): AccessorHandlerResult /** * Checks if Accesor has access to the Package, for reading. * Errors from this method are related to access/permission issues, or that the package doesn't exist. * @returns undefined if all is OK / string with error message */ - abstract checkPackageReadAccess(): Promise + abstract checkPackageReadAccess(): Promise /** * Do a check if it actually is possible to access the package. * Errors from this method are related to the actual access of the package (such as resource is busy). * @returns undefined if all is OK / string with error message */ - abstract tryPackageRead(): Promise + abstract tryPackageRead(): Promise /** * Checks if the PackageContainer can be written to * @returns undefined if all is OK / string with error message */ - abstract checkPackageContainerWriteAccess(): Promise + abstract checkPackageContainerWriteAccess(): Promise /** * Extracts and returns the version from the package * @returns the vesion of the package @@ -91,21 +91,21 @@ export abstract class GenericAccessorHandle { * Performs a cronjob on the Package container * @returns undefined if all is OK / string with error message */ - abstract runCronJob(packageContainerExp: PackageContainerExpectation): Promise + abstract runCronJob(packageContainerExp: PackageContainerExpectation): Promise /** * Setup monitors on the Package container * @returns undefined if all is OK / string with error message */ abstract setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise + ): Promise /** * Tear down monitors on the Package container * @returns undefined if all is OK / string with error message */ abstract disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise + ): Promise protected setCache(key: string, value: T): T { if (!this.worker.accessorCache[this.type]) { @@ -132,6 +132,19 @@ export abstract class GenericAccessorHandle { } } +/** Default result returned from most accessorHandler-methods */ +export type AccessorHandlerResult = + | { + /** Whether the action was successful or not */ + success: true + } + | { + /** Whether the action was successful or not */ + success: false + /** If the action isn't successful, the reason why */ + reason: Reason + } + /** * A class emitted from putPackageStream() and putPackageInfo(), used to signal the progression of an ongoing write operation. * Users of this class are required to emit the events 'error' on error and 'close' upon completion diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index 6b41611a..6efcf286 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -1,5 +1,11 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { GenericAccessorHandle, PackageReadInfo, PackageReadStream, PutPackageHandler } from './genericHandle' +import { + GenericAccessorHandle, + PackageReadInfo, + PackageReadStream, + PutPackageHandler, + AccessorHandlerResult, +} from './genericHandle' import { Expectation, PackageContainerExpectation } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' @@ -38,33 +44,51 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + async checkPackageReadAccess(): Promise { const header = await this.fetchHeader() if (header.status >= 400) { - return `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}` + return { + success: false, + reason: { + user: `Got error code ${header.status} when trying to fetch package`, + tech: `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}`, + }, + } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { // TODO: how to do this? - return undefined + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { // todo: how to check this? - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { const header = await this.fetchHeader() @@ -143,7 +167,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { @@ -156,9 +180,11 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -170,13 +196,13 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + ): Promise { // todo: implement monitors - return undefined + return { success: true } } get fullUrl(): string { return [ @@ -185,15 +211,35 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericFileAccessorHand return path.join(this.folderPath, this.filePath) } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { if (!this.accessor.allowRead) { - return `Not allowed to read` + return { success: false, reason: { user: `Not allowed to read`, tech: `Not allowed to read` } } } return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { if (!this.accessor.allowWrite) { - return `Not allowed to write` + return { success: false, reason: { user: `Not allowed to write`, tech: `Not allowed to write` } } } return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.LOCAL_FOLDER) { - return `LocalFolder Accessor type is not LOCAL_FOLDER ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `LocalFolder Accessor type is not LOCAL_FOLDER ("${this.accessor.type}")!`, + }, + } } - if (!this.accessor.folderPath) return `Folder path not set` + if (!this.accessor.folderPath) + return { success: false, reason: { user: `Folder path not set`, tech: `Folder path not set` } } if (!this.content.onlyContainerAccess) { - if (!this.filePath) return `File path not set` + if (!this.filePath) + return { success: false, reason: { user: `File path not set`, tech: `File path not set` } } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { try { await fsAccess(this.fullPath, fs.constants.R_OK) // The file exists and can be read } catch (err) { // File is not readable - return `Not able to access file: ${err.toString()}` + return { + success: false, + reason: { + user: `File doesn't exist`, + tech: `Not able to access file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { try { // Check if we can open the file for reading: const fd = await fsOpen(this.fullPath, 'r+') @@ -98,24 +112,36 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand await fsClose(fd) } catch (err) { if (err && err.code === 'EBUSY') { - return `Not able to read file (busy)` + return { + success: false, + reason: { user: `Not able to read file (file is busy)`, tech: err.toString() }, + } } else if (err && err.code === 'ENOENT') { - return `File does not exist (ENOENT)` + return { success: false, reason: { user: `File does not exist`, tech: err.toString() } } } else { - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { user: `Not able to read file`, tech: err.toString() }, + } } } - return undefined // all good + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { try { await fsAccess(this.folderPath, fs.constants.W_OK) // The file exists } catch (err) { - // File is not readable - return `Not able to write to file: ${err.toString()}` + // File is not writeable + return { + success: false, + reason: { + user: `Not able to write to file`, + tech: `Not able to write to file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { const stat = await fsStat(this.fullPath) @@ -208,7 +234,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand async removeMetadata(): Promise { await this.unlinkIfExists(this.metadataPath) } - async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { @@ -221,9 +247,11 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand } } - return undefined + return { success: true } } - async setupPackageContainerMonitors(packageContainerExp: PackageContainerExpectation): Promise { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -235,11 +263,11 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand } } - return undefined // all good + return { success: true } } async disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise { + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -250,7 +278,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand assertNever(monitor) } } - return undefined // all good + return { success: true } } /** Called when the package is supposed to be in place */ diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index d3b51f2d..2ac82397 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -6,8 +6,9 @@ import { PackageReadInfoBaseType, PackageReadInfoQuantelClip, PutPackageHandler, + AccessorHandlerResult, } from './genericHandle' -import { Expectation, literal } from '@shared/api' +import { Expectation, literal, Reason } from '@shared/api' import { GenericWorker } from '../worker' import { ClipData, ClipDataSummary, ServerInfo } from 'tv-automation-quantel-gateway-client/dist/quantelTypes' @@ -48,36 +49,63 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async checkPackageReadAccess(): Promise { const quantel = await this.getQuantelGateway() // Search for a clip that match: @@ -85,24 +113,42 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async tryPackageRead(): Promise { const quantel = await this.getQuantelGateway() const clipSummary = await this.searchForLatestClip(quantel) - if (!clipSummary) return `No clip found` + if (!clipSummary) return { success: false, reason: { user: `No clip found`, tech: `No clip found` } } if (!parseInt(clipSummary.Frames, 10)) { - return `Clip "${clipSummary.ClipGUID}" has no frames` + return { + success: false, + reason: { + user: `The clip has no frames`, + tech: `Clip "${clipSummary.ClipGUID}" has no frames`, + }, + } } if (parseInt(clipSummary.Frames, 10) < MINIMUM_FRAMES) { // Check that it is meaningfully playable - return `Clip "${clipSummary.ClipGUID}" hasn't received enough frames` + return { + success: false, + reason: { + user: `The clip hasn't received enough frames`, + tech: `Clip "${clipSummary.ClipGUID}" hasn't received enough frames (${clipSummary.Frames})`, + }, + } } // 5/5/21: Removed check for completed - OA tests shoes it does nothing for placeholders / Richard // if (!clipSummary.Completed || !clipSummary.Completed.length) { @@ -110,9 +156,9 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async checkPackageContainerWriteAccess(): Promise { const quantel = await this.getQuantelGateway() const server = await quantel.getServer() @@ -123,7 +169,7 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { const quantel = await this.getQuantelGateway() @@ -246,33 +292,61 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { // Not supported, do nothing } - async runCronJob(): Promise { - return undefined // not applicable + async runCronJob(): Promise { + return { + success: false, + reason: { user: `There is an internal issue in Package Manager`, tech: 'runCronJob not supported' }, + } // not applicable } - async setupPackageContainerMonitors(): Promise { - return undefined // not applicable + async setupPackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'setupPackageContainerMonitors, not supported', + }, + } // not applicable } - async disposePackageContainerMonitors(): Promise { - return undefined // not applicable + async disposePackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'disposePackageContainerMonitors, not supported', + }, + } // not applicable } async getClip(): Promise { const quantel = await this.getQuantelGateway() - return await this.searchForLatestClip(quantel) + return this.searchForLatestClip(quantel) } async getClipDetails(clipId: number): Promise { const quantel = await this.getQuantelGateway() - return await quantel.getClip(clipId) + return quantel.getClip(clipId) } - async getTransformerStreamURL(): Promise<{ baseURL: string; url: string; fullURL: string } | undefined> { - if (!this.accessor.transformerURL) return undefined + get transformerURL(): string | undefined { + return this.accessor.transformerURL + } + async getTransformerStreamURL(): Promise< + { success: true; baseURL: string; url: string; fullURL: string } | { success: false; reason: Reason } + > { + if (!this.accessor.transformerURL) + return { + success: false, + reason: { + user: `transformerURL is not set in settings`, + tech: `transformerURL not set`, + }, + } const clip = await this.getClip() if (clip) { const baseURL = this.accessor.transformerURL const url = `/quantel/homezone/clips/streams/${clip.ClipID}/stream.mpd` return { + success: true, baseURL, url, fullURL: [ @@ -280,8 +354,15 @@ export class QuantelAccessorHandle extends GenericAccessorHandle void - done: (actualVersionHash: string, reason: string, result: any) => void - error: (error: string) => void + done: (actualVersionHash: string, reason: Reason, result: any) => void + error: (reason: string) => void } export declare interface IWorkInProgress { properties: ExpectationManagerWorkerAgent.WorkInProgressProperties @@ -49,7 +49,7 @@ export class WorkInProgress extends EventEmitter implements IWorkInProgress { } } // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - _reportComplete(actualVersionHash: string, reason: string, result: any): void { + _reportComplete(actualVersionHash: string, reason: Reason, result: any): void { this.emit('done', actualVersionHash, reason, result) } _reportError(err: Error): void { diff --git a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts index f76671a4..25071ed6 100644 --- a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts +++ b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts @@ -32,7 +32,7 @@ export class LinuxWorker extends GenericWorker { async doYouSupportExpectation(_exp: Expectation.Any): Promise { return { support: false, - reason: `Not implemented yet`, + reason: { user: `Not implemented yet`, tech: `Not implemented yet` }, } } async init(): Promise { @@ -62,7 +62,7 @@ export class LinuxWorker extends GenericWorker { ): Promise { return { support: false, - reason: `Not implemented yet`, + reason: { user: `Not implemented yet`, tech: `Not implemented yet` }, } } async runPackageContainerCronJob( diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index f228bc44..52462739 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -93,20 +93,26 @@ export const FileCopy: ExpectationWindowsHandler = { return { ready: false, sourceExists: true, - reason: `Source is not stable (${userReadableDiff(versionDiff)})`, + reason: { + user: `Waiting for source file to stop growing`, + tech: `Source is not stable (${userReadableDiff(versionDiff)})`, + }, } } } // Also check if we actually can read from the package, // this might help in some cases if the file is currently transferring - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: { + // user: 'Ready to start copying', + // tech: `${lookupSource.reason}, ${lookupTarget.reason}`, + // }, } }, isExpectationFullfilled: async ( @@ -118,16 +124,29 @@ export const FileCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issuePackage = await lookupTarget.handle.checkPackageReadAccess() - if (issuePackage) { - return { fulfilled: false, reason: `File does not exist: ${issuePackage.toString()}` } + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } } // check that the file is of the right version: const actualTargetVersion = await lookupTarget.handle.fetchMetadata() - if (!actualTargetVersion) return { fulfilled: false, reason: `Metadata missing` } + if (!actualTargetVersion) + return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } const lookupSource = await lookupCopySources(worker, exp) if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) @@ -135,13 +154,13 @@ export const FileCopy: ExpectationWindowsHandler = { const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const issueVersions = compareUniversalVersions(makeUniversalVersion(actualSourceVersion), actualTargetVersion) - if (issueVersions) { - return { fulfilled: false, reason: issueVersions } + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } } return { fulfilled: true, - reason: `File "${exp.endRequirement.content.filePath}" already exists on target`, + // reason: `File "${exp.endRequirement.content.filePath}" already exists on target`, } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -215,7 +234,10 @@ export const FileCopy: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Copy completed at ${Date.now()}`, + }, undefined ) }) @@ -293,7 +315,10 @@ export const FileCopy: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Copy completed at ${Date.now()}`, + }, undefined ) })().catch((err) => { @@ -316,16 +341,31 @@ export const FileCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed file "${exp.endRequirement.content.filePath}" from target` } + return { + removed: true, + // reason: `Removed file "${exp.endRequirement.content.filePath}" from target` + } }, } function isFileCopy(exp: Expectation.Any): exp is Expectation.FileCopy { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index 1894e38d..383ce1bd 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -5,7 +5,7 @@ import { GenericAccessorHandle } from '../../../accessorHandlers/genericHandle' import { GenericWorker } from '../../../worker' import { compareActualExpectVersions, findBestPackageContainerWithAccessToPackage } from '../lib/lib' import { Diff } from 'deep-diff' -import { Expectation, ReturnTypeDoYouSupportExpectation } from '@shared/api' +import { Expectation, Reason, ReturnTypeDoYouSupportExpectation } from '@shared/api' /** Check that a worker has access to the packageContainers through its accessors */ export function checkWorkerHasAccessToPackageContainersOnPackage( @@ -22,9 +22,12 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( if (!accessSourcePackageContainer) { return { support: false, - reason: `Doesn't have access to any of the source packageContainers (${checks.sources - .map((o) => o.containerId) - .join(', ')})`, + reason: { + user: `There is an issue with the configuration of the Worker, it doesn't have access to any of the source PackageContainers`, + tech: `Worker doesn't have access to any of the source packageContainers (${checks.sources + .map((o) => o.containerId) + .join(', ')})`, + }, } } } @@ -35,43 +38,46 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( if (!accessTargetPackageContainer) { return { support: false, - reason: `Doesn't have access to any of the target packageContainers (${checks.targets - .map((o) => o.containerId) - .join(', ')})`, + reason: { + user: `There is an issue with the configuration of the Worker, it doesn't have access to any of the target PackageContainers`, + tech: `Worker doesn't have access to any of the target packageContainers (${checks.targets + .map((o) => o.containerId) + .join(', ')})`, + }, } } } - const hasAccessTo: string[] = [] - if (accessSourcePackageContainer) { - hasAccessTo.push( - `source "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"` - ) - } - if (accessTargetPackageContainer) { - hasAccessTo.push( - `target "${accessTargetPackageContainer.packageContainer.label}" through accessor "${accessTargetPackageContainer.accessorId}"` - ) - } + // const hasAccessTo: string[] = [] + // if (accessSourcePackageContainer) { + // hasAccessTo.push( + // `source "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"` + // ) + // } + // if (accessTargetPackageContainer) { + // hasAccessTo.push( + // `target "${accessTargetPackageContainer.packageContainer.label}" through accessor "${accessTargetPackageContainer.accessorId}"` + // ) + // } return { support: true, - reason: `Has access to ${hasAccessTo.join(' and ')}`, + // reason: `Has access to ${hasAccessTo.join(' and ')}`, } } export type LookupPackageContainer = | { + ready: true accessor: AccessorOnPackage.Any handle: GenericAccessorHandle - ready: true - reason: string + // reason: Reason } | { - accessor: undefined - handle: undefined ready: false - reason: string + accessor: undefined + // handle: undefined + reason: Reason } interface LookupChecks { /** Check that the accessor-handle supports reading */ @@ -95,7 +101,7 @@ export async function lookupAccessorHandles( checks: LookupChecks ): Promise> { /** undefined if all good, error string otherwise */ - let errorReason: undefined | string = 'No target found' + let errorReason: undefined | Reason = { user: 'No target found', tech: 'No target found' } // See if the file is available at any of the targets: for (const { packageContainer, accessorId, accessor } of prioritizeAccessors(packageContainers)) { @@ -111,22 +117,33 @@ export async function lookupAccessorHandles( if (checks.read) { // Check that the accessor-handle supports reading: - const issueHandleRead = handle.checkHandleRead() - if (issueHandleRead) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issueHandleRead}` + const readResult = handle.checkHandleRead() + if (!readResult.success) { + errorReason = { + user: `There is an issue with the configuration for the PackageContainer "${ + packageContainer.label + }" (on accessor "${accessor.label || accessorId}"): ${readResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + readResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.readPackage) { // Check that the Package can be read: - const issuePackageReadAccess = await handle.checkPackageReadAccess() - if (issuePackageReadAccess) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issuePackageReadAccess}` + const readResult = await handle.checkPackageReadAccess() + if (!readResult.success) { + errorReason = { + user: `Can't read the Package from PackageContainer "${packageContainer.label}" (on accessor "${ + accessor.label || accessorId + }"), due to: ${readResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + readResult.reason.tech + }`, + } + continue // Maybe next accessor works? } } @@ -134,30 +151,45 @@ export async function lookupAccessorHandles( // Check that the version of the Package is correct: const actualSourceVersion = await handle.getPackageActualVersion() - const issuePackageVersion = compareActualExpectVersions(actualSourceVersion, checks.packageVersion) - if (issuePackageVersion) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issuePackageVersion}` + const compareVersionResult = compareActualExpectVersions(actualSourceVersion, checks.packageVersion) + if (!compareVersionResult.success) { + errorReason = { + user: `Won't read from the package, due to: ${compareVersionResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + compareVersionResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.write) { // Check that the accessor-handle supports writing: - const issueHandleWrite = handle.checkHandleWrite() - if (issueHandleWrite) { - errorReason = `${packageContainer.label}: lookupTargets: Accessor "${ - accessor.label || accessorId - }": ${issueHandleWrite}` + const writeResult = handle.checkHandleWrite() + if (!writeResult.success) { + errorReason = { + user: `There is an issue with the configuration for the PackageContainer "${ + packageContainer.label + }" (on accessor "${accessor.label || accessorId}"): ${writeResult.reason.user}`, + tech: `${packageContainer.label}: lookupTargets: Accessor "${accessor.label || accessorId}": ${ + writeResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.writePackageContainer) { // Check that it is possible to write to write to the package container: - const issuePackage = await handle.checkPackageContainerWriteAccess() - if (issuePackage) { - errorReason = `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${issuePackage}` + const writeAccessResult = await handle.checkPackageContainerWriteAccess() + if (!writeAccessResult.success) { + errorReason = { + user: `Can't write to the PackageContainer "${packageContainer.label}" (on accessor "${ + accessor.label || accessorId + }"), due to: ${writeAccessResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + writeAccessResult.reason.tech + }`, + } continue // Maybe next accessor works? } } @@ -168,15 +200,14 @@ export async function lookupAccessorHandles( accessor: accessor, handle: handle, ready: true, - reason: `Can access target "${packageContainer.label}" through accessor "${ - accessor.label || accessorId - }"`, + // reason: `Can access target "${packageContainer.label}" through accessor "${ + // accessor.label || accessorId + // }"`, } } } return { accessor: undefined, - handle: undefined, ready: false, reason: errorReason, } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts index 711fbd00..dbacdbd3 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts @@ -135,7 +135,7 @@ export function scanFieldOrder( } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Source Clip not found`) + if (!httpStreamURL.success) throw new Error(`Source Clip not found (${httpStreamURL.reason.tech})`) args.push('-seekable 0') args.push(`-i "${httpStreamURL.fullURL}"`) @@ -235,7 +235,7 @@ export function scanMoreInfo( } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Source Clip not found`) + if (!httpStreamURL.success) throw new Error(`Source Clip not found (${httpStreamURL.reason.tech})`) args.push('-seekable 0') args.push(`-i "${httpStreamURL.fullURL}"`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index 2053149e..3e4a370d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -30,7 +30,14 @@ export const MediaFilePreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, targets: exp.endRequirement.targets, @@ -54,13 +61,13 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -72,13 +79,32 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupSource = await lookupPreviewSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Preview does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -86,11 +112,23 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No preview file found' } + return { + fulfilled: false, + reason: { user: `The preview needs to be re-generated`, tech: `No preview metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Preview version doesn't match preview file` } + return { + fulfilled: false, + reason: { + user: `The preview needs to be re-generated`, + tech: `Preview version doesn't match source file`, + }, + } } else { - return { fulfilled: true, reason: 'Preview already matches preview file' } + return { + fulfilled: true, + // reason: { user: `Preview already matches preview file`, tech: `Preview already matches preview file` }, + } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -128,8 +166,8 @@ export const MediaFilePreview: ExpectationWindowsHandler = { // On cancel ffMpegProcess?.cancel() }).do(async () => { - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -179,7 +217,10 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } @@ -199,16 +240,31 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove preview file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed preview file "${exp.endRequirement.content.filePath}" from target` } + return { + removed: true, + // reason: { user: ``, tech: `Removed preview file "${exp.endRequirement.content.filePath}" from target` }, + } }, } function isMediaFilePreview(exp: Expectation.Any): exp is Expectation.MediaFilePreview { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index 234bd46e..05b468f1 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -36,7 +36,14 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -59,13 +66,13 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -77,13 +84,32 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const lookupSource = await lookupThumbnailSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Thumbnail does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -91,11 +117,20 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No thumbnail file found' } + return { + fulfilled: false, + reason: { user: `The thumbnail needs to be re-generated`, tech: `No thumbnail metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Thumbnail version doesn't match thumbnail file` } + return { + fulfilled: false, + reason: { + user: `The thumbnail needs to be re-generated`, + tech: `Thumbnail version doesn't match thumbnail file`, + }, + } } else { - return { fulfilled: true, reason: 'Thumbnail already matches thumbnail file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -138,9 +173,9 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { ) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) { - throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) { + throw new Error(tryReadPackage.reason.tech) } const actualSourceVersion = await sourceHandle.getPackageActualVersion() @@ -201,7 +236,10 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) }) @@ -217,11 +255,29 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isMediaFileThumbnail(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } - await lookupTarget.handle.removePackage() + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed thumbnail' } + return { removed: true } }, } function isMediaFileThumbnail(exp: Expectation.Any): exp is Expectation.MediaFileThumbnail { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts index 8fc6d492..e2f9f579 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts @@ -35,7 +35,14 @@ export const PackageDeepScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFProbe) return { support: false, reason: 'Cannot access FFProbe executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -59,13 +66,13 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const lookupTarget = await lookupDeepScanSources(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -77,10 +84,22 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const lookupSource = await lookupDeepScanSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupDeepScanTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -100,7 +119,7 @@ export const PackageDeepScan: ExpectationWindowsHandler = { } return { fulfilled: false, reason: packageInfoSynced.reason } } else { - return { fulfilled: true, reason: packageInfoSynced.reason } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -139,8 +158,8 @@ export const PackageDeepScan: ExpectationWindowsHandler = { if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const sourceVersionHash = hashObj(actualSourceVersion) @@ -187,7 +206,10 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Scan completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Scan completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } else { @@ -202,12 +224,29 @@ export const PackageDeepScan: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isPackageDeepScan(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupDeepScanTargets(worker, exp) - if (!lookupTarget.ready) return { removed: false, reason: `Not able to access target: ${lookupTarget.reason}` } + if (!lookupTarget.ready) + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } if (!isCorePackageInfoAccessorHandle(lookupTarget.handle)) throw new Error(`Target AccessHandler type is wrong`) - await lookupTarget.handle.removePackageInfo('deepScan', exp) + try { + await lookupTarget.handle.removePackageInfo('deepScan', exp) + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove the scan result due to an internal error`, + tech: `Cannot remove CorePackageInfo: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed scan info from Store' } + return { removed: true } }, } function isPackageDeepScan(exp: Expectation.Any): exp is Expectation.PackageDeepScan { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts index c436ea1a..8c7e59aa 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts @@ -33,7 +33,14 @@ export const PackageScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFProbe) return { support: false, reason: 'Cannot access FFProbe executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -57,13 +64,12 @@ export const PackageScan: ExpectationWindowsHandler = { const lookupTarget = await lookupScanSources(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -75,10 +81,22 @@ export const PackageScan: ExpectationWindowsHandler = { const lookupSource = await lookupScanSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupScanTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -98,7 +116,7 @@ export const PackageScan: ExpectationWindowsHandler = { } return { fulfilled: false, reason: packageInfoSynced.reason } } else { - return { fulfilled: true, reason: packageInfoSynced.reason } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -137,8 +155,8 @@ export const PackageScan: ExpectationWindowsHandler = { if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const sourceVersionHash = hashObj(actualSourceVersion) @@ -164,7 +182,10 @@ export const PackageScan: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Scan completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Scan completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } else { @@ -179,12 +200,29 @@ export const PackageScan: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isPackageScan(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupScanTargets(worker, exp) - if (!lookupTarget.ready) return { removed: false, reason: `Not able to access target: ${lookupTarget.reason}` } + if (!lookupTarget.ready) + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } if (!isCorePackageInfoAccessorHandle(lookupTarget.handle)) throw new Error(`Target AccessHandler type is wrong`) - await lookupTarget.handle.removePackageInfo('scan', exp) + try { + await lookupTarget.handle.removePackageInfo('scan', exp) + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove the scan result due to an internal error`, + tech: `Cannot remove CorePackageInfo: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed scan info from Store' } + return { removed: true } }, } function isPackageScan(exp: Expectation.Any): exp is Expectation.PackageScan { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts index ad54c3d5..e66297a4 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts @@ -41,7 +41,14 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } if (lookupTarget.accessor.type === Accessor.AccessType.QUANTEL) { - if (!lookupTarget.accessor.serverId) return { ready: false, reason: `Target Accessor has no serverId set` } + if (!lookupTarget.accessor.serverId) + return { + ready: false, + reason: { + user: `There is an issue in the settings: The Accessor "${lookupTarget.handle.accessorId}" has no serverId set`, + tech: `Target Accessor "${lookupTarget.handle.accessorId}" has no serverId set`, + }, + } } // // Do a check, to ensure that the source and targets are Quantel: @@ -51,13 +58,13 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { // return { ready: false, reason: `Target Accessor type not supported: ${lookupSource.accessor.type}` } // Also check if we actually can read from the package: - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -69,16 +76,32 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issuePackage = await lookupTarget.handle.checkPackageReadAccess() - if (issuePackage) { - return { fulfilled: false, reason: `Clip not found: ${issuePackage.toString()}` } + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } } // Does the clip exist on the target? const actualTargetVersion = await lookupTarget.handle.getPackageActualVersion() - if (!actualTargetVersion) return { fulfilled: false, reason: `No package found on target` } + if (!actualTargetVersion) + return { + fulfilled: false, + reason: { user: `No clip found on target`, tech: `No clip found on target` }, + } const lookupSource = await lookupCopySources(worker, exp) if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) @@ -91,15 +114,12 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { makeUniversalVersion(actualSourceVersion), makeUniversalVersion(actualTargetVersion) ) - if (issueVersions) { - return { fulfilled: false, reason: issueVersions } + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } } return { fulfilled: true, - reason: `Clip "${ - exp.endRequirement.content.guid || exp.endRequirement.content.title - }" already exists on target`, } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -210,7 +230,10 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) })().catch((err) => { @@ -232,18 +255,29 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove clip: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove clip due to an internal error`, + tech: `Cannot remove preview clip: ${err.toString()}`, + }, + } } return { removed: true, - reason: `Removed clip "${exp.endRequirement.content.guid || exp.endRequirement.content.title}" from target`, } }, } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index 9a948f39..8f3a65e1 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -29,7 +29,14 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, targets: exp.endRequirement.targets, @@ -53,22 +60,26 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryResult = await lookupSource.handle.tryPackageRead() + if (!tryResult.success) return { ready: false, reason: tryResult.reason } // This is a bit special, as we use the Quantel HTTP-transformer to get a HLS-stream of the video: if (!isQuantelClipAccessorHandle(lookupSource.handle)) throw new Error(`Source AccessHandler type is wrong`) + const httpStreamURL = await lookupSource.handle.getTransformerStreamURL() - if (!httpStreamURL) return { ready: false, reason: `Preview source not found` } + if (!httpStreamURL.success) + return { + ready: false, + reason: httpStreamURL.reason, + } const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, httpStreamURL) - const issueReadingHTTP = await sourceHTTPHandle.tryPackageRead() - if (issueReadingHTTP) return { ready: false, reason: issueReadingHTTP } + const tryReadingHTTP = await sourceHTTPHandle.tryPackageRead() + if (!tryReadingHTTP.success) return { ready: false, reason: tryReadingHTTP.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -80,13 +91,32 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupSource = await lookupPreviewSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Preview does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -94,11 +124,20 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No preview file found' } + return { + fulfilled: false, + reason: { user: `The preview needs to be re-generated`, tech: `No preview metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Preview version doesn't match preview file` } + return { + fulfilled: false, + reason: { + user: `The preview needs to be re-generated`, + tech: `Preview version doesn't match source file`, + }, + } } else { - return { fulfilled: true, reason: 'Preview already matches preview file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -132,7 +171,7 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { // This is a bit special, as we use the Quantel HTTP-transformer to get a HLS-stream of the video: const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Preview source not found`) + if (!httpStreamURL.success) throw new Error(httpStreamURL.reason.tech) const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, httpStreamURL) let ffMpegProcess: FFMpegProcess | undefined @@ -141,7 +180,7 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { ffMpegProcess?.cancel() }).do(async () => { const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + if (!issueReadPackage.success) throw new Error(issueReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -191,7 +230,10 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } @@ -211,16 +253,28 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove preview file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed preview file "${exp.endRequirement.content.filePath}" from target` } + return { removed: true } }, } function isQuantelClipPreview(exp: Expectation.Any): exp is Expectation.QuantelClipPreview { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index f0a7c9c0..6dccb2e1 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -8,6 +8,7 @@ import { ReturnTypeIsExpectationReadyToStartWorkingOn, ReturnTypeRemoveExpectation, literal, + Reason, } from '@shared/api' import { getStandardCost } from '../lib/lib' import { GenericWorker } from '../../../worker' @@ -34,7 +35,14 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (!windowsWorker.hasFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker: FFMpeg not found', + tech: 'Cannot access FFMpeg executable', + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -57,21 +65,24 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryResult = await lookupSource.handle.tryPackageRead() + if (!tryResult.success) return { ready: false, reason: tryResult.reason } // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail: const thumbnailURL = await getThumbnailURL(exp, lookupSource) - if (!thumbnailURL) return { ready: false, reason: `Thumbnail source not found` } + if (!thumbnailURL.success) + return { + ready: false, + reason: thumbnailURL.reason, + } const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, thumbnailURL) - const issueReadingHTTP = await sourceHTTPHandle.tryPackageRead() - if (issueReadingHTTP) return { ready: false, reason: issueReadingHTTP } + const tryReadingHTTP = await sourceHTTPHandle.tryPackageRead() + if (!tryReadingHTTP.success) return { ready: false, reason: tryReadingHTTP.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -83,13 +94,32 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const lookupSource = await lookupThumbnailSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Thumbnail does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const expectedTargetMetadata: Metadata = getMetadata(exp, actualSourceVersion) @@ -97,11 +127,20 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const targetMetadata = await lookupTarget.handle.fetchMetadata() if (!targetMetadata) { - return { fulfilled: false, reason: 'No thumbnail file found' } + return { + fulfilled: false, + reason: { user: `The thumbnail needs to be re-generated`, tech: `No thumbnail metadata file found` }, + } } else if (targetMetadata.sourceVersionHash !== expectedTargetMetadata.sourceVersionHash) { - return { fulfilled: false, reason: `Thumbnail version hash doesn't match thumnail file` } + return { + fulfilled: false, + reason: { + user: `The thumbnail needs to be re-generated`, + tech: `Thumbnail version doesn't match thumbnail file`, + }, + } } else { - return { fulfilled: true, reason: 'Thumbnail already matches thumnail file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -136,7 +175,7 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail: const thumbnailURL = await getThumbnailURL(exp, lookupSource) - if (!thumbnailURL) throw new Error(`Can't start working due to source: Thumbnail url not found`) + if (!thumbnailURL.success) throw new Error(`Can't start working due to source: ${thumbnailURL.reason}`) const sourceHTTPHandle = getSourceHTTPHandle(worker, sourceHandle, thumbnailURL) let wasCancelled = false @@ -184,7 +223,10 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( targetMetadata.sourceVersionHash, - `Thumbnail fetched in ${Math.round(duration / 100) / 10}s`, + { + user: `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) })().catch((err) => { @@ -204,11 +246,29 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isQuantelClipThumbnail(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } - await lookupTarget.handle.removePackage() + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed thumbnail' } + return { removed: true } }, } function isQuantelClipThumbnail(exp: Expectation.Any): exp is Expectation.QuantelClipThumbnail { @@ -233,13 +293,20 @@ function getMetadata(exp: Expectation.QuantelClipThumbnail, actualSourceVersion: async function getThumbnailURL( exp: Expectation.QuantelClipThumbnail, lookupSource: LookupPackageContainer -): Promise<{ baseURL: string; url: string } | undefined> { +): Promise<{ success: true; baseURL: string; url: string } | { success: false; reason: Reason }> { if (!lookupSource.accessor) throw new Error(`Source accessor not set!`) if (lookupSource.accessor.type !== Accessor.AccessType.QUANTEL) throw new Error(`Source accessor should have been a Quantel ("${lookupSource.accessor.type}")`) if (!isQuantelClipAccessorHandle(lookupSource.handle)) throw new Error(`Source AccessHandler type is wrong`) - if (!lookupSource.accessor.transformerURL) return undefined + if (!lookupSource.accessor.transformerURL) + return { + success: false, + reason: { + user: `transformerURL is not set in settings`, + tech: `transformerURL not set`, + }, + } const clip = await lookupSource.handle.getClip() if (clip) { @@ -255,11 +322,19 @@ async function getThumbnailURL( } return { + success: true, baseURL: lookupSource.accessor.transformerURL, url: `/quantel/homezone/clips/stills/${clip.ClipID}/${frame}.${width ? width + '.' : ''}jpg`, } + } else { + return { + success: false, + reason: { + user: `Source clip not found`, + tech: `Source clip not found`, + }, + } } - return undefined } export function getSourceHTTPHandle( worker: GenericWorker, diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts index ebfbbdd5..8e29fb4e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts @@ -7,11 +7,12 @@ import { getAccessorCost, getAccessorStaticHandle } from '../../../accessorHandl import { GenericWorker } from '../../../worker' import { Expectation } from '@shared/api' import { prioritizeAccessors } from '../../../lib/lib' +import { AccessorHandlerResult } from '../../../accessorHandlers/genericHandle' export function compareActualExpectVersions( actualVersion: Expectation.Version.Any, expectVersion: Expectation.Version.ExpectAny -): undefined | string { +): AccessorHandlerResult { const expectProperties = makeUniversalVersion(expectVersion) const actualProperties = makeUniversalVersion(actualVersion) @@ -20,25 +21,37 @@ export function compareActualExpectVersions( const actual = actualProperties[key] as VersionProperty if (expect.value !== undefined && actual.value && expect.value !== actual.value) { - return `Actual ${actual.name} differ from expected (${expect.value}, ${actual.value})` + return { + success: false, + reason: { + user: 'Actual version differs from expected', + tech: `Actual ${actual.name} differ from expected (${expect.value}, ${actual.value})`, + }, + } } } - return undefined // All good! + return { success: true } } export function compareUniversalVersions( sourceVersion: UniversalVersion, targetVersion: UniversalVersion -): undefined | string { +): AccessorHandlerResult { for (const key of Object.keys(sourceVersion)) { const source = sourceVersion[key] as VersionProperty const target = targetVersion[key] as VersionProperty if (source.value !== target.value) { - return `Target ${source.name} differ from source (${target.value}, ${source.value})` + return { + success: false, + reason: { + user: 'Target version differs from Source', + tech: `Target ${source.name} differ from source (${target.value}, ${source.value})`, + }, + } } } - return undefined // All good! + return { success: true } } export function makeUniversalVersion( diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts index fabc47bf..79676dc0 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts @@ -4,6 +4,7 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, ReturnTypeDisposePackageContainerMonitors, + Reason, } from '@shared/api' import { Accessor, PackageContainer, PackageContainerOnPackage } from '@sofie-automation/blueprints-integration' import { GenericAccessorHandle } from '../../accessorHandlers/genericHandle' @@ -23,39 +24,43 @@ export async function runPackageContainerCronJob( ): Promise { // Quick-check: If there are no cronjobs at all, no need to check: if (!Object.keys(packageContainer.cronjobs).length) { - return { completed: true } // all good + return { success: true } // all good } const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'cronjob') - if (!lookup.ready) return { completed: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.runCronJob(packageContainer) - if (result) return { completed: false, reason: result } - else return { completed: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else return { success: true } // all good } export async function setupPackageContainerMonitors( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'monitor') - if (!lookup.ready) return { setupOk: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.setupPackageContainerMonitors(packageContainer) - if (result) return { setupOk: false, reason: result, monitors: {} } - else return { setupOk: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else + return { + success: true, + monitors: {}, // To me implemented: monitor ids + } } export async function disposePackageContainerMonitors( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'monitor') - if (!lookup.ready) return { disposed: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.disposePackageContainerMonitors(packageContainer) - if (result) return { disposed: false, reason: result } - else return { disposed: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else return { success: true } // all good } function checkWorkerHasAccessToPackageContainer( @@ -72,12 +77,15 @@ function checkWorkerHasAccessToPackageContainer( if (accessSourcePackageContainer) { return { support: true, - reason: `Has access to packageContainer "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"`, + // reason: `Has access to packageContainer "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"`, } } else { return { support: false, - reason: `Doesn't have access to the packageContainer (${containerId})`, + reason: { + user: `Worker doesn't support working with PackageContainer "${containerId}" (check settings?)`, + tech: `Worker doesn't have any access to the PackageContainer "${containerId}"`, + }, } } } @@ -121,11 +129,11 @@ export type LookupPackageContainer = accessor: Accessor.Any handle: GenericAccessorHandle ready: true - reason: string + reason: Reason } | { accessor: undefined handle: undefined ready: false - reason: string + reason: Reason } diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 6a428e16..ccbe43cd 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -191,16 +191,21 @@ export class WorkerAgent { this.logger.error(err) }) }) - workInProgress.on('error', (error) => { + workInProgress.on('error', (error: string) => { this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to error. (${this.currentJobs.length})` ) - expectedManager.api.wipEventError(wipId, error).catch((err) => { - this.logger.error('Error in wipEventError') - this.logger.error(err) - }) + expectedManager.api + .wipEventError(wipId, { + user: 'Work aborted due to an error', + tech: error, + }) + .catch((err) => { + this.logger.error('Error in wipEventError') + this.logger.error(err) + }) delete this.worksInProgress[`${wipId}`] }) workInProgress.on('done', (actualVersionHash, reason, result) => { diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index d8811464..abc9a17a 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -3,7 +3,7 @@ import * as Workforce from '@shared/workforce' import * as Worker from '@shared/worker' import * as Winston from 'winston' -import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, SingleAppConfig } from '@shared/api' +import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, Reason, SingleAppConfig } from '@shared/api' // import deepExtend from 'deep-extend' import { ExpectationManager, ExpectationManagerCallbacks } from '@shared/expectation-manager' import { CoreMockAPI } from './coreMockAPI' @@ -120,7 +120,7 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise { if (!expectationStatuses[expectationId]) { @@ -202,7 +202,7 @@ export interface ExpectationStatuses { statusInfo: { status?: string progress?: number - statusReason?: string + statusReason?: Reason } } } From 9382f29c94681958219f9767f09d80f7a05754b4 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 11 Jun 2021 16:01:35 +0200 Subject: [PATCH 23/67] fix: rename HTTP accessor to HTTPProxy, since it is in fact a specific http-server of package manager --- shared/packages/api/src/expectationApi.ts | 1 + .../src/worker/accessorHandlers/accessor.ts | 14 +++++++------- .../accessorHandlers/{http.ts => httpProxy.ts} | 12 ++++++------ shared/packages/worker/src/worker/lib/lib.ts | 1 + .../expectationHandlers/fileCopy.ts | 10 +++++----- .../expectationHandlers/lib/ffmpeg.ts | 8 ++++---- .../expectationHandlers/lib/scan.ts | 18 +++++++++--------- .../expectationHandlers/mediaFilePreview.ts | 6 +++--- .../expectationHandlers/mediaFileThumbnail.ts | 12 ++++++------ .../expectationHandlers/packageDeepScan.ts | 6 +++--- .../expectationHandlers/packageScan.ts | 6 +++--- .../expectationHandlers/quantelClipPreview.ts | 6 +++--- .../quantelClipThumbnail.ts | 16 ++++++++-------- 13 files changed, 59 insertions(+), 57 deletions(-) rename shared/packages/worker/src/worker/accessorHandlers/{http.ts => httpProxy.ts} (96%) diff --git a/shared/packages/api/src/expectationApi.ts b/shared/packages/api/src/expectationApi.ts index 46a12292..5234bfa6 100644 --- a/shared/packages/api/src/expectationApi.ts +++ b/shared/packages/api/src/expectationApi.ts @@ -262,6 +262,7 @@ export namespace Expectation { | AccessorOnPackage.LocalFolder | AccessorOnPackage.FileShare | AccessorOnPackage.HTTP + | AccessorOnPackage.HTTPProxy } } /** Defines a PackageContainer for CorePackage (A collection in Sofie-Core accessible through an API). */ diff --git a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts index f7121ff5..f18c6864 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts @@ -3,7 +3,7 @@ import { GenericWorker } from '../worker' import { CorePackageInfoAccessorHandle } from './corePackageInfo' import { FileShareAccessorHandle } from './fileShare' import { GenericAccessorHandle } from './genericHandle' -import { HTTPAccessorHandle } from './http' +import { HTTPProxyAccessorHandle } from './httpProxy' import { LocalFolderAccessorHandle } from './localFolder' import { QuantelAccessorHandle } from './quantel' @@ -28,8 +28,8 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { return LocalFolderAccessorHandle } else if (accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO) { return CorePackageInfoAccessorHandle - } else if (accessor.type === Accessor.AccessType.HTTP) { - return HTTPAccessorHandle + } else if (accessor.type === Accessor.AccessType.HTTP_PROXY) { + return HTTPProxyAccessorHandle } else if (accessor.type === Accessor.AccessType.FILE_SHARE) { return FileShareAccessorHandle } else if (accessor.type === Accessor.AccessType.QUANTEL) { @@ -53,10 +53,10 @@ export function isCorePackageInfoAccessorHandle( ): accessorHandler is CorePackageInfoAccessorHandle { return accessorHandler.type === CorePackageInfoAccessorHandle.type } -export function isHTTPAccessorHandle( +export function isHTTPProxyAccessorHandle( accessorHandler: GenericAccessorHandle -): accessorHandler is HTTPAccessorHandle { - return accessorHandler.type === HTTPAccessorHandle.type +): accessorHandler is HTTPProxyAccessorHandle { + return accessorHandler.type === HTTPProxyAccessorHandle.type } export function isFileShareAccessorHandle( accessorHandler: GenericAccessorHandle @@ -82,7 +82,7 @@ export function getAccessorCost(accessorType: Accessor.AccessType | undefined): // -------------------------------------------------------- case Accessor.AccessType.FILE_SHARE: return 2 - case Accessor.AccessType.HTTP: + case Accessor.AccessType.HTTP_PROXY: return 3 case undefined: diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts similarity index 96% rename from shared/packages/worker/src/worker/accessorHandlers/http.ts rename to shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts index 6efcf286..258382d3 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts @@ -13,9 +13,9 @@ import FormData from 'form-data' import AbortController from 'abort-controller' import { assertNever } from '../lib/lib' -/** Accessor handle for accessing files in a local folder */ -export class HTTPAccessorHandle extends GenericAccessorHandle { - static readonly type = 'http' +/** Accessor handle for accessing files in HTTP- */ +export class HTTPProxyAccessorHandle extends GenericAccessorHandle { + static readonly type = 'http-proxy' private content: { onlyContainerAccess?: boolean filePath?: string @@ -28,7 +28,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle( targetHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle, + | HTTPProxyAccessorHandle, actualSourceVersionHash: string, onDone: () => Promise ): Promise { @@ -79,7 +79,7 @@ export async function runffMpeg( } else if (isFileShareAccessorHandle(targetHandle)) { await targetHandle.prepareFileAccess() args.push(`"${targetHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(targetHandle)) { + } else if (isHTTPProxyAccessorHandle(targetHandle)) { pipeStdOut = true args.push('pipe:1') // pipe output to stdout } else { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts index dbacdbd3..53b654d3 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts @@ -4,7 +4,7 @@ import { isQuantelClipAccessorHandle, isLocalFolderAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, } from '../../../../accessorHandlers/accessor' import { LocalFolderAccessorHandle } from '../../../../accessorHandlers/localFolder' import { QuantelAccessorHandle } from '../../../../accessorHandlers/quantel' @@ -13,7 +13,7 @@ import { assertNever } from '../../../../lib/lib' import { FieldOrder, ScanAnomaly } from './coreApi' import { generateFFProbeFromClipData } from './quantelFormats' import { FileShareAccessorHandle } from '../../../../accessorHandlers/fileShare' -import { HTTPAccessorHandle } from '../../../../accessorHandlers/http' +import { HTTPProxyAccessorHandle } from '../../../../accessorHandlers/httpProxy' interface FFProbeScanResult { // to be defined... @@ -25,14 +25,14 @@ export function scanWithFFProbe( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle ): CancelablePromise { return new CancelablePromise(async (resolve, reject, onCancel) => { if ( isLocalFolderAccessorHandle(sourceHandle) || isFileShareAccessorHandle(sourceHandle) || - isHTTPAccessorHandle(sourceHandle) + isHTTPProxyAccessorHandle(sourceHandle) ) { let inputPath: string let filePath: string @@ -43,7 +43,7 @@ export function scanWithFFProbe( await sourceHandle.prepareFileAccess() inputPath = sourceHandle.fullPath filePath = sourceHandle.filePath - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullUrl filePath = sourceHandle.filePath } else { @@ -103,7 +103,7 @@ export function scanFieldOrder( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle, targetVersion: Expectation.PackageDeepScan['endRequirement']['version'] ): CancelablePromise { @@ -130,7 +130,7 @@ export function scanFieldOrder( } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() args.push(`-i "${sourceHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { args.push(`-i "${sourceHandle.fullUrl}"`) } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() @@ -178,7 +178,7 @@ export function scanMoreInfo( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle, previouslyScanned: FFProbeScanResult, targetVersion: Expectation.PackageDeepScan['endRequirement']['version'], @@ -230,7 +230,7 @@ export function scanMoreInfo( } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() args.push(`-i "${sourceHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { args.push(`-i "${sourceHandle.fullUrl}"`) } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index 3e4a370d..d2642771 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -13,7 +13,7 @@ import { } from '@shared/api' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' @@ -150,14 +150,14 @@ export const MediaFilePreview: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { // We can read the source and write the preview directly. if (!isLocalFolderAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index 05b468f1..b44a0c2b 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -13,7 +13,7 @@ import { GenericWorker } from '../../../worker' import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' @@ -153,23 +153,23 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) && + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) + !isHTTPProxyAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) @@ -207,7 +207,7 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() inputPath = sourceHandle.fullPath - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullUrl } else { assertNever(sourceHandle) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts index e2f9f579..51a6ead8 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts @@ -14,7 +14,7 @@ import { import { isCorePackageInfoAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -143,14 +143,14 @@ export const PackageDeepScan: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupSource.accessor.type === Accessor.AccessType.HTTP || + lookupSource.accessor.type === Accessor.AccessType.HTTP_PROXY || lookupSource.accessor.type === Accessor.AccessType.QUANTEL) && lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO ) { if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) && + !isHTTPProxyAccessorHandle(sourceHandle) && !isQuantelClipAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts index 8c7e59aa..bcebf546 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts @@ -14,7 +14,7 @@ import { import { isCorePackageInfoAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -141,14 +141,14 @@ export const PackageScan: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupSource.accessor.type === Accessor.AccessType.HTTP || + lookupSource.accessor.type === Accessor.AccessType.HTTP_PROXY || lookupSource.accessor.type === Accessor.AccessType.QUANTEL) && lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO ) { if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) && + !isHTTPProxyAccessorHandle(sourceHandle) && !isQuantelClipAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index 8f3a65e1..5ffd612a 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -13,7 +13,7 @@ import { } from '@shared/api' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -158,14 +158,14 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.QUANTEL && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { // We can read the source and write the preview directly. if (!isQuantelClipAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index 6dccb2e1..982d4ec9 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -16,14 +16,14 @@ import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { getAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' import { checkWorkerHasAccessToPackageContainersOnPackage, lookupAccessorHandles, LookupPackageContainer } from './lib' import { GenericAccessorHandle, PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' -import { HTTPAccessorHandle } from '../../../accessorHandlers/http' +import { HTTPProxyAccessorHandle } from '../../../accessorHandlers/httpProxy' import { WindowsWorker } from '../windowsWorker' /** @@ -162,14 +162,14 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.QUANTEL && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { if (!isQuantelClipAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) @@ -340,15 +340,15 @@ export function getSourceHTTPHandle( worker: GenericWorker, sourceHandle: GenericAccessorHandle, thumbnailURL: { baseURL: string; url: string } -): HTTPAccessorHandle { +): HTTPProxyAccessorHandle { // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail, // so we have a QUANTEL source, but we construct an HTTP source from it to use instead: const handle = getAccessorHandle( worker, sourceHandle.accessorId + '__http', - literal({ - type: Accessor.AccessType.HTTP, + literal({ + type: Accessor.AccessType.HTTP_PROXY, baseUrl: thumbnailURL.baseURL, // networkId?: string url: thumbnailURL.url, @@ -356,7 +356,7 @@ export function getSourceHTTPHandle( { filePath: thumbnailURL.url }, {} ) - if (!isHTTPAccessorHandle(handle)) throw new Error(`getSourceHTTPHandle: got a non-HTTP handle!`) + if (!isHTTPProxyAccessorHandle(handle)) throw new Error(`getSourceHTTPHandle: got a non-HTTP handle!`) return handle } From b814089ae0d668e5dc5768ccb654b5fb4d965335 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 11 Jun 2021 16:02:33 +0200 Subject: [PATCH 24/67] feat: preliminary implementation of JSONDagaCopy expectation and HTTP accessor --- .../generic/src/expectationGenerator.ts | 59 +++ shared/packages/api/src/expectationApi.ts | 19 + .../src/worker/accessorHandlers/http.ts | 411 ++++++++++++++++++ .../expectationHandlers/jsonDataCopy.ts | 282 ++++++++++++ 4 files changed, 771 insertions(+) create mode 100644 shared/packages/worker/src/worker/accessorHandlers/http.ts create mode 100644 shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 0b92e0fc..8742d782 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -24,6 +24,14 @@ export interface ExpectedPackageWrapQuantel extends ExpectedPackageWrap { accessors: ExpectedPackage.ExpectedPackageQuantelClip['sources'][0]['accessors'] }[] } +export interface ExpectedPackageWrapJSONData extends ExpectedPackageWrap { + expectedPackage: ExpectedPackage.ExpectedPackageJSONData + sources: { + containerId: string + label: string + accessors: ExpectedPackage.ExpectedPackageJSONData['sources'][0]['accessors'] + }[] +} type GenerateExpectation = Expectation.Base & { sideEffect?: ExpectedPackage.Base['sideEffect'] @@ -134,6 +142,8 @@ export function generateExpectations( exp = generateMediaFileCopy(managerId, packageWrap, settings) } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { exp = generateQuantelCopy(managerId, packageWrap) + } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.JSON_DATA) { + exp = generateJsonDataCopy(managerId, packageWrap) } if (exp) { prioritizeExpectation(packageWrap, exp) @@ -663,6 +673,55 @@ function generateQuantelClipPreview( }) } +function generateJsonDataCopy( + managerId: string, + expWrap: ExpectedPackageWrap, + settings: PackageManagerSettings +): Expectation.JSONDataCopy { + const expWrapMediaFile = expWrap as ExpectedPackageWrapJSONData + + const exp: Expectation.JsonDataCopy = { + id: '', // set later + priority: expWrap.priority * 10 || 0, + managerId: managerId, + fromPackages: [ + { + id: expWrap.expectedPackage._id, + expectedContentVersionHash: expWrap.expectedPackage.contentVersionHash, + }, + ], + type: Expectation.Type.JSON_DATA_COPY, + statusReport: { + label: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}"`, + description: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}" from "${JSON.stringify( + expWrapMediaFile.sources + )}"`, + requiredForPlayout: true, + displayRank: 0, + sendReport: !expWrap.external, + }, + + startRequirement: { + sources: expWrapMediaFile.sources, + }, + + endRequirement: { + targets: expWrapMediaFile.targets as [Expectation.SpecificPackageContainerOnPackage.CorePackage], + content: expWrapMediaFile.expectedPackage.content, + version: { + type: Expectation.Version.Type.CORE_PACKAGE_INFO, + ...expWrapMediaFile.expectedPackage.version, + }, + }, + workOptions: { + removeDelay: settings.delayRemoval, + useTemporaryFilePath: settings.useTemporaryFilePath, + }, + } + exp.id = hashObj(exp.endRequirement) + return exp +} + // function generateMediaFileHTTPCopy(expectation: Expectation.FileCopy): Expectation.FileCopy { // // Copy file to HTTP: (TMP!) // const tmpCopy: Expectation.FileCopy = { diff --git a/shared/packages/api/src/expectationApi.ts b/shared/packages/api/src/expectationApi.ts index 5234bfa6..d4ca2a9d 100644 --- a/shared/packages/api/src/expectationApi.ts +++ b/shared/packages/api/src/expectationApi.ts @@ -23,6 +23,7 @@ export namespace Expectation { // | QuantelClipDeepScan | QuantelClipThumbnail | QuantelClipPreview + | JsonDataCopy /** Defines the Expectation type, used to separate the different Expectations */ export enum Type { @@ -38,6 +39,8 @@ export namespace Expectation { // QUANTEL_CLIP_DEEP_SCAN = 'quantel_clip_deep_scan', QUANTEL_CLIP_THUMBNAIL = 'quantel_clip_thumbnail', QUANTEL_CLIP_PREVIEW = 'quantel_clip_preview', + + JSON_DATA_COPY = 'json_data_copy', } /** Common attributes of all Expectations */ @@ -251,6 +254,22 @@ export namespace Expectation { } workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } + /** Defines a File Copy. A File is to be copied from one of the Sources, to the Target. */ + export interface JsonDataCopy extends Base { + type: Type.JSON_DATA_COPY + + startRequirement: { + sources: SpecificPackageContainerOnPackage.File[] + } + endRequirement: { + targets: [SpecificPackageContainerOnPackage.File] + content: { + path: string + } + version: Version.ExpectedFileOnDisk // maybe something else? + } + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath + } /** Contains definitions of specific PackageContainer types, used in the Expectation-definitions */ // eslint-disable-next-line @typescript-eslint/no-namespace diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts new file mode 100644 index 00000000..cd64fa13 --- /dev/null +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -0,0 +1,411 @@ +import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' +import { + GenericAccessorHandle, + PackageReadInfo, + PackageReadStream, + PutPackageHandler, + AccessorHandlerResult, +} from './genericHandle' +import { Expectation, PackageContainerExpectation } from '@shared/api' +import { GenericWorker } from '../worker' +import fetch from 'node-fetch' +import FormData from 'form-data' +import AbortController from 'abort-controller' +import { assertNever } from '../lib/lib' + +/** Accessor handle for accessing files in a local folder */ +export class HTTPAccessorHandle extends GenericAccessorHandle { + static readonly type = 'http' + private content: { + onlyContainerAccess?: boolean + path?: string + } + private workOptions: Expectation.WorkOptions.RemoveDelay + constructor( + worker: GenericWorker, + public readonly accessorId: string, + private accessor: AccessorOnPackage.HTTP, + content: any, // eslint-disable-line @typescript-eslint/explicit-module-boundary-types + workOptions: any // eslint-disable-line @typescript-eslint/explicit-module-boundary-types + ) { + super(worker, accessorId, accessor, content, HTTPAccessorHandle.type) + + // Verify content data: + if (!content.onlyContainerAccess) { + if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') + } + this.content = content + + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') + throw new Error('Bad input data: workOptions.removeDelay is not a number!') + this.workOptions = workOptions + } + static doYouSupportAccess(worker: GenericWorker, accessor0: AccessorOnPackage.Any): boolean { + const accessor = accessor0 as AccessorOnPackage.HTTP + return !accessor.networkId || worker.location.localNetworkIds.includes(accessor.networkId) + } + checkHandleRead(): AccessorHandlerResult { + if (!this.accessor.allowRead) { + return { + success: false, + reason: { + user: `Not allowed to read`, + tech: `Not allowed to read`, + }, + } + } + return this.checkAccessor() + } + checkHandleWrite(): AccessorHandlerResult { + if (!this.accessor.allowWrite) { + return { + success: false, + reason: { + user: `Not allowed to write`, + tech: `Not allowed to write`, + }, + } + } + return this.checkAccessor() + } + async checkPackageReadAccess(): Promise { + const header = await this.fetchHeader() + + if (header.status >= 400) { + return { + success: false, + reason: { + user: `Got error code ${header.status} when trying to fetch package`, + tech: `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}`, + }, + } + } + return { success: true } + } + async tryPackageRead(): Promise { + + + // TODO: Do a OPTIONS request? + fetch(the url, { + method: 'options' + }) + // 204 or 404 is "not found" + // Access-Control-Allow-Methods should contain GET + return { success: true } + } + async checkPackageContainerWriteAccess(): Promise { + // todo: how to check this? + return { success: true } + } + async getPackageActualVersion(): Promise { + const header = await this.fetchHeader() + + return this.convertHeadersToVersion(header.headers) + } + async removePackage(): Promise { + if (this.workOptions.removeDelay) { + await this.delayPackageRemoval(this.workOptions.removeDelay) + } else { + await this.removeMetadata() + await this.deletePackageIfExists(this.fullUrl) + } + } + async getPackageReadStream(): Promise { + const controller = new AbortController() + const res = await fetch(this.fullUrl, { signal: controller.signal }) + + return { + readStream: res.body, + cancel: () => { + controller.abort() + }, + } + } + async putPackageStream(_sourceStream: NodeJS.ReadableStream): Promise { + throw new Error('HTTP.putPackageStream: Not supported') + } + async getPackageReadInfo(): Promise<{ readInfo: PackageReadInfo; cancel: () => void }> { + throw new Error('HTTP.getPackageReadInfo: Not supported') + } + async putPackageInfo(_readInfo: PackageReadInfo): Promise { + throw new Error('HTTP.putPackageInfo: Not supported') + } + async finalizePackage(): Promise { + // do nothing + } + + async fetchMetadata(): Promise { + return undefined + } + async updateMetadata(_metadata: Metadata): Promise { + // Not supported + } + async removeMetadata(): Promise { + // Not supported + } + + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] + for (const cronjob of cronjobs) { + if (cronjob === 'interval') { + // ignore + } else if (cronjob === 'cleanup') { + await this.removeDuePackages() + } else { + // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: + assertNever(cronjob) + } + } + + return { success: true } + } + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { + const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] + for (const monitor of monitors) { + if (monitor === 'packages') { + // todo: implement monitors + throw new Error('Not implemented yet') + } else { + // Assert that cronjob is of type "never", to ensure that all types of monitors are handled: + assertNever(monitor) + } + } + + return { success: true } + } + async disposePackageContainerMonitors( + _packageContainerExp: PackageContainerExpectation + ): Promise { + // todo: implement monitors + return { success: true } + } + get fullUrl(): string { + return [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + this.path.replace(/^\//, ''), // trim leading slash + ].join('/') + } + + private checkAccessor(): AccessorHandlerResult { + if (this.accessor.type !== Accessor.AccessType.HTTP) { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `HTTP Accessor type is not HTTP ("${this.accessor.type}")!`, + }, + } + } + if (!this.accessor.baseUrl) + return { + success: false, + reason: { + user: `Accessor baseUrl not set`, + tech: `Accessor baseUrl not set`, + }, + } + if (!this.content.onlyContainerAccess) { + if (!this.path) + return { + success: false, + reason: { + user: `filePath not set`, + tech: `filePath not set`, + }, + } + } + return { success: true } + } + private get baseUrl(): string { + if (!this.accessor.baseUrl) throw new Error(`HTTPAccessorHandle: accessor.baseUrl not set!`) + return this.accessor.baseUrl + } + get path(): string { + if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') + const filePath = this.accessor.url || this.content.path + if (!filePath) throw new Error(`HTTPAccessorHandle: path not set!`) + return filePath + } + private convertHeadersToVersion(headers: HTTPHeaders): Expectation.Version.HTTPFile { + return { + type: Expectation.Version.Type.HTTP_FILE, + + contentType: headers.contentType || '', + contentLength: parseInt(headers.contentLength || '0', 10) || 0, + modified: headers.lastModified ? new Date(headers.lastModified).getTime() : 0, + etags: [], // headers.etags, // todo! + } + } + private async fetchHeader() { + const controller = new AbortController() + const res = await fetch(this.fullUrl, { signal: controller.signal }) + + res.body.on('error', () => { + // Swallow the error. Since we're aborting the request, we're not interested in the body anyway. + }) + + const headers: HTTPHeaders = { + contentType: res.headers.get('content-type'), + contentLength: res.headers.get('content-length'), + lastModified: res.headers.get('last-modified'), + etags: res.headers.get('etag'), + } + // We've got the headers, abort the call so we don't have to download the whole file: + controller.abort() + + return { + status: res.status, + statusText: res.statusText, + headers: headers, + } + } + + async delayPackageRemoval(ttl: number): Promise { + const packagesToRemove = await this.getPackagesToRemove() + + const filePath = this.path + + // Search for a pre-existing entry: + let found = false + for (const entry of packagesToRemove) { + if (entry.filePath === filePath) { + // extend the TTL if it was found: + entry.removeTime = Date.now() + ttl + + found = true + break + } + } + if (!found) { + packagesToRemove.push({ + filePath: filePath, + removeTime: Date.now() + ttl, + }) + } + + await this.storePackagesToRemove(packagesToRemove) + } + /** Clear a scheduled later removal of a package */ + async clearPackageRemoval(): Promise { + const packagesToRemove = await this.getPackagesToRemove() + + const filePath = this.path + + let found = false + for (let i = 0; i < packagesToRemove.length; i++) { + const entry = packagesToRemove[i] + if (entry.filePath === filePath) { + packagesToRemove.splice(i, 1) + found = true + break + } + } + if (found) { + await this.storePackagesToRemove(packagesToRemove) + } + } + /** Remove any packages that are due for removal */ + async removeDuePackages(): Promise { + let packagesToRemove = await this.getPackagesToRemove() + + const removedFilePaths: string[] = [] + for (const entry of packagesToRemove) { + // Check if it is time to remove the package: + if (entry.removeTime < Date.now()) { + // it is time to remove the package: + const fullUrl: string = [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + entry.filePath, + ].join('/') + + await this.deletePackageIfExists(this.getMetadataPath(fullUrl)) + await this.deletePackageIfExists(fullUrl) + removedFilePaths.push(entry.filePath) + } + } + + // Fetch again, to decrease the risk of race-conditions: + packagesToRemove = await this.getPackagesToRemove() + let changed = false + // Remove paths from array: + for (let i = 0; i < packagesToRemove.length; i++) { + const entry = packagesToRemove[i] + if (removedFilePaths.includes(entry.filePath)) { + packagesToRemove.splice(i, 1) + changed = true + break + } + } + if (changed) { + await this.storePackagesToRemove(packagesToRemove) + } + } + private async deletePackageIfExists(url: string): Promise { + const result = await fetch(url, { + method: 'DELETE', + }) + if (result.status === 404) return undefined // that's ok + if (result.status >= 400) { + const text = await result.text() + throw new Error( + `deletePackageIfExists: Bad response: [${result.status}]: ${result.statusText}, DELETE ${this.fullUrl}, ${text}` + ) + } + } + /** Full path to the file containing deferred removals */ + private get deferRemovePackagesPath(): string { + return [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + '__removePackages.json', + ].join('/') + } + /** */ + private async getPackagesToRemove(): Promise { + return (await this.fetchJSON(this.deferRemovePackagesPath)) ?? [] + } + private async storePackagesToRemove(packagesToRemove: DelayPackageRemovalEntry[]): Promise { + await this.storeJSON(this.deferRemovePackagesPath, packagesToRemove) + } + private async fetchJSON(url: string): Promise { + const result = await fetch(url) + if (result.status === 404) return undefined + if (result.status >= 400) { + const text = await result.text() + throw new Error( + `getPackagesToRemove: Bad response: [${result.status}]: ${result.statusText}, GET ${url}, ${text}` + ) + } + return result.json() + } + private async storeJSON(url: string, data: any): Promise { + const formData = new FormData() + formData.append('text', JSON.stringify(data)) + const result = await fetch(url, { + method: 'POST', + body: formData, + }) + if (result.status >= 400) { + const text = await result.text() + throw new Error(`storeJSON: Bad response: [${result.status}]: ${result.statusText}, POST ${url}, ${text}`) + } + } + /** Full path to the metadata file */ + private getMetadataPath(fullUrl: string) { + return fullUrl + '_metadata.json' + } +} +interface HTTPHeaders { + contentType: string | null + contentLength: string | null + lastModified: string | null + etags: string | null +} + +interface DelayPackageRemovalEntry { + /** Local file path */ + filePath: string + /** Unix timestamp for when it's clear to remove the file */ + removeTime: number +} diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts new file mode 100644 index 00000000..650e1494 --- /dev/null +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts @@ -0,0 +1,282 @@ +import { Accessor } from '@sofie-automation/blueprints-integration' +import { GenericWorker } from '../../../worker' +import { roboCopyFile } from '../lib/robocopy' +// import { diff } from 'deep-diff' +import { UniversalVersion, compareUniversalVersions, makeUniversalVersion, getStandardCost } from '../lib/lib' +import { ExpectationWindowsHandler } from './expectationWindowsHandler' +import { + hashObj, + Expectation, + ReturnTypeDoYouSupportExpectation, + ReturnTypeGetCostFortExpectation, + ReturnTypeIsExpectationFullfilled, + ReturnTypeIsExpectationReadyToStartWorkingOn, + ReturnTypeRemoveExpectation, +} from '@shared/api' +import { + isCorePackageInfoAccessorHandle, + isFileShareAccessorHandle, + isHTTPProxyAccessorHandle, + isLocalFolderAccessorHandle, +} from '../../../accessorHandlers/accessor' +import { ByteCounter } from '../../../lib/streamByteCounter' +import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' +import { + checkWorkerHasAccessToPackageContainersOnPackage, + lookupAccessorHandles, + LookupPackageContainer, + userReadableDiff, + waitTime, +} from './lib' +import { CancelablePromise } from '../../../lib/cancelablePromise' +import { PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' +import { diff } from 'deep-diff' + +/** + * Copies a file from one of the sources and into the target PackageContainer + */ +export const JsonDataCopy: ExpectationWindowsHandler = { + doYouSupportExpectation(exp: Expectation.Any, genericWorker: GenericWorker): ReturnTypeDoYouSupportExpectation { + return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { + sources: exp.startRequirement.sources, + targets: exp.endRequirement.targets, + }) + }, + getCostForExpectation: async ( + exp: Expectation.Any, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + return getStandardCost(exp, worker) + }, + isExpectationReadyToStartWorkingOn: async ( + exp: Expectation.Any, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) return { ready: lookupSource.ready, sourceExists: false, reason: lookupSource.reason } + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } + + // Also check if we actually can read from the package, + // this might help in some cases if the file is currently transferring + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } + + return { + ready: true, + sourceExists: true, + } + }, + isExpectationFullfilled: async ( + exp: Expectation.Any, + _wasFullfilled: boolean, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } + + const issuePackage = await lookupTarget.handle.checkPackageReadAccess() + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } + } + + // check that the file is of the right version: + const actualTargetVersion = await lookupTarget.handle.fetchMetadata() + if (!actualTargetVersion) + return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + + const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() + + const issueVersions = compareUniversalVersions(makeUniversalVersion(actualSourceVersion), actualTargetVersion) + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } + } + + return { + fulfilled: true, + } + }, + workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + // Copies the file from Source to Target + + const startTime = Date.now() + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + + const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() + const actualSourceVersionHash = hashObj(actualSourceVersion) + const actualSourceUVersion = makeUniversalVersion(actualSourceVersion) + + const sourceHandle = lookupSource.handle + const targetHandle = lookupTarget.handle + if ( + (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || + lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || + lookupSource.accessor.type === Accessor.AccessType.HTTP) && + lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO + ) { + // We can copy by using streams: + if ( + !isLocalFolderAccessorHandle(lookupSource.handle) && + !isFileShareAccessorHandle(lookupSource.handle) && + !isHTTPProxyAccessorHandle(lookupSource.handle) + ) + throw new Error(`Source AccessHandler type is wrong`) + if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Source AccessHandler type is wrong`) + + let wasCancelled = false + let sourceStream: PackageReadStream | undefined = undefined + let writeStream: PutPackageHandler | undefined = undefined + const workInProgress = new WorkInProgress({ workLabel: 'Copying, using streams' }, async () => { + // on cancel work + wasCancelled = true + await new Promise((resolve, reject) => { + writeStream?.once('close', () => { + targetHandle + .removePackage() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + sourceStream?.cancel() + writeStream?.abort() + }) + }).do(async () => { + workInProgress._reportProgress(actualSourceVersionHash, 0.1) + + if (wasCancelled) return + sourceStream = await lookupSource.handle.getPackageReadStream() + writeStream = await targetHandle.putPackageStream(sourceStream.readStream) + + workInProgress._reportProgress(actualSourceVersionHash, 0.5) + + sourceStream.readStream.on('error', (err) => { + workInProgress._reportError(err) + }) + writeStream.on('error', (err) => { + workInProgress._reportError(err) + }) + writeStream.once('close', () => { + if (wasCancelled) return // ignore + setImmediate(() => { + // Copying is done + ;(async () => { + await targetHandle.finalizePackage() + // await targetHandle.updateMetadata() + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + { + user: `Completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) + }) + }) + }) + + return workInProgress + } else { + throw new Error( + `JsonDataCopy.workOnExpectation: Unsupported accessor source-target pair "${lookupSource.accessor.type}"-"${lookupTarget.accessor.type}"` + ) + } + }, + removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + // Remove the file on the location + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } + + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove json-data due to an internal error`, + tech: `Cannot remove json-data: ${err.toString()}`, + }, + } + } + + return { + removed: true, + // reason: `Removed file "${exp.endRequirement.content.filePath}" from target` + } + }, +} +function isJsonDataCopy(exp: Expectation.Any): exp is Expectation.JsonDataCopy { + return exp.type === Expectation.Type.JSON_DATA_COPY +} + +function lookupCopySources( + worker: GenericWorker, + exp: Expectation.JsonDataCopy +): Promise> { + return lookupAccessorHandles( + worker, + exp.startRequirement.sources, + exp.endRequirement.content, + exp.workOptions, + { + read: true, + readPackage: true, + packageVersion: exp.endRequirement.version, + } + ) +} +function lookupCopyTargets( + worker: GenericWorker, + exp: Expectation.JsonDataCopy +): Promise> { + return lookupAccessorHandles( + worker, + exp.endRequirement.targets, + exp.endRequirement.content, + exp.workOptions, + { + write: true, + writePackageContainer: true, + } + ) +} From 3fb26030f3a2cc0d1be47a74d828804a392d02e5 Mon Sep 17 00:00:00 2001 From: Ola Christian Gundelsby Date: Wed, 16 Jun 2021 12:25:53 +0200 Subject: [PATCH 25/67] chore: cherry-pick commit: 17049c7 "fix: some errors, clarifying comments and todos added after session with Johan" --- .../worker/src/worker/accessorHandlers/fileShare.ts | 1 + shared/packages/worker/src/worker/accessorHandlers/http.ts | 6 +----- .../worker/src/worker/accessorHandlers/httpProxy.ts | 1 + .../worker/src/worker/accessorHandlers/localFolder.ts | 1 + .../packages/worker/src/worker/accessorHandlers/quantel.ts | 1 + 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 1170b135..8d5d7083 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -30,6 +30,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } = {} private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean filePath?: string } diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index cd64fa13..bdeb496b 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -17,6 +17,7 @@ import { assertNever } from '../lib/lib' export class HTTPAccessorHandle extends GenericAccessorHandle { static readonly type = 'http' private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean path?: string } @@ -83,12 +84,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { - - // TODO: Do a OPTIONS request? - fetch(the url, { - method: 'options' - }) // 204 or 404 is "not found" // Access-Control-Allow-Methods should contain GET return { success: true } diff --git a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts index 258382d3..14ff00df 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts @@ -17,6 +17,7 @@ import { assertNever } from '../lib/lib' export class HTTPProxyAccessorHandle extends GenericAccessorHandle { static readonly type = 'http-proxy' private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean filePath?: string } diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index 022099ac..35371374 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -22,6 +22,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand static readonly type = LocalFolderAccessorHandleType private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean filePath?: string } diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index 2ac82397..99f16a77 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -19,6 +19,7 @@ const MINIMUM_FRAMES = 10 export class QuantelAccessorHandle extends GenericAccessorHandle { static readonly type = 'quantel' private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean guid?: string title?: string From ffd041399d37fbb67ad6d018e1fb3b9a2ce821b4 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 16 Jun 2021 16:14:03 +0200 Subject: [PATCH 26/67] chore: FOR_DEVELOPERS.md --- FOR_DEVELOPERS.md | 79 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/FOR_DEVELOPERS.md b/FOR_DEVELOPERS.md index 670a9a75..85fc9154 100644 --- a/FOR_DEVELOPERS.md +++ b/FOR_DEVELOPERS.md @@ -1,23 +1,20 @@ - This document contains documentation intended for developers of this repo. - # Key concepts -![System overview](./images/System-overview.png "System overview") +![System overview](./images/System-overview.png 'System overview') ## Workforce -*Note: There can be only one (1) Workforce in a setup.* +_Note: There can be only one (1) Workforce in a setup._ The Workforce keeps track of which `ExpectationManagers` and `Workers` are online, and mediates the contact between the two. _Future functionality: The Workforce is responsible for tracking the total workload and spin up/down workers accordingly._ - ## Package Manager -*Note: There can be multiple Package Managers in a setup* +_Note: There can be multiple Package Managers in a setup_ The Package Manager receives [Packages](#packages) from [Sofie Core](https://github.com/nrkno/tv-automation-server-core) and generates [Expectations](#expectations) from them. @@ -35,22 +32,20 @@ A typical lifetime of an Expectation is: 4. `WORKING`: Intermediary state while the Worker is working. 5. `FULFILLED`: From time-to-time, the Expectation is re-checked if it still is `FULFILLED` - ## Worker -*Note: There can be multiple Workers in a setup* +_Note: There can be multiple Workers in a setup_ The Worker is the process which actually does the work. It exposes an API with methods for the `ExpectationManager` to call in order to check status of Packages and perform the work. The Worker is (almost completely) **stateless**, which allows it to expose a lambda-like API. This allows for there being a pool of Workers where the workload can be easilly shared between the Workers. - _Future functionality: There are multiple different types of Workers. Some are running on a Windows-machine with direct access to that maching. Some are running in Linux/Docker containers, or even off-premise._ ### ExpectationHandlers & AccessorHandlers -![Expectation and Accessor handlers](./images/handlers.png "Expectation and Accessor handlers") +![Expectation and Accessor handlers](./images/handlers.png 'Expectation and Accessor handlers') Internally, the Worker is structured by separating the `ExpectationHandlers` and the `AccessorHandlers`. @@ -59,7 +54,6 @@ The `ExpectationHandlers` handles the high-level functionality required for a ce For example, when [copying a file](shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts) from one folder to another, the `ExpectationHandler` will handle things like "check if the source package exists", "check if the target package already exists" but it's never touching the files directly, only talking to the the `AccessorHandler`. The `AccessorHandler` [exposes a few generic methods](./shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts), like "check if we can read from a Package" (ie does a file exist), etc. - ## HTTP-server The HTTP-server application is a simple HTTP-server which allows for uploading and downloading of Packages with a RESTful API. @@ -94,3 +88,66 @@ A PackageContainer is separated from an **Accessor**, which is the "way to acces _See [expectationApi.ts](shared/packages/api/src/expectationApi.ts)._ An Expectation is what the PackageManager uses to figure out what should be done and how to do it. One example is "Copy a file from a source into a target". + +# Examples: + +Below are some examples of the data: + +## Example A, a file is to be copied + +```javascript +// The input to Package Manager (from Sofie-Core): + +const packageContainers = { + source0: { + label: 'Source 0', + accessors: { + local: { + type: 'local_folder', + folderPath: 'D:\\media\\source0', + allowRead: true, + }, + }, + }, + target0: { + label: 'Target 0', + accessors: { + local: { + type: 'local_folder', + folderPath: 'D:\\media\\target0', + allowRead: true, + allowWrite: true, + }, + }, + }, +} +const expectedPackages = [ + { + type: 'media_file', + _id: 'test', + contentVersionHash: 'abc1234', + content: { + filePath: 'myFocalFolder\\amb.mp4', + }, + version: {}, + sources: [ + { + containerId: 'source0', + accessors: { + local: { + type: 'local_folder', + filePath: 'amb.mp4', + }, + }, + }, + ], + layers: ['target0'], + sideEffect: { + previewContainerId: null, + previewPackageSettings: null, + thumbnailContainerId: null, + thumbnailPackageSettings: null, + }, + }, +] +``` From aaa41746d9b8bc1a6c6a62a90886ccac2717253a Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 12:30:45 +0200 Subject: [PATCH 27/67] chore: handle optional accessors --- .../packages/generic/src/expectationGenerator.ts | 6 +++--- apps/package-manager/packages/generic/src/packageManager.ts | 2 +- .../worker/src/worker/accessorHandlers/lib/FileHandler.ts | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 8742d782..375d69c5 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -13,7 +13,7 @@ export interface ExpectedPackageWrapMediaFile extends ExpectedPackageWrap { sources: { containerId: string label: string - accessors: ExpectedPackage.ExpectedPackageMediaFile['sources'][0]['accessors'] + accessors: NonNullable }[] } export interface ExpectedPackageWrapQuantel extends ExpectedPackageWrap { @@ -21,7 +21,7 @@ export interface ExpectedPackageWrapQuantel extends ExpectedPackageWrap { sources: { containerId: string label: string - accessors: ExpectedPackage.ExpectedPackageQuantelClip['sources'][0]['accessors'] + accessors: NonNullable }[] } export interface ExpectedPackageWrapJSONData extends ExpectedPackageWrap { @@ -29,7 +29,7 @@ export interface ExpectedPackageWrapJSONData extends ExpectedPackageWrap { sources: { containerId: string label: string - accessors: ExpectedPackage.ExpectedPackageJSONData['sources'][0]['accessors'] + accessors: NonNullable }[] } diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 3465c6cb..d31b4558 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -628,7 +628,7 @@ function wrapExpectedPackage( } const accessorIds = _.uniq( - Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors)) + Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors || {})) ) for (const accessorId of accessorIds) { diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index bd050210..94b7c125 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -207,6 +207,9 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso ], sideEffect: options.sideEffect, } + if (!expPackage.sources[0].accessors) { + expPackage.sources[0].accessors = {} + } if (this._type === LocalFolderAccessorHandleType) { expPackage.sources[0].accessors[ this.accessorId From d5c13991ecf40bfd33f1c3b0545357ed67de1d64 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 12:33:32 +0200 Subject: [PATCH 28/67] fix: various fixes for jsonDataCopy & HTTP accessor --- .../generic/src/expectationGenerator.ts | 8 ++++---- .../src/worker/accessorHandlers/accessor.ts | 4 ++++ .../src/worker/accessorHandlers/httpProxy.ts | 2 +- .../expectationHandlers/jsonDataCopy.ts | 17 +++-------------- .../workers/windowsWorker/windowsWorker.ts | 3 +++ 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 375d69c5..7fa289a7 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -143,7 +143,7 @@ export function generateExpectations( } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { exp = generateQuantelCopy(managerId, packageWrap) } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.JSON_DATA) { - exp = generateJsonDataCopy(managerId, packageWrap) + exp = generateJsonDataCopy(managerId, packageWrap, settings) } if (exp) { prioritizeExpectation(packageWrap, exp) @@ -677,7 +677,7 @@ function generateJsonDataCopy( managerId: string, expWrap: ExpectedPackageWrap, settings: PackageManagerSettings -): Expectation.JSONDataCopy { +): Expectation.JsonDataCopy { const expWrapMediaFile = expWrap as ExpectedPackageWrapJSONData const exp: Expectation.JsonDataCopy = { @@ -706,10 +706,10 @@ function generateJsonDataCopy( }, endRequirement: { - targets: expWrapMediaFile.targets as [Expectation.SpecificPackageContainerOnPackage.CorePackage], + targets: expWrapMediaFile.targets as [Expectation.SpecificPackageContainerOnPackage.File], content: expWrapMediaFile.expectedPackage.content, version: { - type: Expectation.Version.Type.CORE_PACKAGE_INFO, + type: Expectation.Version.Type.FILE_ON_DISK, ...expWrapMediaFile.expectedPackage.version, }, }, diff --git a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts index f18c6864..4af2b4b1 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts @@ -3,6 +3,7 @@ import { GenericWorker } from '../worker' import { CorePackageInfoAccessorHandle } from './corePackageInfo' import { FileShareAccessorHandle } from './fileShare' import { GenericAccessorHandle } from './genericHandle' +import { HTTPAccessorHandle } from './http' import { HTTPProxyAccessorHandle } from './httpProxy' import { LocalFolderAccessorHandle } from './localFolder' import { QuantelAccessorHandle } from './quantel' @@ -28,6 +29,8 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { return LocalFolderAccessorHandle } else if (accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO) { return CorePackageInfoAccessorHandle + } else if (accessor.type === Accessor.AccessType.HTTP) { + return HTTPAccessorHandle } else if (accessor.type === Accessor.AccessType.HTTP_PROXY) { return HTTPProxyAccessorHandle } else if (accessor.type === Accessor.AccessType.FILE_SHARE) { @@ -83,6 +86,7 @@ export function getAccessorCost(accessorType: Accessor.AccessType | undefined): case Accessor.AccessType.FILE_SHARE: return 2 case Accessor.AccessType.HTTP_PROXY: + case Accessor.AccessType.HTTP: return 3 case undefined: diff --git a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts index 14ff00df..540bfb05 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts @@ -25,7 +25,7 @@ export class HTTPProxyAccessorHandle extends GenericAccessorHandle Date: Fri, 18 Jun 2021 12:34:56 +0200 Subject: [PATCH 29/67] fix: improve roboustness and doc --- .../generic/src/expectationGenerator.ts | 100 ++++++++++-------- .../packages/generic/src/packageManager.ts | 1 - 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 7fa289a7..e84af27e 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -207,28 +207,35 @@ export function generateExpectations( } if (expectation0.sideEffect?.thumbnailContainerId && expectation0.sideEffect?.thumbnailPackageSettings) { - const packageContainer: PackageContainer = - packageContainers[expectation0.sideEffect.thumbnailContainerId] - - const thumbnail = generateMediaFileThumbnail( - expectation, - expectation0.sideEffect.thumbnailContainerId, - expectation0.sideEffect.thumbnailPackageSettings, - packageContainer - ) - expectations[thumbnail.id] = thumbnail + const packageContainer = packageContainers[expectation0.sideEffect.thumbnailContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const thumbnail = generateMediaFileThumbnail( + expectation, + expectation0.sideEffect.thumbnailContainerId, + expectation0.sideEffect.thumbnailPackageSettings, + packageContainer + ) + expectations[thumbnail.id] = thumbnail + } } if (expectation0.sideEffect?.previewContainerId && expectation0.sideEffect?.previewPackageSettings) { - const packageContainer: PackageContainer = packageContainers[expectation0.sideEffect.previewContainerId] - - const preview = generateMediaFilePreview( - expectation, - expectation0.sideEffect.previewContainerId, - expectation0.sideEffect.previewPackageSettings, - packageContainer - ) - expectations[preview.id] = preview + const packageContainer = packageContainers[expectation0.sideEffect.previewContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const preview = generateMediaFilePreview( + expectation, + expectation0.sideEffect.previewContainerId, + expectation0.sideEffect.previewPackageSettings, + packageContainer + ) + expectations[preview.id] = preview + } } } else if (expectation0.type === Expectation.Type.QUANTEL_CLIP_COPY) { const expectation = expectation0 as Expectation.QuantelClipCopy @@ -244,28 +251,35 @@ export function generateExpectations( } if (expectation0.sideEffect?.thumbnailContainerId && expectation0.sideEffect?.thumbnailPackageSettings) { - const packageContainer: PackageContainer = - packageContainers[expectation0.sideEffect.thumbnailContainerId] - - const thumbnail = generateQuantelClipThumbnail( - expectation, - expectation0.sideEffect.thumbnailContainerId, - expectation0.sideEffect.thumbnailPackageSettings, - packageContainer - ) - expectations[thumbnail.id] = thumbnail + const packageContainer = packageContainers[expectation0.sideEffect.thumbnailContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const thumbnail = generateQuantelClipThumbnail( + expectation, + expectation0.sideEffect.thumbnailContainerId, + expectation0.sideEffect.thumbnailPackageSettings, + packageContainer + ) + expectations[thumbnail.id] = thumbnail + } } if (expectation0.sideEffect?.previewContainerId && expectation0.sideEffect?.previewPackageSettings) { - const packageContainer: PackageContainer = packageContainers[expectation0.sideEffect.previewContainerId] - - const preview = generateQuantelClipPreview( - expectation, - expectation0.sideEffect.previewContainerId, - expectation0.sideEffect.previewPackageSettings, - packageContainer - ) - expectations[preview.id] = preview + const packageContainer = packageContainers[expectation0.sideEffect.previewContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const preview = generateQuantelClipPreview( + expectation, + expectation0.sideEffect.previewContainerId, + expectation0.sideEffect.previewPackageSettings, + packageContainer + ) + expectations[preview.id] = preview + } } } } @@ -298,9 +312,9 @@ function generateMediaFileCopy( type: Expectation.Type.FILE_COPY, statusReport: { label: `Copy media "${expWrapMediaFile.expectedPackage.content.filePath}"`, - description: `Copy media "${expWrapMediaFile.expectedPackage.content.filePath}" to the playout-device "${ + description: `Copy media file "${expWrapMediaFile.expectedPackage.content.filePath}" to the device "${ expWrapMediaFile.playoutDeviceId - }", from "${JSON.stringify(expWrapMediaFile.sources)}"`, + }", from ${expWrapMediaFile.sources.map((source) => `"${source.label}"`).join(', ')}`, requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, @@ -346,7 +360,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E label: `Copy Quantel clip ${content.title || content.guid}`, description: `Copy Quantel clip ${content.title || content.guid} to server for "${ expWrapQuantelClip.playoutDeviceId - }", from ${expWrapQuantelClip.sources}`, + }", from ${expWrapQuantelClip.sources.map((source) => `"${source.label}"`).join(', ')}`, requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, @@ -382,7 +396,7 @@ function generatePackageScan(expectation: Expectation.FileCopy | Expectation.Qua statusReport: { label: `Scan ${expectation.statusReport.label}`, - description: `Scanning is used to provide Sofie GUI with status about the media`, + description: `Scanning the media, to provide data to the Sofie GUI (like for zebra-stripes, etc).`, requiredForPlayout: false, displayRank: 10, sendReport: expectation.statusReport.sendReport, @@ -428,7 +442,7 @@ function generatePackageDeepScan( statusReport: { label: `Deep Scan ${expectation.statusReport.label}`, - description: `Deep scanning includes scene-detection, black/freeze frames etc.`, + description: `Deep scanning media file, in order to detect scenes, black/freeze frames etc.`, requiredForPlayout: false, displayRank: 10, sendReport: expectation.statusReport.sendReport, diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index d31b4558..7a652d25 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -657,7 +657,6 @@ function wrapExpectedPackage( if (packageContainerId) { const lookedUpTarget = packageContainers[packageContainerId] if (lookedUpTarget) { - // Todo: should the be any combination of properties here? combinedTargets.push({ ...omit(clone(lookedUpTarget), 'accessors'), accessors: lookedUpTarget.accessors as { From ac616a91701eca37c0770fec707b70f4fa4a9ce8 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 12:35:21 +0200 Subject: [PATCH 30/67] fix: handle internal error better --- .../expectationManager/src/expectationManager.ts | 5 ++++- .../src/worker/workers/windowsWorker/windowsWorker.ts | 10 +--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index bf124aaf..c9ea7ae9 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -854,7 +854,10 @@ export class ExpectationManager { } catch (err) { this.logger.error('Error thrown in evaluateExpectationState') this.logger.error(err) - this.updateTrackedExpStatus(trackedExp, undefined, err.toString()) + this.updateTrackedExpStatus(trackedExp, undefined, { + user: 'Internal error in Package Manager', + tech: err.toString(), + }) } } /** Returns the appropriate time to wait before checking a fulfilled expectation again */ diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts index 16fe677a..9f47918a 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts @@ -46,15 +46,7 @@ export class WindowsWorker extends GenericWorker { super(logger, config, location, sendMessageToManager, WindowsWorker.type) } async doYouSupportExpectation(exp: Expectation.Any): Promise { - try { - return this.getExpectationHandler(exp).doYouSupportExpectation(exp, this, this) - } catch (err) { - // Does not support the type - return { - support: false, - reason: err.toString(), - } - } + return this.getExpectationHandler(exp).doYouSupportExpectation(exp, this, this) } async init(): Promise { this.hasFFMpeg = !!(await hasFFMpeg()) From 80b2aaf2b34afae3e65814664439da075a8ae3ba Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 12:36:59 +0200 Subject: [PATCH 31/67] feat: add Core-callable method troubleshoot() to use for sending a data-dump to a Core user, for troubleshooting. --- .../packages/generic/src/coreHandler.ts | 3 ++ .../packages/generic/src/packageManager.ts | 29 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/package-manager/packages/generic/src/coreHandler.ts b/apps/package-manager/packages/generic/src/coreHandler.ts index ae6d50ad..99ee61da 100644 --- a/apps/package-manager/packages/generic/src/coreHandler.ts +++ b/apps/package-manager/packages/generic/src/coreHandler.ts @@ -411,4 +411,7 @@ export class CoreHandler { abortExpectation(workId: string): void { return this._packageManagerHandler?.abortExpectation(workId) } + troubleshoot(): any { + return this._packageManagerHandler?.getDataSnapshot() + } } diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 7a652d25..028af9d7 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -51,6 +51,22 @@ export class PackageManagerHandler { } callbacksHandler: ExpectationManagerCallbacksHandler + private dataSnapshot: { + updated: number + expectedPackages: ResultingExpectedPackage[] + packageContainers: PackageContainers + expectations: { + [id: string]: Expectation.Any + } + packageContainerExpectations: { [id: string]: PackageContainerExpectation } + } = { + updated: 0, + expectedPackages: [], + packageContainers: {}, + expectations: {}, + packageContainerExpectations: {}, + } + constructor( public logger: LoggerInstance, private managerId: string, @@ -239,6 +255,9 @@ export class PackageManagerHandler { this.logger.info(`Has ${expectedPackages.length} expectedPackages`) // this.logger.info(JSON.stringify(expectedPackages, null, 2)) + this.dataSnapshot.expectedPackages = expectedPackages + this.dataSnapshot.packageContainers = this.packageContainersCache + // Step 1: Generate expectations: const expectations = generateExpectations( this.logger, @@ -250,14 +269,19 @@ export class PackageManagerHandler { this.settings ) this.logger.info(`Has ${Object.keys(expectations).length} expectations`) + // this.logger.info(JSON.stringify(expectations, null, 2)) + this.dataSnapshot.expectations = expectations + const packageContainerExpectations = generatePackageContainerExpectations( this.expectationManager.managerId, this.packageContainersCache, activePlaylist ) this.logger.info(`Has ${Object.keys(packageContainerExpectations).length} packageContainerExpectations`) + ;(this.dataSnapshot.packageContainerExpectations = packageContainerExpectations), + (this.dataSnapshot.updated = Date.now()) + this.ensureMandatoryPackageContainerExpectations(packageContainerExpectations) - // this.logger.info(JSON.stringify(expectations, null, 2)) // Step 2: Track and handle new expectations: this.expectationManager.updatePackageContainerExpectations(packageContainerExpectations) @@ -276,6 +300,9 @@ export class PackageManagerHandler { // This method can be called from core this.expectationManager.abortExpectation(workId) } + public getDataSnapshot() { + return this.dataSnapshot + } /** Ensures that the packageContainerExpectations containes the mandatory expectations */ private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { From f8ddcdeedf5d113176a4d788390c8eac8c37b43e Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 15:02:56 +0200 Subject: [PATCH 32/67] chore: improve some wordings --- .../generic/src/expectationGenerator.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index e84af27e..953e7f46 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -311,7 +311,7 @@ function generateMediaFileCopy( ], type: Expectation.Type.FILE_COPY, statusReport: { - label: `Copy media "${expWrapMediaFile.expectedPackage.content.filePath}"`, + label: `Copying media "${expWrapMediaFile.expectedPackage.content.filePath}"`, description: `Copy media file "${expWrapMediaFile.expectedPackage.content.filePath}" to the device "${ expWrapMediaFile.playoutDeviceId }", from ${expWrapMediaFile.sources.map((source) => `"${source.label}"`).join(', ')}`, @@ -395,8 +395,8 @@ function generatePackageScan(expectation: Expectation.FileCopy | Expectation.Qua fromPackages: expectation.fromPackages, statusReport: { - label: `Scan ${expectation.statusReport.label}`, - description: `Scanning the media, to provide data to the Sofie GUI (like for zebra-stripes, etc).`, + label: `Scanning`, + description: `Scanning the media, to provide data to the Sofie GUI`, requiredForPlayout: false, displayRank: 10, sendReport: expectation.statusReport.sendReport, @@ -441,10 +441,10 @@ function generatePackageDeepScan( fromPackages: expectation.fromPackages, statusReport: { - label: `Deep Scan ${expectation.statusReport.label}`, - description: `Deep scanning media file, in order to detect scenes, black/freeze frames etc.`, + label: `Deep Scanning`, + description: `Detecting scenes, black frames, freeze frames etc.`, requiredForPlayout: false, - displayRank: 10, + displayRank: 11, sendReport: expectation.statusReport.sendReport, }, @@ -496,7 +496,7 @@ function generateMediaFileThumbnail( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate thumbnail for ${expectation.statusReport.label}`, + label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, requiredForPlayout: false, displayRank: 11, @@ -547,7 +547,7 @@ function generateMediaFilePreview( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate preview for ${expectation.statusReport.label}`, + label: `Generating preview`, description: `Preview is used in Sofie GUI`, requiredForPlayout: false, displayRank: 12, @@ -598,7 +598,7 @@ function generateQuantelClipThumbnail( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate thumbnail for ${expectation.statusReport.label}`, + label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, requiredForPlayout: false, displayRank: 11, @@ -648,7 +648,7 @@ function generateQuantelClipPreview( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate preview for ${expectation.statusReport.label}`, + label: `Generating preview`, description: `Preview is used in Sofie GUI`, requiredForPlayout: false, displayRank: 12, @@ -706,7 +706,7 @@ function generateJsonDataCopy( ], type: Expectation.Type.JSON_DATA_COPY, statusReport: { - label: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}"`, + label: `Copying JSON data`, description: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}" from "${JSON.stringify( expWrapMediaFile.sources )}"`, From 8f559d5301dffe3700944520275de80ff4dedf76 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 18 Jun 2021 15:05:23 +0200 Subject: [PATCH 33/67] fix: bug in handling of Reason-strings --- .../packages/expectationManager/src/expectationManager.ts | 6 +++--- .../workers/windowsWorker/expectationHandlers/fileCopy.ts | 8 ++++---- .../windowsWorker/expectationHandlers/jsonDataCopy.ts | 6 +++--- .../windowsWorker/expectationHandlers/mediaFilePreview.ts | 6 +++--- .../expectationHandlers/mediaFileThumbnail.ts | 6 +++--- .../windowsWorker/expectationHandlers/packageDeepScan.ts | 6 +++--- .../windowsWorker/expectationHandlers/packageScan.ts | 4 ++-- .../windowsWorker/expectationHandlers/quantelClipCopy.ts | 8 ++++---- .../expectationHandlers/quantelClipPreview.ts | 4 ++-- .../expectationHandlers/quantelClipThumbnail.ts | 6 +++--- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index c9ea7ae9..dd11ff56 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -1129,8 +1129,8 @@ export class ExpectationManager { ) if (!disposeMonitorResult.success) { this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: `Unable to remove monitor, due to ${disposeMonitorResult.reason}`, - tech: `Unable to dispose monitor: ${disposeMonitorResult.reason}`, + user: `Unable to remove monitor, due to ${disposeMonitorResult.reason.user}`, + tech: `Unable to dispose monitor: ${disposeMonitorResult.reason.tech}`, }) continue // Break further execution for this PackageContainer } @@ -1233,7 +1233,7 @@ export class ExpectationManager { if (updatedReason) { this.logger.info( - `PackageContainerStatus "${trackedPackageContainer.packageContainer.label}": Reason: "${trackedPackageContainer.status.reason}"` + `PackageContainerStatus "${trackedPackageContainer.packageContainer.label}": Reason: "${trackedPackageContainer.status.reason.tech}"` ) } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index e30a5099..b04af1eb 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -111,7 +111,7 @@ export const FileCopy: ExpectationWindowsHandler = { sourceExists: true, // reason: { // user: 'Ready to start copying', - // tech: `${lookupSource.reason}, ${lookupTarget.reason}`, + // tech: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, // }, } }, @@ -149,7 +149,7 @@ export const FileCopy: ExpectationWindowsHandler = { return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -170,10 +170,10 @@ export const FileCopy: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupCopyTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts index 6a8ba029..5f0c2a42 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts @@ -93,7 +93,7 @@ export const JsonDataCopy: ExpectationWindowsHandler = { return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -113,10 +113,10 @@ export const JsonDataCopy: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupCopyTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index d2642771..dcbc4270 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -67,7 +67,7 @@ export const MediaFilePreview: ExpectationWindowsHandler = { return { ready: true, sourceExists: true, - // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -138,10 +138,10 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupPreviewSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupPreviewTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index b44a0c2b..5767be99 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -72,7 +72,7 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { return { ready: true, sourceExists: true, - // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -140,10 +140,10 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupThumbnailSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let ffMpegProcess: FFMpegProcess | undefined const workInProgress = new WorkInProgress({ workLabel: 'Generating thumbnail' }, async () => { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts index 51a6ead8..6211e6bc 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts @@ -72,7 +72,7 @@ export const PackageDeepScan: ExpectationWindowsHandler = { return { ready: true, sourceExists: true, - // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -128,10 +128,10 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupDeepScanSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupDeepScanTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let currentProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Scanning file' }, async () => { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts index bcebf546..0e9e264c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts @@ -125,10 +125,10 @@ export const PackageScan: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupScanSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupScanTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let currentProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Scanning file' }, async () => { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts index e66297a4..0666711c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts @@ -64,7 +64,7 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { return { ready: true, sourceExists: true, - // reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -104,7 +104,7 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { } const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } // Check that the target clip is of the right version: @@ -129,10 +129,10 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupCopyTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index 5ffd612a..c3404e9e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -147,10 +147,10 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupPreviewSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupPreviewTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index 982d4ec9..260fef2d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -150,10 +150,10 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupThumbnailSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle @@ -175,7 +175,7 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail: const thumbnailURL = await getThumbnailURL(exp, lookupSource) - if (!thumbnailURL.success) throw new Error(`Can't start working due to source: ${thumbnailURL.reason}`) + if (!thumbnailURL.success) throw new Error(`Can't start working due to source: ${thumbnailURL.reason.tech}`) const sourceHTTPHandle = getSourceHTTPHandle(worker, sourceHandle, thumbnailURL) let wasCancelled = false From 73101da515c76c7cc86969b08608dec6da945ead Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 23 Jun 2021 08:44:02 +0200 Subject: [PATCH 34/67] fix: replace TrackedExpectationState enum with identical enum from core-integration --- .../packages/generic/src/packageManager.ts | 8 +- package.json | 4 +- .../src/expectationManager.ts | 134 ++++++++++-------- yarn.lock | 72 ++++++---- 4 files changed, 122 insertions(+), 96 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 028af9d7..0fc03d86 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -377,7 +377,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks expectaction: Expectation.Any | null, actualVersionHash: string | null, statusInfo: { - status?: string + status?: ExpectedPackageStatusAPI.WorkStatusState progress?: number statusReason?: Reason } @@ -389,10 +389,10 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks } else { if (!expectaction.statusReport.sendReport) return // Don't report the status + // Default properties: const workStatus: ExpectedPackageStatusAPI.WorkStatus = { - // Default properties: ...{ - status: 'N/A', + status: ExpectedPackageStatusAPI.WorkStatusState.NEW, progress: 0, statusReason: { user: '', tech: '' }, }, @@ -400,7 +400,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks ...((this.toReportExpectationStatus[expectationId]?.workStatus || {}) as Partial), // Intentionally cast to Partial<>, to make typings in const workStatus more strict - // Updated porperties: + // Updated properties: ...expectaction.statusReport, ...statusInfo, diff --git a/package.json b/package.json index fb85873c..b21d60b6 100644 --- a/package.json +++ b/package.json @@ -56,8 +56,8 @@ "node": ">=12.3.0" }, "dependencies": { - "@sofie-automation/blueprints-integration": "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0", - "@sofie-automation/server-core-integration": "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" + "@sofie-automation/blueprints-integration": "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0", + "@sofie-automation/server-core-integration": "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "husky": { diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index dd11ff56..4e22a388 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -260,7 +260,7 @@ export class ExpectationManager { ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash wip.trackedExp.status.workProgress = progress @@ -291,9 +291,13 @@ export class ExpectationManager { ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash - this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.FULFILLED, reason) + this.updateTrackedExpStatus( + wip.trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, + reason + ) this.callbacks.reportExpectationStatus( wip.trackedExp.id, wip.trackedExp.exp, @@ -319,9 +323,13 @@ export class ExpectationManager { wipEventError: async (wipId: number, reason: Reason): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.errorCount++ - this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, reason) + this.updateTrackedExpStatus( + wip.trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + reason + ) this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { status: wip.trackedExp.state, statusReason: wip.trackedExp.reason, @@ -378,7 +386,7 @@ export class ExpectationManager { } else if (!_.isEqual(this.trackedExpectations[id].exp, exp)) { const trackedExp = this.trackedExpectations[id] - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { this.logger.info(`Cancelling ${trackedExp.id} due to update`) await trackedExp.status.workInProgressCancel() @@ -390,7 +398,7 @@ export class ExpectationManager { const trackedExp: TrackedExpectation = { id: id, exp: exp, - state: TrackedExpectationState.NEW, + state: ExpectedPackageStatusAPI.WorkStatusState.NEW, availableWorkers: [], lastEvaluationTime: 0, errorCount: 0, @@ -426,14 +434,14 @@ export class ExpectationManager { const trackedExp = this.trackedExpectations[id] - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { this.logger.info(`Cancelling ${trackedExp.id} due to removed`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.REMOVED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.REMOVED trackedExp.lastEvaluationTime = 0 // To rerun ASAP } } @@ -449,14 +457,14 @@ export class ExpectationManager { for (const id of Object.keys(this.receivedUpdates.restartExpectations)) { const trackedExp = this.trackedExpectations[id] if (trackedExp) { - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { this.logger.info(`Cancelling ${trackedExp.id} due to restart`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.RESTARTED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.RESTARTED trackedExp.lastEvaluationTime = 0 // To rerun ASAP } } @@ -466,14 +474,14 @@ export class ExpectationManager { for (const id of Object.keys(this.receivedUpdates.abortExpectations)) { const trackedExp = this.trackedExpectations[id] if (trackedExp) { - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { this.logger.info(`Cancelling ${trackedExp.id} due to abort`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.ABORTED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.ABORTED } } this.receivedUpdates.abortExpectations = {} @@ -532,10 +540,10 @@ export class ExpectationManager { // Step 1: Evaluate the Expectations which are in the states that can be handled in parallel: for (const handleState of [ // Note: The order of these is important, as the states normally progress in this order: - TrackedExpectationState.ABORTED, - TrackedExpectationState.NEW, - TrackedExpectationState.WAITING, - TrackedExpectationState.FULFILLED, + ExpectedPackageStatusAPI.WorkStatusState.ABORTED, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, ]) { // Filter out the ones that are in the state we're about to handle: const trackedWithState = tracked.filter((trackedExp) => trackedExp.state === handleState) @@ -615,7 +623,7 @@ export class ExpectationManager { if (trackedExp.session.hadError) return // do nothing try { - if (trackedExp.state === TrackedExpectationState.NEW) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW) { // Check which workers might want to handle it: // Reset properties: @@ -638,18 +646,18 @@ export class ExpectationManager { }) ) if (trackedExp.availableWorkers.length) { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.WAITING, { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.WAITING, { user: `${trackedExp.availableWorkers.length} workers available, about to start...`, tech: `Found ${trackedExp.availableWorkers.length} workers who supports this Expectation`, }) trackedExp.session.triggerExpectationAgain = true } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { user: `Found no workers who supports this Expectation, due to: ${notSupportReason.user}`, tech: `Found no workers who supports this Expectation: "${notSupportReason.tech}"`, }) } - } else if (trackedExp.state === TrackedExpectationState.WAITING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { // Check if the expectation is ready to start: await this.assignWorkerToSession(trackedExp.session, trackedExp) @@ -662,7 +670,11 @@ export class ExpectationManager { ) if (fulfilled.fulfilled) { // The expectation is already fulfilled: - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.FULFILLED, undefined) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, + undefined + ) if (this.handleTriggerByFullfilledIds(trackedExp)) { // Something was triggered, run again ASAP: trackedExp.session.triggerOtherExpectationsAgain = true @@ -679,7 +691,7 @@ export class ExpectationManager { if (readyToStart.ready) { this.updateTrackedExpStatus( trackedExp, - TrackedExpectationState.READY, + ExpectedPackageStatusAPI.WorkStatusState.READY, { user: 'About to start working..', tech: 'About to start working..', @@ -691,7 +703,7 @@ export class ExpectationManager { // Not ready to start this.updateTrackedExpStatus( trackedExp, - TrackedExpectationState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, readyToStart.reason, newStatus ) @@ -706,7 +718,7 @@ export class ExpectationManager { this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.READY) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) { // Start working on it: await this.assignWorkerToSession(trackedExp.session, trackedExp) @@ -732,7 +744,7 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, - TrackedExpectationState.WORKING, + ExpectedPackageStatusAPI.WorkStatusState.WORKING, undefined, wipInfo.properties ) @@ -745,10 +757,10 @@ export class ExpectationManager { this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.WORKING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { // It is already working, don't do anything // TODO: work-timeout? - } else if (trackedExp.state === TrackedExpectationState.FULFILLED) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { // TODO: Some monitor that is able to invalidate if it isn't fullfilled anymore? if (timeSinceLastEvaluation > this.getFullfilledWaitTime()) { @@ -762,12 +774,16 @@ export class ExpectationManager { if (fulfilled.fulfilled) { // Yes it is still fullfiled // No need to update the tracked state, since it's already fullfilled: - // this.updateTrackedExp(trackedExp, TrackedExpectationState.FULFILLED, fulfilled.reason) + // this.updateTrackedExp(trackedExp, WorkStatusState.FULFILLED, fulfilled.reason) } else { // It appears like it's not fullfilled anymore trackedExp.status.actualVersionHash = undefined trackedExp.status.workProgress = undefined - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.WAITING, fulfilled.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + fulfilled.reason + ) trackedExp.session.triggerExpectationAgain = true } } else { @@ -782,7 +798,7 @@ export class ExpectationManager { } else { // Do nothing } - } else if (trackedExp.state === TrackedExpectationState.REMOVED) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.REMOVED) { await this.assignWorkerToSession(trackedExp.session, trackedExp) if (trackedExp.session.assignedWorker) { const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) @@ -791,7 +807,11 @@ export class ExpectationManager { this.callbacks.reportExpectationStatus(trackedExp.id, null, null, {}) } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.REMOVED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.REMOVED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -802,19 +822,23 @@ export class ExpectationManager { this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.RESTARTED) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.RESTARTED) { await this.assignWorkerToSession(trackedExp.session, trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { user: 'Ready to start (after restart)', tech: 'Ready to start (after restart)', }) trackedExp.session.triggerExpectationAgain = true } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.RESTARTED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.RESTARTED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -825,19 +849,23 @@ export class ExpectationManager { this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.ABORTED) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.ABORTED) { await this.assignWorkerToSession(trackedExp.session, trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { // This will cause the expectation to be intentionally stuck in the ABORTED state. - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.ABORTED, { user: 'Aborted', tech: 'Aborted', }) } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.ABORTED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -872,7 +900,7 @@ export class ExpectationManager { /** Update the state and status of a trackedExpectation */ private updateTrackedExpStatus( trackedExp: TrackedExpectation, - state: TrackedExpectationState | undefined, + state: ExpectedPackageStatusAPI.WorkStatusState | undefined, reason: Reason | undefined, newStatus?: Partial ) { @@ -921,7 +949,7 @@ export class ExpectationManager { } } private updatePackageContainerPackageStatus(trackedExp: TrackedExpectation) { - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { for (const fromPackage of trackedExp.exp.fromPackages) { // TODO: this is probably not eh right thing to do: for (const packageContainer of trackedExp.exp.endRequirement.targets) { @@ -941,9 +969,9 @@ export class ExpectationManager { private getPackageStatus( trackedExp: TrackedExpectation ): ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus { - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { return ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - } else if (trackedExp.state === TrackedExpectationState.WORKING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { return trackedExp.status.targetCanBeUsedWhileTransferring ? ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.TRANSFERRING_READY : ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.TRANSFERRING_NOT_READY @@ -1026,7 +1054,7 @@ export class ExpectationManager { */ private handleTriggerByFullfilledIds(trackedExp: TrackedExpectation): boolean { let hasTriggeredSomething = false - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { const toTriggerIds = this._triggerByFullfilledIds[trackedExp.id] || [] for (const id of toTriggerIds) { @@ -1049,7 +1077,7 @@ export class ExpectationManager { // Check if those are fullfilled: let waitingFor: TrackedExpectation | undefined = undefined for (const id of trackedExp.exp.dependsOnFullfilled) { - if (this.trackedExpectations[id].state !== TrackedExpectationState.FULFILLED) { + if (this.trackedExpectations[id].state !== ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { waitingFor = this.trackedExpectations[id] break } @@ -1269,19 +1297,7 @@ export type ExpectationManagerServerOptions = | { type: 'internal' } -/** Denotes the various states of a Tracked Expectation */ -export enum TrackedExpectationState { - NEW = 'new', - WAITING = 'waiting', - READY = 'ready', - WORKING = 'working', - FULFILLED = 'fulfilled', - REMOVED = 'removed', - - // Triggered from Core: - RESTARTED = 'restarted', - ABORTED = 'aborted', -} + interface TrackedExpectation { /** Unique ID of the tracked expectation */ id: string @@ -1289,7 +1305,7 @@ interface TrackedExpectation { exp: Expectation.Any /** The current State of the expectation. */ - state: TrackedExpectationState + state: ExpectedPackageStatusAPI.WorkStatusState /** Reason for the current state. */ reason: Reason @@ -1344,7 +1360,7 @@ export interface ExpectationManagerCallbacks { expectaction: Expectation.Any | null, actualVersionHash: string | null, statusInfo: { - status?: string + status?: ExpectedPackageStatusAPI.WorkStatusState progress?: number statusReason?: Reason } diff --git a/yarn.lock b/yarn.lock index a7dba6ce..ee398bbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1478,15 +1478,13 @@ tslib "^2.0.3" underscore "1.12.0" -"@sofie-automation/blueprints-integration@1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0": - version "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0.tgz#6e9279af9dac23e88df9e76bb4b41cb9cc04c446" - integrity sha512-4SsMbHcEzUVk24G7maJQXg2i9F8rdfGn/rV0Pn95Gih4OlJxefne4T5WKS6kOY5A3C8ZGDNRolHR/WtrbcOHLA== +"@sofie-automation/blueprints-integration@1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0": + version "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0.tgz#ef204f6fa1ead29e8e566c3232804357e5250c89" + integrity sha512-zxwHV+eHaRaDPvSDiXDCoL8y/1NaqvDlmYQsAi7ZGi9ayGYTbluNbOaGYxD/6Q/Elqze4oC+9wSgmoep6l6Wmg== dependencies: - moment "2.29.1" - timeline-state-resolver-types "5.9.0-nightly-release34-20210511-085833-30ad952a1.0" + timeline-state-resolver-types "6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0" tslib "^2.1.0" - underscore "1.13.1" "@sofie-automation/code-standard-preset@^0.2.1", "@sofie-automation/code-standard-preset@^0.2.2": version "0.2.2" @@ -1507,14 +1505,14 @@ read-pkg-up "^7.0.1" shelljs "^0.8.4" -"@sofie-automation/server-core-integration@1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0": - version "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0.tgz#dc115837aa303bb1c7741140c1a6de5c453f32b5" - integrity sha512-yy3wTxfLef+821BVshwozVBJEwOIZhUpQqFJDcQ9AVZDUzJEl97c+q++V/FvJ2SPbT/5FqFwVp3uzlnVUIWzIA== +"@sofie-automation/server-core-integration@1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0": + version "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0.tgz#1e575cd953dda3a30dc8a66e38c50e327cf3701f" + integrity sha512-ZnPwYBeXcYBR8971XKZxYDfbxPgvq+/BKZGvVmJxot1JEmPiOtybZncSwbn5XkhOig4mJA3VOc+efYIBpEpqVw== dependencies: - data-store "3.1.0" + data-store "^4.0.3" ejson "^2.2.0" - faye-websocket "^0.11.3" + faye-websocket "^0.11.4" got "^11.8.2" tslib "^2.0.3" underscore "^1.12.1" @@ -3361,10 +3359,13 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -data-store@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/data-store/-/data-store-3.1.0.tgz#9464f2c8ac8cad5cd0ebb6992eaf354d7ea8c35c" - integrity sha512-MjbLiqz5IJFD/NZLztrrxc2LZ8KMc42kHWAUSxD/kp2ekzHE8EZfkYP4nQy15aPMwV5vac2dW21Ni72okNAwTQ== +data-store@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/data-store/-/data-store-4.0.3.tgz#d055c55f3fa776daa3a5664cf715d3fe689c3afb" + integrity sha512-AhPSqGVCSrFgi1PtkOLxYRrVhzbsma/drtxNkwTrT2q+LpLTG3AhOW74O/IMRy1cWftBx93SuLLKF19YsKkUMg== + dependencies: + get-value "^3.0.1" + set-value "^3.0.1" data-urls@^2.0.0: version "2.0.0" @@ -4287,10 +4288,10 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -faye-websocket@^0.11.3: - version "0.11.3" - resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.3.tgz#5c0e9a8968e8912c286639fde977a8b209f2508e" - integrity sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA== +faye-websocket@^0.11.4: + version "0.11.4" + resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.4.tgz#7f0d9275cfdd86a1c963dc8b65fcc451edcbb1da" + integrity sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g== dependencies: websocket-driver ">=0.5.1" @@ -4736,6 +4737,13 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= +get-value@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-3.0.1.tgz#5efd2a157f1d6a516d7524e124ac52d0a39ef5a8" + integrity sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA== + dependencies: + isobject "^3.0.1" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -8864,6 +8872,13 @@ set-value@^2.0.0, set-value@^2.0.1: is-plain-object "^2.0.3" split-string "^3.0.1" +set-value@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-3.0.2.tgz#74e8ecd023c33d0f77199d415409a40f21e61b90" + integrity sha512-npjkVoz+ank0zjlV9F47Fdbjfj/PfXyVhZvGALWsyIYU/qrMzpi6avjKW3/7KeSU2Df3I46BrN1xOI1+6vW0hA== + dependencies: + is-plain-object "^2.0.4" + setprototypeof@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" @@ -9648,12 +9663,12 @@ timeline-state-resolver-types@5.5.1: dependencies: tslib "^1.13.0" -timeline-state-resolver-types@5.9.0-nightly-release34-20210511-085833-30ad952a1.0: - version "5.9.0-nightly-release34-20210511-085833-30ad952a1.0" - resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-5.9.0-nightly-release34-20210511-085833-30ad952a1.0.tgz#6edada5ea4199b290645c9ced2a4051ed68985ea" - integrity sha512-3M9CptINGuerTEJGriuez4PqFqyD3Cf2ZZ9lI0cv4mnXOTLt3kwbp4a0uc2HWLBgzPsYPM9JEz2Y20XFSJC4TQ== +timeline-state-resolver-types@6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0: + version "6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0" + resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0.tgz#c0cbf83b95204cc1e4e10b176f120a41d1abca72" + integrity sha512-SqOfKD5orC4HcIX/pVEwfdL+B6LEsP+47McAqljUS7Ha64WC6OxSQfA9TL8I2TEHt07rj85ZbtkXZOBhxO+lFw== dependencies: - tslib "^1.13.0" + tslib "^2.1.0" tmp@^0.0.33: version "0.0.33" @@ -9961,11 +9976,6 @@ underscore@1.12.0, underscore@^1.12.0: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.0.tgz#4814940551fc80587cef7840d1ebb0f16453be97" integrity sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ== -underscore@1.13.1: - version "1.13.1" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.1.tgz#0c1c6bd2df54b6b69f2314066d65b6cde6fcf9d1" - integrity sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g== - underscore@^1.12.1: version "1.12.1" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.1.tgz#7bb8cc9b3d397e201cf8553336d262544ead829e" From 8ab3a49f56ab64d0c27eb4b8703c810e8685214d Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 30 Jun 2021 11:15:21 +0200 Subject: [PATCH 35/67] feat: add WorkStatus property "statusChanged", used in Sofie GUI to indicate that a status has changed. --- .../packages/generic/src/packageManager.ts | 39 +++++++++++++++---- package.json | 4 +- .../src/expectationManager.ts | 2 +- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 0fc03d86..970bdb50 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -389,16 +389,18 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks } else { if (!expectaction.statusReport.sendReport) return // Don't report the status - // Default properties: + const previouslyReported = this.toReportExpectationStatus[expectationId]?.workStatus + const workStatus: ExpectedPackageStatusAPI.WorkStatus = { + // Default properties: ...{ status: ExpectedPackageStatusAPI.WorkStatusState.NEW, + statusChanged: 0, progress: 0, statusReason: { user: '', tech: '' }, }, // Previous properties: - ...((this.toReportExpectationStatus[expectationId]?.workStatus || - {}) as Partial), // Intentionally cast to Partial<>, to make typings in const workStatus more strict + ...((previouslyReported || {}) as Partial), // Intentionally cast to Partial<>, to make typings in const workStatus more strict // Updated properties: ...expectaction.statusReport, @@ -416,32 +418,55 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks }), } + // Update statusChanged: + workStatus.statusChanged = previouslyReported?.statusChanged || Date.now() + if ( + workStatus.status !== previouslyReported?.status || + workStatus.progress !== previouslyReported?.progress + // (not checking statusReason, as that should not affect statusChanged) + ) { + workStatus.statusChanged = Date.now() + } + this.triggerSendUpdateExpectationStatus(expectationId, workStatus) } } public reportPackageContainerPackageStatus( containerId: string, packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + packageStatus: Omit | null ): void { const packageContainerPackageId = `${containerId}_${packageId}` if (!packageStatus) { this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, null) } else { - const o: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { + const previouslyReported = this.toReportPackageStatus[packageContainerPackageId]?.packageStatus + + const containerStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { // Default properties: ...{ status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, progress: 0, + statusChanged: 0, statusReason: { user: '', tech: '' }, }, // pre-existing properties: - ...(((this.toReportPackageStatus[packageContainerPackageId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in the outer spread-assignment more strict + ...((previouslyReported || {}) as Partial), // Intentionally cast to Partial<>, to make typings in const containerStatus more strict // Updated properties: ...packageStatus, } - this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, o) + // Update statusChanged: + containerStatus.statusChanged = previouslyReported?.statusChanged || Date.now() + if ( + containerStatus.status !== previouslyReported?.status || + containerStatus.progress !== previouslyReported?.progress + // (not checking statusReason, as that should not affect statusChanged) + ) { + containerStatus.statusChanged = Date.now() + } + + this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, containerStatus) } } public reportPackageContainerExpectationStatus( diff --git a/package.json b/package.json index b21d60b6..06404ad0 100644 --- a/package.json +++ b/package.json @@ -56,8 +56,8 @@ "node": ">=12.3.0" }, "dependencies": { - "@sofie-automation/blueprints-integration": "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0", - "@sofie-automation/server-core-integration": "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" + "@sofie-automation/blueprints-integration": "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0", + "@sofie-automation/server-core-integration": "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "husky": { diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 4e22a388..385a0307 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -1368,7 +1368,7 @@ export interface ExpectationManagerCallbacks { reportPackageContainerPackageStatus: ( containerId: string, packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + packageStatus: Omit | null ) => void reportPackageContainerExpectationStatus: ( containerId: string, From 2dcbdec30d539a069850eab104066b35a7356b6b Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 20 Aug 2021 07:16:07 +0200 Subject: [PATCH 36/67] chore: bunmp required node version (for worker-threads support) --- apps/_boilerplate/app/package.json | 2 +- apps/_boilerplate/packages/generic/package.json | 4 ++-- apps/http-server/app/package.json | 4 ++-- apps/http-server/packages/generic/package.json | 4 ++-- apps/package-manager/app/package.json | 4 ++-- apps/package-manager/packages/generic/package.json | 4 ++-- apps/quantel-http-transformer-proxy/app/package.json | 2 +- .../packages/generic/package.json | 4 ++-- apps/single-app/app/package.json | 4 ++-- apps/worker/app/package.json | 4 ++-- apps/worker/packages/generic/package.json | 4 ++-- apps/workforce/app/package.json | 4 ++-- apps/workforce/packages/generic/package.json | 4 ++-- commonPackage.json | 2 +- package.json | 2 +- shared/packages/api/package.json | 4 ++-- shared/packages/expectationManager/package.json | 4 ++-- shared/packages/worker/package.json | 4 ++-- shared/packages/workforce/package.json | 4 ++-- tests/internal-tests/package.json | 4 ++-- 20 files changed, 36 insertions(+), 36 deletions(-) diff --git a/apps/_boilerplate/app/package.json b/apps/_boilerplate/app/package.json index 2d30537a..7ad0ae2c 100644 --- a/apps/_boilerplate/app/package.json +++ b/apps/_boilerplate/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/_boilerplate/packages/generic/package.json b/apps/_boilerplate/packages/generic/package.json index 57a24903..8ad6904c 100644 --- a/apps/_boilerplate/packages/generic/package.json +++ b/apps/_boilerplate/packages/generic/package.json @@ -19,7 +19,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -29,4 +29,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/http-server/app/package.json b/apps/http-server/app/package.json index 0ce9aa87..b46af52e 100644 --- a/apps/http-server/app/package.json +++ b/apps/http-server/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -33,4 +33,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index 852d7665..6305ab80 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -48,7 +48,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -58,4 +58,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/package-manager/app/package.json b/apps/package-manager/app/package.json index 684baf9f..8de9eae4 100644 --- a/apps/package-manager/app/package.json +++ b/apps/package-manager/app/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/package-manager/packages/generic/package.json b/apps/package-manager/packages/generic/package.json index 419a3fcb..7aba13af 100644 --- a/apps/package-manager/packages/generic/package.json +++ b/apps/package-manager/packages/generic/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/quantel-http-transformer-proxy/app/package.json b/apps/quantel-http-transformer-proxy/app/package.json index 5d4eeef8..96b02e23 100644 --- a/apps/quantel-http-transformer-proxy/app/package.json +++ b/apps/quantel-http-transformer-proxy/app/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index 3af4be38..fb811abd 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -53,7 +53,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -63,4 +63,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index 3eb66d4a..caa448d0 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -29,7 +29,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -39,4 +39,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/worker/app/package.json b/apps/worker/app/package.json index 40613280..feee832b 100644 --- a/apps/worker/app/package.json +++ b/apps/worker/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -33,4 +33,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/worker/packages/generic/package.json b/apps/worker/packages/generic/package.json index eed5c165..0f4a6a9d 100644 --- a/apps/worker/packages/generic/package.json +++ b/apps/worker/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/workforce/app/package.json b/apps/workforce/app/package.json index defc4eab..ad4a76c8 100644 --- a/apps/workforce/app/package.json +++ b/apps/workforce/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -33,4 +33,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/workforce/packages/generic/package.json b/apps/workforce/packages/generic/package.json index 18778c26..52a37fbb 100644 --- a/apps/workforce/packages/generic/package.json +++ b/apps/workforce/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/commonPackage.json b/commonPackage.json index 558b1485..ba9450d5 100644 --- a/commonPackage.json +++ b/commonPackage.json @@ -4,7 +4,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "devDependencies": { "lint-staged": "^7.2.0" diff --git a/package.json b/package.json index 06404ad0..882e066c 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "fs-extra": "^9.1.0" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "dependencies": { "@sofie-automation/blueprints-integration": "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0", diff --git a/shared/packages/api/package.json b/shared/packages/api/package.json index 957f1f68..504e50b2 100644 --- a/shared/packages/api/package.json +++ b/shared/packages/api/package.json @@ -29,7 +29,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -39,4 +39,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/expectationManager/package.json b/shared/packages/expectationManager/package.json index 99ba85b7..65f148d3 100644 --- a/shared/packages/expectationManager/package.json +++ b/shared/packages/expectationManager/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "devDependencies": { "lint-staged": "^7.2.0" @@ -35,4 +35,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json index 0b6ff040..f56ad333 100644 --- a/shared/packages/worker/package.json +++ b/shared/packages/worker/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "devDependencies": { "@types/deep-diff": "^1.0.0", @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/workforce/package.json b/shared/packages/workforce/package.json index 21c432c8..ff100a13 100644 --- a/shared/packages/workforce/package.json +++ b/shared/packages/workforce/package.json @@ -24,7 +24,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -34,4 +34,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/tests/internal-tests/package.json b/tests/internal-tests/package.json index bf8e050d..670af7a7 100644 --- a/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file From e8f4d2b2f1ec4fe21cc8df3756b6f6989e192bc7 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 23 Aug 2021 15:08:55 +0200 Subject: [PATCH 37/67] chore: improve build:changed script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 882e066c..c108e91e 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "ci": "yarn install && yarn build && yarn lint && yarn test", "setup": "lerna bootstrap", "build": "lerna run build --stream", - "build:changed": "lerna run --since origin/master --include-dependents build", + "build:changed": "lerna run build --since head --exclude-dependents --stream", "lint": "lerna exec -- eslint . --ext .ts,.tsx", "lintfix": "yarn lint --fix", "lint:changed": "lerna exec --since origin/master --include-dependents -- eslint . --ext .js,.jsx,.ts,.tsx", From a4a7b3ac0aa654bcf97278c05eb3d5df50a31ae7 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 23 Aug 2021 15:15:23 +0200 Subject: [PATCH 38/67] feat: add AppContainer and basic functionality for workforce to spin up workers --- apps/appcontainer-node/app/.gitignore | 2 + apps/appcontainer-node/app/jest.config.js | 8 + apps/appcontainer-node/app/package.json | 36 +++ .../app/scripts/copy-natives.js | 33 +++ .../app/src/__tests__/test.spec.ts | 7 + apps/appcontainer-node/app/src/index.ts | 5 + apps/appcontainer-node/app/tsconfig.json | 6 + .../packages/generic/jest.config.js | 8 + .../packages/generic/package.json | 35 +++ .../generic/src/__tests__/temp.spec.ts | 6 + .../packages/generic/src/appContainer.ts | 205 ++++++++++++++++++ .../packages/generic/src/index.ts | 18 ++ .../packages/generic/src/workforceApi.ts | 17 ++ .../packages/generic/tsconfig.json | 6 + .../packages/generic/src/connector.ts | 13 +- apps/single-app/app/package.json | 3 +- apps/single-app/app/src/index.ts | 67 +----- apps/single-app/app/src/singleApp.ts | 61 ++++++ shared/packages/api/src/appContainer.ts | 50 +++++ shared/packages/api/src/config.ts | 57 ++++- shared/packages/api/src/index.ts | 1 + shared/packages/api/src/methods.ts | 16 ++ .../packages/api/src/websocketConnection.ts | 2 +- shared/packages/api/src/websocketServer.ts | 7 + .../src/expectationManager.ts | 47 ++-- shared/packages/worker/src/workerAgent.ts | 7 +- .../packages/workforce/src/appContainerApi.ts | 27 +++ .../packages/workforce/src/workerHandler.ts | 114 ++++++++++ shared/packages/workforce/src/workforce.ts | 119 ++++++++-- yarn.lock | 16 +- 30 files changed, 879 insertions(+), 120 deletions(-) create mode 100644 apps/appcontainer-node/app/.gitignore create mode 100644 apps/appcontainer-node/app/jest.config.js create mode 100644 apps/appcontainer-node/app/package.json create mode 100644 apps/appcontainer-node/app/scripts/copy-natives.js create mode 100644 apps/appcontainer-node/app/src/__tests__/test.spec.ts create mode 100644 apps/appcontainer-node/app/src/index.ts create mode 100644 apps/appcontainer-node/app/tsconfig.json create mode 100644 apps/appcontainer-node/packages/generic/jest.config.js create mode 100644 apps/appcontainer-node/packages/generic/package.json create mode 100644 apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts create mode 100644 apps/appcontainer-node/packages/generic/src/appContainer.ts create mode 100644 apps/appcontainer-node/packages/generic/src/index.ts create mode 100644 apps/appcontainer-node/packages/generic/src/workforceApi.ts create mode 100644 apps/appcontainer-node/packages/generic/tsconfig.json create mode 100644 apps/single-app/app/src/singleApp.ts create mode 100644 shared/packages/api/src/appContainer.ts create mode 100644 shared/packages/workforce/src/appContainerApi.ts create mode 100644 shared/packages/workforce/src/workerHandler.ts diff --git a/apps/appcontainer-node/app/.gitignore b/apps/appcontainer-node/app/.gitignore new file mode 100644 index 00000000..d11e3a5a --- /dev/null +++ b/apps/appcontainer-node/app/.gitignore @@ -0,0 +1,2 @@ +log.log +deploy/* diff --git a/apps/appcontainer-node/app/jest.config.js b/apps/appcontainer-node/app/jest.config.js new file mode 100644 index 00000000..ba142d81 --- /dev/null +++ b/apps/appcontainer-node/app/jest.config.js @@ -0,0 +1,8 @@ +const base = require('../../../jest.config.base'); +const packageJson = require('./package'); + +module.exports = { + ...base, + name: packageJson.name, + displayName: packageJson.name, +}; diff --git a/apps/appcontainer-node/app/package.json b/apps/appcontainer-node/app/package.json new file mode 100644 index 00000000..e6f9795c --- /dev/null +++ b/apps/appcontainer-node/app/package.json @@ -0,0 +1,36 @@ +{ + "name": "@appcontainer-node/app", + "version": "1.0.0", + "description": "AppContainer-Node.js", + "private": true, + "scripts": { + "build": "rimraf dist && yarn build:main", + "build:main": "tsc -p tsconfig.json", + "build-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "__test": "jest", + "start": "node dist/index.js", + "precommit": "lint-staged" + }, + "devDependencies": { + "nexe": "^3.3.7", + "lint-staged": "^7.2.0" + }, + "dependencies": { + "@appcontainer-node/generic": "*" + }, + "peerDependencies": { + "@sofie-automation/blueprints-integration": "*" + }, + "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", + "engines": { + "node": ">=12.11.0" + }, + "lint-staged": { + "*.{js,css,json,md,scss}": [ + "prettier" + ], + "*.{ts,tsx}": [ + "eslint" + ] + } +} diff --git a/apps/appcontainer-node/app/scripts/copy-natives.js b/apps/appcontainer-node/app/scripts/copy-natives.js new file mode 100644 index 00000000..763c96fb --- /dev/null +++ b/apps/appcontainer-node/app/scripts/copy-natives.js @@ -0,0 +1,33 @@ +const find = require('find'); +const os = require('os') +const path = require('path') +const fs = require('fs-extra') + +const arch = os.arch() +const platform = os.platform() +const prebuildType = process.argv[2] || `${platform}-${arch}` + +function isFileForPlatform(filename) { + if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { + return true + } else { + return false + } +} + +const dirName = path.join(__dirname, '../../../..') +console.log('Running in', dirName, 'for', prebuildType) + +console.log(process.argv[2]) + +find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { + files.forEach(fullPath => { + if (fullPath.indexOf(dirName) === 0) { + const file = fullPath.substr(dirName.length + 1) + if (isFileForPlatform(file)) { + console.log('Copy prebuild binary:', file) + fs.copySync(file, path.join('deploy', file)) + } + } + }); +}) diff --git a/apps/appcontainer-node/app/src/__tests__/test.spec.ts b/apps/appcontainer-node/app/src/__tests__/test.spec.ts new file mode 100644 index 00000000..b65fc1f7 --- /dev/null +++ b/apps/appcontainer-node/app/src/__tests__/test.spec.ts @@ -0,0 +1,7 @@ +describe('tmp', () => { + test('tmp', () => { + // Note: To enable tests in this package, ensure that the "test" script is present in package.json + expect(1).toEqual(1) + }) +}) +export {} diff --git a/apps/appcontainer-node/app/src/index.ts b/apps/appcontainer-node/app/src/index.ts new file mode 100644 index 00000000..fc2a51a1 --- /dev/null +++ b/apps/appcontainer-node/app/src/index.ts @@ -0,0 +1,5 @@ +import { startProcess } from '@appcontainer-node/generic' +/* eslint-disable no-console */ + +console.log('process started') // This is a message all Sofie processes log upon startup +startProcess().catch(console.error) diff --git a/apps/appcontainer-node/app/tsconfig.json b/apps/appcontainer-node/app/tsconfig.json new file mode 100644 index 00000000..4e2fdaf5 --- /dev/null +++ b/apps/appcontainer-node/app/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + } +} diff --git a/apps/appcontainer-node/packages/generic/jest.config.js b/apps/appcontainer-node/packages/generic/jest.config.js new file mode 100644 index 00000000..269ab60e --- /dev/null +++ b/apps/appcontainer-node/packages/generic/jest.config.js @@ -0,0 +1,8 @@ +const base = require('../../../../jest.config.base') +const packageJson = require('./package') + +module.exports = { + ...base, + name: packageJson.name, + displayName: packageJson.name, +} diff --git a/apps/appcontainer-node/packages/generic/package.json b/apps/appcontainer-node/packages/generic/package.json new file mode 100644 index 00000000..9a35f2dd --- /dev/null +++ b/apps/appcontainer-node/packages/generic/package.json @@ -0,0 +1,35 @@ +{ + "name": "@appcontainer-node/generic", + "version": "1.0.0", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "rimraf dist && yarn build:main", + "build:main": "tsc -p tsconfig.json", + "__test": "jest", + "precommit": "lint-staged" + }, + "peerDependencies": { + "@sofie-automation/blueprints-integration": "*" + }, + "dependencies": { + "@shared/api": "*", + "@shared/worker": "*" + }, + "devDependencies": { + "lint-staged": "^7.2.0" + }, + "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", + "engines": { + "node": ">=12.11.0" + }, + "lint-staged": { + "*.{js,css,json,md,scss}": [ + "prettier" + ], + "*.{ts,tsx}": [ + "eslint" + ] + } +} \ No newline at end of file diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts new file mode 100644 index 00000000..e62eae89 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts @@ -0,0 +1,6 @@ +describe('Temp', () => { + // This is just a placeholder, to be replaced with real tests later on + test('basic math', () => { + expect(1 + 1).toEqual(2) + }) +}) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts new file mode 100644 index 00000000..de76c53e --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -0,0 +1,205 @@ +import * as ChildProcess from 'child_process' +import * as path from 'path' +import * as fs from 'fs' +import { + LoggerInstance, + AppContainerProcessConfig, + ClientConnectionOptions, + AppContainer as NSAppContainer, +} from '@shared/api' +import { WorkforceAPI } from './workforceApi' + +/** Mimimum time between app restarts */ +const RESTART_COOLDOWN = 60 * 1000 // ms + +export class AppContainer { + private workforceAPI: WorkforceAPI + private id: string + private connectionOptions: ClientConnectionOptions + private appId = 0 + + private apps: { + [appId: string]: { + process: ChildProcess.ChildProcess + appType: NSAppContainer.AppType + toBeKilled: boolean + restarts: number + lastRestart: number + } + } = {} + private availableApps: { + [appType: string]: AvailableAppInfo + } = {} + + constructor(private logger: LoggerInstance, private config: AppContainerProcessConfig) { + this.workforceAPI = new WorkforceAPI(this.logger) + + this.id = config.appContainer.appContainerId + this.connectionOptions = this.config.appContainer.workforceURL + ? { + type: 'websocket', + url: this.config.appContainer.workforceURL, + } + : { + type: 'internal', + } + } + async init(): Promise { + if (this.connectionOptions.type === 'websocket') { + this.logger.info(`AppContainer: Connecting to Workforce at "${this.connectionOptions.url}"`) + } + + await this.setupAvailableApps() + + await this.workforceAPI.init(this.id, this.connectionOptions, this) + + await this.workforceAPI.registerAvailableApps( + Object.entries(this.availableApps).map((o) => { + const appType = o[0] as NSAppContainer.AppType + return { + appType: appType, + } + }) + ) + + // Todo later: + // Sent the workforce a list of: + // * the types of workers we can spin up + // * what the running cost is for them + // * how many can be spun up + // * etc... + } + private async setupAvailableApps() { + const getWorkerArgs = (appId: string): string[] => { + return [ + `--workerId=${appId}`, + `--workforceURL=${this.config.appContainer.workforceURL}`, + this.config.appContainer.windowsDriveLetters + ? `--windowsDriveLetters=${this.config.appContainer.windowsDriveLetters?.join(';')}` + : '', + this.config.appContainer.resourceId ? `--resourceId=${this.config.appContainer.resourceId}` : '', + this.config.appContainer.networkIds.length + ? `--networkIds=${this.config.appContainer.networkIds.join(';')}` + : '', + ] + } + if (process.execPath.match(/node.exe$/)) { + // Process runs as a node process, we're probably in development mode. + this.availableApps['worker'] = { + file: process.execPath, + args: (appId: string) => { + return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId)] + }, + } + } else { + // Process is a compiled executable + // Look for the worker executable in the same folder: + + const dirPath = path.dirname(process.execPath) + // Note: nexe causes issues with its virtual file system: https://github.com/nexe/nexe/issues/613#issuecomment-579107593 + + ;(await fs.promises.readdir(dirPath)).forEach((fileName) => { + if (fileName.match(/worker/i)) { + this.availableApps['worker'] = { + file: path.join(dirPath, fileName), + args: (appId: string) => { + return [...getWorkerArgs(appId)] + }, + } + } + }) + } + } + terminate(): void { + this.workforceAPI.terminate() + + // kill child processes + } + + async spinUp(appType: AppType): Promise { + const availableApp = this.availableApps[appType] + if (!availableApp) throw new Error(`Unknown appType "${appType}"`) + + const appId = `${this.id}_${this.appId++}` + + const child = this.setupChildProcess(appType, appId, availableApp) + this.apps[appId] = { + process: child, + appType: appType, + toBeKilled: false, + restarts: 0, + lastRestart: 0, + } + return appId + } + async spinDown(appId: string): Promise { + const app = this.apps[appId] + if (!app) throw new Error(`App "${appId}" not found`) + + app.toBeKilled = true + const success = app.process.kill() + if (!success) throw new Error(`Internal error: Killing of process "${app.process.pid}" failed`) + + app.process.removeAllListeners() + delete this.apps[appId] + } + async getRunningApps(): Promise<{ appId: string; appType: NSAppContainer.AppType }[]> { + return Object.entries(this.apps).map((o) => { + const [appId, app] = o + + return { + appId: appId, + appType: app.appType, + } + }) + } + private setupChildProcess( + appType: AppType, + appId: string, + availableApp: AvailableAppInfo + ): ChildProcess.ChildProcess { + const cwd = process.execPath.match(/node.exe$/) + ? undefined // Process runs as a node process, we're probably in development mode. + : path.dirname(process.execPath) // Process runs as a node process, we're probably in development mode. + + const child = ChildProcess.execFile(availableApp.file, availableApp.args(appId), { + cwd: cwd, + }) + + child.stdout?.on('data', (data) => { + this.logger.debug(`${appId} stdout: ${data}`) + }) + child.stderr?.on('data', (data) => { + this.logger.debug(`${appId} stderr: ${data}`) + }) + child.once('close', (code) => { + const app = this.apps[appId] + if (app && !app.toBeKilled) { + // Try to restart the application + + const timeUntilRestart = Math.max(0, app.lastRestart - Date.now() + RESTART_COOLDOWN) + this.logger.warn( + `App ${app.process.pid} (${appType}) closed with code (${code}), trying to restart in ${timeUntilRestart} ms (restarts: ${app.restarts})` + ) + + setTimeout(() => { + app.lastRestart = Date.now() + app.restarts++ + + app.process.removeAllListeners() + + const newChild = this.setupChildProcess(appType, appId, availableApp) + + app.process = newChild + }, timeUntilRestart) + } + }) + + return child + } +} +type AppType = 'worker' // | other +interface AvailableAppInfo { + file: string + args: (appId: string) => string[] +} diff --git a/apps/appcontainer-node/packages/generic/src/index.ts b/apps/appcontainer-node/packages/generic/src/index.ts new file mode 100644 index 00000000..bd87a44b --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/index.ts @@ -0,0 +1,18 @@ +import { getAppContainerConfig, setupLogging } from '@shared/api' +import { AppContainer } from './appContainer' + +export { AppContainer } from './appContainer' + +export async function startProcess(): Promise { + const config = getAppContainerConfig() + + const logger = setupLogging(config) + + logger.info('------------------------------------------------------------------') + logger.info('Starting AppContainer') + logger.info('------------------------------------------------------------------') + + const appContainer = new AppContainer(logger, config) + + appContainer.init().catch(logger.error) +} diff --git a/apps/appcontainer-node/packages/generic/src/workforceApi.ts b/apps/appcontainer-node/packages/generic/src/workforceApi.ts new file mode 100644 index 00000000..aabca95a --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/workforceApi.ts @@ -0,0 +1,17 @@ +import { AdapterClient, LoggerInstance, WorkForceAppContainer, AppContainer } from '@shared/api' + +/** + * Exposes the API-methods of a Workforce, to be called from the AppContainer + * Note: The AppContainer connects to the Workforce, therefore the AppContainer is the AdapterClient here. + * The corresponding other side is implemented at shared/packages/workforce/src/appContainerApi.ts + */ +export class WorkforceAPI + extends AdapterClient + implements WorkForceAppContainer.WorkForce { + constructor(logger: LoggerInstance) { + super(logger, 'appContainer') + } + async registerAvailableApps(availableApps: { appType: AppContainer.AppType }[]): Promise { + return this._sendMessage('registerAvailableApps', availableApps) + } +} diff --git a/apps/appcontainer-node/packages/generic/tsconfig.json b/apps/appcontainer-node/packages/generic/tsconfig.json new file mode 100644 index 00000000..b07e3e02 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + } +} diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts index d5fe2516..5b22a75c 100644 --- a/apps/package-manager/packages/generic/src/connector.ts +++ b/apps/package-manager/packages/generic/src/connector.ts @@ -36,12 +36,13 @@ export class Connector { this._process = new ProcessHandler(this._logger) this.coreHandler = new CoreHandler(this._logger, this.config.packageManager) - const packageManagerServerOptions: ExpectationManagerServerOptions = config.packageManager.port - ? { - type: 'websocket', - port: config.packageManager.port, - } - : { type: 'internal' } + const packageManagerServerOptions: ExpectationManagerServerOptions = + config.packageManager.port !== null + ? { + type: 'websocket', + port: config.packageManager.port, + } + : { type: 'internal' } const workForceConnectionOptions: ClientConnectionOptions = config.packageManager.workforceURL ? { diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index caa448d0..3e3ba2c9 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -16,6 +16,7 @@ "lint-staged": "^7.2.0" }, "dependencies": { + "@appcontainer-node/generic": "*", "@http-server/generic": "*", "@package-manager/generic": "*", "@quantel-http-transformer-proxy/generic": "*", @@ -39,4 +40,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/single-app/app/src/index.ts b/apps/single-app/app/src/index.ts index 2f3d488b..e0c64269 100644 --- a/apps/single-app/app/src/index.ts +++ b/apps/single-app/app/src/index.ts @@ -1,69 +1,6 @@ -import * as HTTPServer from '@http-server/generic' -import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/generic' -import * as PackageManager from '@package-manager/generic' -import * as Workforce from '@shared/workforce' -import * as Worker from '@shared/worker' -import { getSingleAppConfig, setupLogging } from '@shared/api' +import { startSingleApp } from './singleApp' // eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup -const config = getSingleAppConfig() -const logger = setupLogging(config) - -;(async function start() { - // Override some of the arguments, as they arent used in the single-app - config.packageManager.port = null - config.packageManager.accessUrl = null - config.packageManager.workforceURL = null - config.workforce.port = null - config.worker.workforceURL = null - - logger.info('------------------------------------------------------------------') - logger.info('Starting Package Manager - Single App') - - logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) - logger.info('------------------------------------------------------------------') - logger.info(JSON.stringify(config, undefined, 2)) - logger.info('------------------------------------------------------------------') - - const connector = new PackageManager.Connector(logger, config) - const expectationManager = connector.getExpectationManager() - - logger.info('Initializing HTTP proxy Server') - const httpServer = new HTTPServer.PackageProxyServer(logger, config) - await httpServer.init() - - logger.info('Initializing Quantel HTTP Transform proxy Server') - const quantelHTTPTransformerProxy = new QuantelHTTPTransformerProxy.QuantelHTTPTransformerProxy(logger, config) - await quantelHTTPTransformerProxy.init() - - logger.info('Initializing Workforce') - const workforce = new Workforce.Workforce(logger, config) - await workforce.init() - - logger.info('Initializing Package Manager (and Expectation Manager)') - expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) - await connector.init() - - const workerAgents: any[] = [] - for (let i = 0; i < config.singleApp.workerCount; i++) { - logger.info('Initializing worker') - const workerAgent = new Worker.WorkerAgent(logger, { - ...config, - worker: { - ...config.worker, - workerId: config.worker.workerId + '_' + i, - }, - }) - workerAgents.push(workerAgent) - - workerAgent.hookToWorkforce(workforce.getWorkerAgentHook()) - workerAgent.hookToExpectationManager(expectationManager.managerId, expectationManager.getWorkerAgentHook()) - await workerAgent.init() - } - - logger.info('------------------------------------------------------------------') - logger.info('Initialization complete') - logger.info('------------------------------------------------------------------') -})().catch(logger.error) +startSingleApp().catch(console.log) diff --git a/apps/single-app/app/src/singleApp.ts b/apps/single-app/app/src/singleApp.ts new file mode 100644 index 00000000..6e990a60 --- /dev/null +++ b/apps/single-app/app/src/singleApp.ts @@ -0,0 +1,61 @@ +// import * as ChildProcess from 'child_process' +// import * as path from 'path' + +import * as HTTPServer from '@http-server/generic' +import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/generic' +import * as PackageManager from '@package-manager/generic' +import * as Workforce from '@shared/workforce' +import * as AppConatainerNode from '@appcontainer-node/generic' +import { getSingleAppConfig, setupLogging } from '@shared/api' +// import { MessageToAppContainerSpinUp, MessageToAppContainerType } from './__api' + +export async function startSingleApp(): Promise { + const config = getSingleAppConfig() + const logger = setupLogging(config) + // Override some of the arguments, as they arent used in the single-app + config.packageManager.port = 0 // 0 = Set the packageManager port to whatever is available + config.packageManager.accessUrl = 'ws:127.0.0.1' + config.packageManager.workforceURL = null // Filled in later + config.workforce.port = 0 // 0 = Set the workforce port to whatever is available + + logger.info('------------------------------------------------------------------') + logger.info('Starting Package Manager - Single App') + + logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) + logger.info('------------------------------------------------------------------') + console.log(JSON.stringify(config, undefined, 2)) + logger.info('------------------------------------------------------------------') + + logger.info('Initializing Workforce') + const workforce = new Workforce.Workforce(logger, config) + await workforce.init() + if (!workforce.getPort()) throw new Error(`Internal Error: Got no workforce port`) + const workforceURL = `ws://127.0.0.1:${workforce.getPort()}` + + config.packageManager.workforceURL = workforceURL + config.appContainer.workforceURL = workforceURL + + logger.info('Initializing AppContainer') + const appContainer = new AppConatainerNode.AppContainer(logger, config) + await appContainer.init() + + logger.info('Initializing Package Manager Connector') + const connector = new PackageManager.Connector(logger, config) + const expectationManager = connector.getExpectationManager() + + logger.info('Initializing HTTP proxy Server') + const httpServer = new HTTPServer.PackageProxyServer(logger, config) + await httpServer.init() + + logger.info('Initializing Quantel HTTP Transform proxy Server') + const quantelHTTPTransformerProxy = new QuantelHTTPTransformerProxy.QuantelHTTPTransformerProxy(logger, config) + await quantelHTTPTransformerProxy.init() + + logger.info('Initializing Package Manager (and Expectation Manager)') + expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) + await connector.init() + + logger.info('------------------------------------------------------------------') + logger.info('Initialization complete') + logger.info('------------------------------------------------------------------') +} diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts new file mode 100644 index 00000000..afce7a06 --- /dev/null +++ b/shared/packages/api/src/appContainer.ts @@ -0,0 +1,50 @@ +import { WorkerAgentConfig } from './worker' + +/** The AppContainer is a host application responsible for spawning other applications */ + +export interface AppContainerConfig { + workforceURL: string | null + appContainerId: string + + resourceId: string + networkIds: string[] + + windowsDriveLetters: WorkerAgentConfig['windowsDriveLetters'] +} + +// eslint-disable-next-line @typescript-eslint/no-namespace +export namespace AppContainer { + export type AppType = 'worker' // | other + + export enum Type { + NODEJS = 'nodejs', + // DOCKER = 'docker', + // KUBERNETES = 'kubernetes', + } + + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Generic { + /** Information on how to access the AppContainer */ + export interface AppContainer { + type: Type + } + + /** Information about an App running in an AppContainer */ + export interface App { + /** Uniquely identifies a running instance of an app. */ + appId: string + } + } + + /** NodeJS app container */ + // eslint-disable-next-line @typescript-eslint/no-namespace + export namespace NodeJS { + export interface AppContainer extends Generic.AppContainer { + /** URL to the REST interface */ + url: string + } + export interface App extends Generic.App { + type: string // to be better defined later? + } + } +} diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 01af456b..aa6b1b83 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -2,6 +2,7 @@ import { Options } from 'yargs' import yargs = require('yargs/yargs') import _ from 'underscore' import { WorkerAgentConfig } from './worker' +import { AppContainerConfig } from './appContainer' /* * This file contains various CLI argument definitions, used by the various processes that together constitutes the Package Manager @@ -124,6 +125,35 @@ const workerArguments = defineArguments({ describe: 'Identifier of the local networks this worker has access to ("networkA;networkB")', }, }) +/** CLI-argument-definitions for the AppContainer process */ +const appContainerArguments = defineArguments({ + appContainerId: { + type: 'string', + default: process.env.APP_CONTAINER_ID || 'appContainer0', + describe: 'Unique id of the appContainer', + }, + workforceURL: { + type: 'string', + default: process.env.WORKFORCE_URL || 'ws://localhost:8070', + describe: 'The URL to the Workforce', + }, + + resourceId: { + type: 'string', + default: process.env.WORKER_NETWORK_ID || 'default', + describe: 'Identifier of the local resource/computer this worker runs on', + }, + networkIds: { + type: 'string', + default: process.env.WORKER_NETWORK_ID || 'default', + describe: 'Identifier of the local networks this worker has access to ("networkA;networkB")', + }, + windowsDriveLetters: { + type: 'string', + default: process.env.WORKER_WINDOWS_DRIVE_LETTERS || 'X;Y;Z', + describe: 'Which Windows Drive letters can be used to map shares. ("X;Y;Z") ', + }, +}) /** CLI-argument-definitions for the "Single" process */ const singleAppArguments = defineArguments({ workerCount: { @@ -199,7 +229,7 @@ export function getHTTPServerConfig(): HTTPServerConfig { }).argv if (!argv.apiKeyWrite && argv.apiKeyRead) { - throw `Error: When apiKeyRead is given, apiKeyWrite is required!` + throw new Error(`Error: When apiKeyRead is given, apiKeyWrite is required!`) } return { @@ -278,6 +308,28 @@ export function getWorkerConfig(): WorkerConfig { }, } } +// Configuration for the AppContainer Application: ------------------------------ +export interface AppContainerProcessConfig { + process: ProcessConfig + appContainer: AppContainerConfig +} +export function getAppContainerConfig(): AppContainerProcessConfig { + const argv = yargs(process.argv.slice(2)).options({ + ...appContainerArguments, + ...processOptions, + }).argv + + return { + process: getProcessConfig(argv), + appContainer: { + appContainerId: argv.appContainerId, + workforceURL: argv.workforceURL, + resourceId: argv.resourceId, + networkIds: argv.networkIds ? argv.networkIds.split(';') : [], + windowsDriveLetters: argv.windowsDriveLetters ? argv.windowsDriveLetters.split(';') : [], + }, + } +} // Configuration for the Single-app Application: ------------------------------ export interface SingleAppConfig @@ -285,6 +337,7 @@ export interface SingleAppConfig HTTPServerConfig, PackageManagerConfig, WorkerConfig, + AppContainerProcessConfig, QuantelHTTPTransformerProxyConfig { singleApp: { workerCount: number @@ -299,6 +352,7 @@ export function getSingleAppConfig(): SingleAppConfig { ...workerArguments, ...processOptions, ...singleAppArguments, + ...appContainerArguments, ...quantelHTTPTransformerProxyConfigArguments, } // Remove some that are not used in the Single-App, so that they won't show up when running '--help': @@ -323,6 +377,7 @@ export function getSingleAppConfig(): SingleAppConfig { singleApp: { workerCount: argv.workerCount || 1, }, + appContainer: getAppContainerConfig().appContainer, quantelHTTPTransformerProxy: getQuantelHTTPTransformerProxyConfig().quantelHTTPTransformerProxy, } } diff --git a/shared/packages/api/src/index.ts b/shared/packages/api/src/index.ts index 130f7462..7742e372 100644 --- a/shared/packages/api/src/index.ts +++ b/shared/packages/api/src/index.ts @@ -1,5 +1,6 @@ export * from './adapterClient' export * from './adapterServer' +export * from './appContainer' export * from './config' export * from './expectationApi' export * from './lib' diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 5eca03bd..252cfef5 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -2,6 +2,7 @@ import { ExpectedPackage, ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { Expectation } from './expectationApi' import { PackageContainerExpectation } from './packageContainerApi' +import { AppContainer as NSAppContainer } from './appContainer' import { ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -149,3 +150,18 @@ export namespace ExpectationManagerWorkerAgent { result?: any } } +/** Methods used by WorkForce and AppContainer */ +export namespace WorkForceAppContainer { + /** Methods on AppContainer, called by WorkForce */ + export interface AppContainer { + spinUp: ( + appType: 'worker' // | other + ) => Promise + spinDown: (appId: string) => Promise + getRunningApps: () => Promise<{ appId: string; appType: NSAppContainer.AppType }[]> + } + /** Methods on WorkForce, called by AppContainer */ + export interface WorkForce { + registerAvailableApps: (availableApps: { appType: NSAppContainer.AppType }[]) => Promise + } +} diff --git a/shared/packages/api/src/websocketConnection.ts b/shared/packages/api/src/websocketConnection.ts index aac1141c..b40b3d3c 100644 --- a/shared/packages/api/src/websocketConnection.ts +++ b/shared/packages/api/src/websocketConnection.ts @@ -103,7 +103,7 @@ export interface MessageReply { } export interface MessageIdentifyClient { internalType: 'identify_client' - clientType: 'N/A' | 'workerAgent' | 'expectationManager' + clientType: 'N/A' | 'workerAgent' | 'expectationManager' | 'appContainer' id: string } diff --git a/shared/packages/api/src/websocketServer.ts b/shared/packages/api/src/websocketServer.ts index 37fdb505..eea12fab 100644 --- a/shared/packages/api/src/websocketServer.ts +++ b/shared/packages/api/src/websocketServer.ts @@ -39,6 +39,13 @@ export class WebsocketServer { this.clients.forEach((client) => client.close()) this.wss.close() } + get port(): number { + const address = this.wss.address() + if (typeof address === 'string') + throw new Error(`Internal error: to be implemented: wss.address() as string "${address}"`) + + return address.port + } } export class ClientConnection extends WebsocketConnection { diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 385a0307..df18f4b6 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -97,31 +97,40 @@ export class ExpectationManager { public readonly managerId: string, private serverOptions: ExpectationManagerServerOptions, /** At what url the ExpectationManager can be reached on */ - private serverAccessUrl: string | undefined, + private serverAccessBaseUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, private callbacks: ExpectationManagerCallbacks ) { this.workforceAPI = new WorkforceAPI(this.logger) if (this.serverOptions.type === 'websocket') { - this.logger.info(`Expectation Manager on port ${this.serverOptions.port}`) this.websocketServer = new WebsocketServer(this.serverOptions.port, (client: ClientConnection) => { // A new client has connected - this.logger.info(`New ${client.clientType} connected, id "${client.clientId}"`) + this.logger.info(`ExpectationManager: New ${client.clientType} connected, id "${client.clientId}"`) - if (client.clientType === 'workerAgent') { - const expectationManagerMethods = this.getWorkerAgentAPI(client.clientId) + switch (client.clientType) { + case 'workerAgent': { + const expectationManagerMethods = this.getWorkerAgentAPI(client.clientId) - const api = new WorkerAgentAPI(expectationManagerMethods, { - type: 'websocket', - clientConnection: client, - }) + const api = new WorkerAgentAPI(expectationManagerMethods, { + type: 'websocket', + clientConnection: client, + }) - this.workerAgents[client.clientId] = { api } - } else { - throw new Error(`Unknown clientType "${client.clientType}"`) + this.workerAgents[client.clientId] = { api } + break + } + case 'N/A': + case 'expectationManager': + case 'appContainer': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: { + assertNever(client.clientType) + throw new Error(`ExpectationManager: Unknown clientType "${client.clientType}"`) + } } }) + this.logger.info(`Expectation Manager running on port ${this.websocketServer.port}`) } else { // todo: handle direct connections } @@ -131,8 +140,16 @@ export class ExpectationManager { async init(): Promise { await this.workforceAPI.init(this.managerId, this.workForceConnectionOptions, this) - const serverAccessUrl = - this.workForceConnectionOptions.type === 'internal' ? '__internal' : this.serverAccessUrl + let serverAccessUrl: string + if (this.workForceConnectionOptions.type === 'internal') { + serverAccessUrl = '__internal' + } else { + serverAccessUrl = this.serverAccessBaseUrl || 'ws://127.0.0.1' + if (this.serverOptions.type === 'websocket' && this.serverOptions.port === 0) { + // When the configured port i 0, the next free port is picked + serverAccessUrl += `:${this.websocketServer?.port}` + } + } if (!serverAccessUrl) throw new Error(`ExpectationManager.serverAccessUrl not set!`) @@ -1093,7 +1110,7 @@ export class ExpectationManager { } } - return await workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) + return workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) } private async _updateReceivedPackageContainerExpectations() { this.receivedUpdates.packageContainersHasBeenUpdated = false diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index ccbe43cd..d8da74c5 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -83,6 +83,9 @@ export class WorkerAgent { ) } async init(): Promise { + if (this.connectionOptions.type === 'websocket') { + this.logger.info(`Worker: Connecting to Workforce at "${this.connectionOptions.url}"`) + } await this.workforceAPI.init(this.id, this.connectionOptions, this) const list = await this.workforceAPI.getExpectationManagerList() @@ -134,14 +137,14 @@ export class WorkerAgent { } private async connectToExpectationManager(id: string, url: string): Promise { - this.logger.info(`Connecting to Expectation Manager "${id}" at url "${url}"`) + this.logger.info(`Worker: Connecting to Expectation Manager "${id}" at url "${url}"`) const expectedManager = (this.expectationManagers[id] = { url: url, api: new ExpectationManagerAPI(this.logger), }) const methods: ExpectationManagerWorkerAgent.WorkerAgent = literal({ doYouSupportExpectation: async (exp: Expectation.Any): Promise => { - return await this._worker.doYouSupportExpectation(exp) + return this._worker.doYouSupportExpectation(exp) }, getCostForExpectation: async ( exp: Expectation.Any diff --git a/shared/packages/workforce/src/appContainerApi.ts b/shared/packages/workforce/src/appContainerApi.ts new file mode 100644 index 00000000..cc822ae9 --- /dev/null +++ b/shared/packages/workforce/src/appContainerApi.ts @@ -0,0 +1,27 @@ +import { WorkForceAppContainer, AdapterServer, AdapterServerOptions, AppContainer } from '@shared/api' + +/** + * Exposes the API-methods of a AppContainer, to be called from the Workforce + * Note: The AppContainer connects to the Workforce, therefore the Workforce is the AdapterServer here. + * The corresponding other side is implemented at shared/packages/worker/src/workforceApi.ts + */ +export class AppContainerAPI + extends AdapterServer + implements WorkForceAppContainer.AppContainer { + constructor( + methods: WorkForceAppContainer.WorkForce, + options: AdapterServerOptions + ) { + super(methods, options) + } + + async spinUp(appType: AppContainer.AppType): Promise { + return this._sendMessage('spinUp', appType) + } + async spinDown(appId: string): Promise { + return this._sendMessage('spinDown', appId) + } + async getRunningApps(): Promise<{ appId: string; appType: AppContainer.AppType }[]> { + return this._sendMessage('getRunningApps') + } +} diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts new file mode 100644 index 00000000..d5d91c14 --- /dev/null +++ b/shared/packages/workforce/src/workerHandler.ts @@ -0,0 +1,114 @@ +import { AppContainer } from '@shared/api' +import { Workforce } from './workforce' + +const UPDATE_INTERVAL = 10 * 1000 + +/** Is in charge of spinning up/down Workers */ +export class WorkerHandler { + private updateTimeout: NodeJS.Timer | null = null + private updateAgain = false + private updateInterval: NodeJS.Timeout + private terminated = false + + private workers: PlannedWorker[] = [] + + constructor(private workForce: Workforce) { + this.updateInterval = setInterval(() => { + this.triggerUpdate() + }, UPDATE_INTERVAL) + } + public terminate(): void { + clearInterval(this.updateInterval) + this.terminated = true + } + public triggerUpdate(): void { + if (this.terminated) return + + if (!this.updateTimeout) { + this.updateAgain = false + this.updateTimeout = setTimeout(() => { + this.update() + .catch((error) => { + this.workForce.logger.error(error) + }) + .finally(() => { + this.updateTimeout = null + if (this.updateAgain) this.triggerUpdate() + }) + }, 500) + } else { + this.updateAgain = true + } + } + private async update(): Promise { + // This is a temporary stupid implementation, + // to be reworked later.. + const needs: AppTarget[] = [ + { + appType: 'worker', + }, + { + appType: 'worker', + }, + { + appType: 'worker', + }, + ] + const haves: PlannedWorker[] = this.workers.map((worker) => { + return Object.assign({}, worker) + }) + + // Initial check to see which needs are already fulfilled: + for (const need of needs) { + // Do we have anything that fullfills the need? + for (const have of haves) { + if (have.appType === need.appType) { + // ^ Later, we'll add more checks here ^ + need.fulfilled = true + } + } + } + for (const need of needs) { + if (need.fulfilled) continue + + // See which AppContainers can fullfill our need: + let found = false + for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + if (found) break + if (!appContainer.initialized) continue + + for (const availableApp of appContainer.availableApps) { + if (availableApp.appType === need.appType) { + // Spin up that worker: + + this.workForce.logger.info( + `Workforce: Spinning up another worker (${availableApp.appType}) on "${appContainerId}"` + ) + + const newPlannedWorker: PlannedWorker = { + appContainerId: appContainerId, + appType: availableApp.appType, + } + this.workers.push(newPlannedWorker) + + const appId = await appContainer.api.spinUp(availableApp.appType) + + newPlannedWorker.appId = appId + + found = true + break + } + } + } + } + } +} +interface PlannedWorker { + appType: AppContainer.AppType + appContainerId: string + appId?: string +} +interface AppTarget { + appType: AppContainer.AppType + fulfilled?: boolean +} diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index a26f6ad1..6a9b91cc 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -6,16 +6,21 @@ import { Hook, LoggerInstance, WorkforceConfig, + assertNever, + WorkForceAppContainer, + AppContainer, } from '@shared/api' +import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' import { WorkerAgentAPI } from './workerAgentApi' +import { WorkerHandler } from './workerHandler' /** * The Workforce class tracks the status of which ExpectationManagers and WorkerAgents are online, * and mediates connections between the two. */ export class Workforce { - private workerAgents: { + public workerAgents: { [workerId: string]: { api: WorkerAgentAPI } @@ -27,38 +32,78 @@ export class Workforce { url?: string } } = {} + public appContainers: { + [id: string]: { + api: AppContainerAPI + initialized: boolean + runningApps: { + appId: string + appType: AppContainer.AppType + }[] + availableApps: { + appType: AppContainer.AppType + }[] + } + } = {} private websocketServer?: WebsocketServer - constructor(private logger: LoggerInstance, config: WorkforceConfig) { - if (config.workforce.port) { + private availableApps: WorkerHandler + + constructor(public logger: LoggerInstance, config: WorkforceConfig) { + if (config.workforce.port !== null) { this.websocketServer = new WebsocketServer(config.workforce.port, (client: ClientConnection) => { // A new client has connected - this.logger.info(`New ${client.clientType} connected, id "${client.clientId}"`) - - if (client.clientType === 'workerAgent') { - const workForceMethods = this.getWorkerAgentAPI() - const api = new WorkerAgentAPI(workForceMethods, { - type: 'websocket', - clientConnection: client, - }) - this.workerAgents[client.clientId] = { api } - } else if (client.clientType === 'expectationManager') { - const workForceMethods = this.getExpectationManagerAPI() - const api = new ExpectationManagerAPI(workForceMethods, { - type: 'websocket', - clientConnection: client, - }) - this.expectationManagers[client.clientId] = { api } - } else { - throw new Error(`Unknown clientType "${client.clientType}"`) + this.logger.info(`Workforce: New client "${client.clientType}" connected, id "${client.clientId}"`) + + switch (client.clientType) { + case 'workerAgent': { + const workForceMethods = this.getWorkerAgentAPI() + const api = new WorkerAgentAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.workerAgents[client.clientId] = { api } + break + } + case 'expectationManager': { + const workForceMethods = this.getExpectationManagerAPI() + const api = new ExpectationManagerAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.expectationManagers[client.clientId] = { api } + break + } + case 'appContainer': { + const workForceMethods = this.getAppContainerAPI(client.clientId) + const api = new AppContainerAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.appContainers[client.clientId] = { + api, + availableApps: [], + runningApps: [], + initialized: false, + } + break + } + + case 'N/A': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: + assertNever(client.clientType) + throw new Error(`Workforce: Unknown clientType "${client.clientType}"`) } }) } + this.availableApps = new WorkerHandler(this) } async init(): Promise { // Nothing to do here at the moment + this.availableApps.triggerUpdate() } terminate(): void { this.websocketServer?.terminate() @@ -94,6 +139,9 @@ export class Workforce { return workForceMethods } } + getPort(): number | undefined { + return this.websocketServer?.port + } /** Return the API-methods that the Workforce exposes to the WorkerAgent */ private getWorkerAgentAPI(): WorkForceWorkerAgent.WorkForce { @@ -121,12 +169,22 @@ export class Workforce { }, } } + /** Return the API-methods that the Workforce exposes to the AppContainer */ + private getAppContainerAPI(clientId: string): WorkForceAppContainer.WorkForce { + return { + registerAvailableApps: async (availableApps: { appType: AppContainer.AppType }[]): Promise => { + await this.registerAvailableApps(clientId, availableApps) + }, + } + } public async registerExpectationManager(managerId: string, url: string): Promise { const em = this.expectationManagers[managerId] if (!em || em.url !== url) { // Added/Changed + this.logger.info(`Workforce: Register ExpectationManager (${managerId}) at url "${url}"`) + // Announce the new expectation manager to the workerAgents: for (const workerAgent of Object.values(this.workerAgents)) { await workerAgent.api.expectationManagerAvailable(managerId, url) @@ -144,4 +202,23 @@ export class Workforce { } } } + public async registerAvailableApps( + clientId: string, + availableApps: { appType: AppContainer.AppType }[] + ): Promise { + this.appContainers[clientId].availableApps = availableApps + + // Ask the AppContainer for a list of its running apps: + this.appContainers[clientId].api + .getRunningApps() + .then((runningApps) => { + this.appContainers[clientId].runningApps = runningApps + this.appContainers[clientId].initialized = true + this.availableApps.triggerUpdate() + }) + .catch((error) => { + this.logger.error('Workforce: Error in getRunningApps') + this.logger.error(error) + }) + } } diff --git a/yarn.lock b/yarn.lock index ee398bbf..29f460b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1478,10 +1478,10 @@ tslib "^2.0.3" underscore "1.12.0" -"@sofie-automation/blueprints-integration@1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0": - version "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0.tgz#ef204f6fa1ead29e8e566c3232804357e5250c89" - integrity sha512-zxwHV+eHaRaDPvSDiXDCoL8y/1NaqvDlmYQsAi7ZGi9ayGYTbluNbOaGYxD/6Q/Elqze4oC+9wSgmoep6l6Wmg== +"@sofie-automation/blueprints-integration@1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0": + version "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0.tgz#5b4a71e36514107a8c69ffee7e9425aab360b460" + integrity sha512-QbPOaYbkoca3UCq3UYOcYrUjjKL/sDpTH3CcLMKCCBxVgsfzLAdsRkzbfOmrgbsx1LS4s4Yv9c+YK68AKzcrvw== dependencies: timeline-state-resolver-types "6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0" tslib "^2.1.0" @@ -1505,10 +1505,10 @@ read-pkg-up "^7.0.1" shelljs "^0.8.4" -"@sofie-automation/server-core-integration@1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0": - version "1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.36.0-nightly-feat-package-management-20210623-061004-51788bc.0.tgz#1e575cd953dda3a30dc8a66e38c50e327cf3701f" - integrity sha512-ZnPwYBeXcYBR8971XKZxYDfbxPgvq+/BKZGvVmJxot1JEmPiOtybZncSwbn5XkhOig4mJA3VOc+efYIBpEpqVw== +"@sofie-automation/server-core-integration@1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0": + version "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0.tgz#51f5cdfe38d50fc58862800eef5f9a370f7171cb" + integrity sha512-Xd3gnz/LxZ3ZlekEPovKnGM4yQL1/+xUBIB67cfMdHBcej9tjMTNbeLxgOA7ZnemjVVEeiikwmMJvwNUiLc+JA== dependencies: data-store "^4.0.3" ejson "^2.2.0" From 16bc9ebf580aa9a1801e09e2f75e5f0c799c4933 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 23 Aug 2021 15:17:54 +0200 Subject: [PATCH 39/67] chore: refactor: return promises --- .../packages/generic/src/packageManager.ts | 3 --- shared/packages/api/src/lib.ts | 3 +++ .../expectationManager/src/workerAgentApi.ts | 22 +++++++++---------- .../expectationManager/src/workforceApi.ts | 2 +- shared/packages/worker/src/workerAgent.ts | 2 +- shared/packages/worker/src/workforceApi.ts | 2 +- .../packages/workforce/src/workerAgentApi.ts | 4 ++-- .../src/__tests__/lib/setupEnv.ts | 8 +++---- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 970bdb50..1fc59324 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -159,8 +159,6 @@ export class PackageManagerHandler { this._observers.push(expectedPackagesObserver) } public triggerUpdatedExpectedPackages(): void { - this.logger.info('_triggerUpdatedExpectedPackages') - if (this._triggerUpdatedExpectedPackagesTimeout) { clearTimeout(this._triggerUpdatedExpectedPackagesTimeout) this._triggerUpdatedExpectedPackagesTimeout = null @@ -168,7 +166,6 @@ export class PackageManagerHandler { this._triggerUpdatedExpectedPackagesTimeout = setTimeout(() => { this._triggerUpdatedExpectedPackagesTimeout = null - this.logger.info('_triggerUpdatedExpectedPackages inner') const expectedPackages: ExpectedPackageWrap[] = [] const packageContainers: PackageContainers = {} diff --git a/shared/packages/api/src/lib.ts b/shared/packages/api/src/lib.ts index fdbcb953..201903e0 100644 --- a/shared/packages/api/src/lib.ts +++ b/shared/packages/api/src/lib.ts @@ -40,3 +40,6 @@ export function hash(str: string): string { const hash0 = crypto.createHash('sha1') return hash0.update(str).digest('hex') } +export function assertNever(_never: never): void { + // Do nothing. This is a type guard +} diff --git a/shared/packages/expectationManager/src/workerAgentApi.ts b/shared/packages/expectationManager/src/workerAgentApi.ts index 36faf87f..3e2266c6 100644 --- a/shared/packages/expectationManager/src/workerAgentApi.ts +++ b/shared/packages/expectationManager/src/workerAgentApi.ts @@ -31,40 +31,40 @@ export class WorkerAgentAPI async doYouSupportExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('doYouSupportExpectation', exp) + return this._sendMessage('doYouSupportExpectation', exp) } async getCostForExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('getCostForExpectation', exp) + return this._sendMessage('getCostForExpectation', exp) } async isExpectationReadyToStartWorkingOn( exp: Expectation.Any ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('isExpectationReadyToStartWorkingOn', exp) + return this._sendMessage('isExpectationReadyToStartWorkingOn', exp) } async isExpectationFullfilled( exp: Expectation.Any, wasFullfilled: boolean ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('isExpectationFullfilled', exp, wasFullfilled) + return this._sendMessage('isExpectationFullfilled', exp, wasFullfilled) } async workOnExpectation( exp: Expectation.Any, cost: ExpectationManagerWorkerAgent.ExpectationCost ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('workOnExpectation', exp, cost) + return this._sendMessage('workOnExpectation', exp, cost) } async removeExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('removeExpectation', exp) + return this._sendMessage('removeExpectation', exp) } async cancelWorkInProgress(wipId: number): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('cancelWorkInProgress', wipId) + return this._sendMessage('cancelWorkInProgress', wipId) } // PackageContainer-related methods: ---------------------------------------------------------------------------------------- @@ -72,24 +72,24 @@ export class WorkerAgentAPI packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('doYouSupportPackageContainer', packageContainer) + return this._sendMessage('doYouSupportPackageContainer', packageContainer) } async runPackageContainerCronJob( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('runPackageContainerCronJob', packageContainer) + return this._sendMessage('runPackageContainerCronJob', packageContainer) } async setupPackageContainerMonitors( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('setupPackageContainerMonitors', packageContainer) + return this._sendMessage('setupPackageContainerMonitors', packageContainer) } async disposePackageContainerMonitors( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('disposePackageContainerMonitors', packageContainer) + return this._sendMessage('disposePackageContainerMonitors', packageContainer) } } diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index 1195e170..859bd141 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -13,6 +13,6 @@ export class WorkforceAPI } async registerExpectationManager(managerId: string, url: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts - return await this._sendMessage('registerExpectationManager', managerId, url) + return this._sendMessage('registerExpectationManager', managerId, url) } } diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index d8da74c5..3d53fe59 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -73,7 +73,7 @@ export class WorkerAgent { const manager = this.expectationManagers[managerId] if (!manager) throw new Error(`ExpectationManager "${managerId}" not found`) - return await manager.api.messageFromWorker(message) + return manager.api.messageFromWorker(message) }, { // todo: tmp: diff --git a/shared/packages/worker/src/workforceApi.ts b/shared/packages/worker/src/workforceApi.ts index 0591178d..4e9d83de 100644 --- a/shared/packages/worker/src/workforceApi.ts +++ b/shared/packages/worker/src/workforceApi.ts @@ -12,6 +12,6 @@ export class WorkforceAPI super(logger, 'workerAgent') } async getExpectationManagerList(): Promise<{ id: string; url: string }[]> { - return await this._sendMessage('getExpectationManagerList', undefined) + return this._sendMessage('getExpectationManagerList', undefined) } } diff --git a/shared/packages/workforce/src/workerAgentApi.ts b/shared/packages/workforce/src/workerAgentApi.ts index 7cb984d4..1463cdbf 100644 --- a/shared/packages/workforce/src/workerAgentApi.ts +++ b/shared/packages/workforce/src/workerAgentApi.ts @@ -16,9 +16,9 @@ export class WorkerAgentAPI } async expectationManagerAvailable(id: string, url: string): Promise { - return await this._sendMessage('expectationManagerAvailable', id, url) + return this._sendMessage('expectationManagerAvailable', id, url) } async expectationManagerGone(id: string): Promise { - return await this._sendMessage('expectationManagerGone', id) + return this._sendMessage('expectationManagerGone', id) } } diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index abc9a17a..17817820 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -156,13 +156,13 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise { switch (message.type) { case 'fetchPackageInfoMetadata': - return await coreApi.fetchPackageInfoMetadata(...message.arguments) + return coreApi.fetchPackageInfoMetadata(...message.arguments) case 'updatePackageInfo': - return await coreApi.updatePackageInfo(...message.arguments) + return coreApi.updatePackageInfo(...message.arguments) case 'removePackageInfo': - return await coreApi.removePackageInfo(...message.arguments) + return coreApi.removePackageInfo(...message.arguments) case 'reportFromMonitorPackages': - return await coreApi.reportFromMonitorPackages(...message.arguments) + return coreApi.reportFromMonitorPackages(...message.arguments) default: // @ts-expect-error message.type is never throw new Error(`Unsupported message type "${message.type}"`) From 7be4fd3e1326d1a18d460fff8c2f439fc9eafb95 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 23 Aug 2021 15:18:50 +0200 Subject: [PATCH 40/67] fix: bug: calling methods failed, due to function scope missing --- shared/packages/api/src/adapterClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/packages/api/src/adapterClient.ts b/shared/packages/api/src/adapterClient.ts index fcf977ae..3fa6fe3b 100644 --- a/shared/packages/api/src/adapterClient.ts +++ b/shared/packages/api/src/adapterClient.ts @@ -30,7 +30,7 @@ export abstract class AdapterClient { // On message from other party: const fcn = (clientMethods as any)[message.type] if (fcn) { - return fcn(...message.args) + return fcn.call(clientMethods, ...message.args) } else { throw new Error(`Unknown method "${message.type}"`) } From 994f142060f9932cf861ae6619f774eb102944ba Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 24 Aug 2021 10:02:19 +0200 Subject: [PATCH 41/67] chore: bump required node version, due to blueprints-integration requiring it --- apps/_boilerplate/app/package.json | 4 ++-- apps/_boilerplate/packages/generic/package.json | 2 +- apps/appcontainer-node/app/package.json | 4 ++-- apps/appcontainer-node/packages/generic/package.json | 2 +- apps/http-server/app/package.json | 2 +- apps/http-server/packages/generic/package.json | 2 +- apps/package-manager/app/package.json | 2 +- apps/package-manager/packages/generic/package.json | 2 +- apps/quantel-http-transformer-proxy/app/package.json | 2 +- .../packages/generic/package.json | 2 +- apps/single-app/app/package.json | 4 ++-- apps/worker/app/package.json | 2 +- apps/worker/packages/generic/package.json | 2 +- apps/workforce/app/package.json | 2 +- apps/workforce/packages/generic/package.json | 2 +- commonPackage.json | 2 +- shared/packages/api/package.json | 2 +- shared/packages/expectationManager/package.json | 2 +- shared/packages/worker/package.json | 2 +- shared/packages/workforce/package.json | 2 +- tests/internal-tests/package.json | 2 +- 21 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/_boilerplate/app/package.json b/apps/_boilerplate/app/package.json index 7ad0ae2c..0093206e 100644 --- a/apps/_boilerplate/app/package.json +++ b/apps/_boilerplate/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -33,4 +33,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/_boilerplate/packages/generic/package.json b/apps/_boilerplate/packages/generic/package.json index 8ad6904c..6bc658c2 100644 --- a/apps/_boilerplate/packages/generic/package.json +++ b/apps/_boilerplate/packages/generic/package.json @@ -19,7 +19,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/appcontainer-node/app/package.json b/apps/appcontainer-node/app/package.json index e6f9795c..774734f4 100644 --- a/apps/appcontainer-node/app/package.json +++ b/apps/appcontainer-node/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -33,4 +33,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/appcontainer-node/packages/generic/package.json b/apps/appcontainer-node/packages/generic/package.json index 9a35f2dd..2e3986b7 100644 --- a/apps/appcontainer-node/packages/generic/package.json +++ b/apps/appcontainer-node/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/http-server/app/package.json b/apps/http-server/app/package.json index b46af52e..bde53183 100644 --- a/apps/http-server/app/package.json +++ b/apps/http-server/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index 6305ab80..7ba4f79e 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -48,7 +48,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/package-manager/app/package.json b/apps/package-manager/app/package.json index 8de9eae4..96489ff8 100644 --- a/apps/package-manager/app/package.json +++ b/apps/package-manager/app/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/package-manager/packages/generic/package.json b/apps/package-manager/packages/generic/package.json index 7aba13af..d669586b 100644 --- a/apps/package-manager/packages/generic/package.json +++ b/apps/package-manager/packages/generic/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/quantel-http-transformer-proxy/app/package.json b/apps/quantel-http-transformer-proxy/app/package.json index 96b02e23..daba3435 100644 --- a/apps/quantel-http-transformer-proxy/app/package.json +++ b/apps/quantel-http-transformer-proxy/app/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index fb811abd..3628c244 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -53,7 +53,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index 3e3ba2c9..7cd802ae 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/worker/app/package.json b/apps/worker/app/package.json index feee832b..ac035cf2 100644 --- a/apps/worker/app/package.json +++ b/apps/worker/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/worker/packages/generic/package.json b/apps/worker/packages/generic/package.json index 0f4a6a9d..8e546cd1 100644 --- a/apps/worker/packages/generic/package.json +++ b/apps/worker/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/workforce/app/package.json b/apps/workforce/app/package.json index ad4a76c8..5a219a7a 100644 --- a/apps/workforce/app/package.json +++ b/apps/workforce/app/package.json @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/workforce/packages/generic/package.json b/apps/workforce/packages/generic/package.json index 52a37fbb..47f21ab9 100644 --- a/apps/workforce/packages/generic/package.json +++ b/apps/workforce/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/commonPackage.json b/commonPackage.json index ba9450d5..cab2b52b 100644 --- a/commonPackage.json +++ b/commonPackage.json @@ -4,7 +4,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "devDependencies": { "lint-staged": "^7.2.0" diff --git a/shared/packages/api/package.json b/shared/packages/api/package.json index 504e50b2..2f4776a1 100644 --- a/shared/packages/api/package.json +++ b/shared/packages/api/package.json @@ -29,7 +29,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/shared/packages/expectationManager/package.json b/shared/packages/expectationManager/package.json index 65f148d3..bfbe93b7 100644 --- a/shared/packages/expectationManager/package.json +++ b/shared/packages/expectationManager/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "devDependencies": { "lint-staged": "^7.2.0" diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json index f56ad333..f83e2a97 100644 --- a/shared/packages/worker/package.json +++ b/shared/packages/worker/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "devDependencies": { "@types/deep-diff": "^1.0.0", diff --git a/shared/packages/workforce/package.json b/shared/packages/workforce/package.json index ff100a13..f923d45a 100644 --- a/shared/packages/workforce/package.json +++ b/shared/packages/workforce/package.json @@ -24,7 +24,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/tests/internal-tests/package.json b/tests/internal-tests/package.json index 670af7a7..99a24e21 100644 --- a/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.11.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ From 62aa9aa96927cd1841cd834db44d3588a5bdbc66 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 24 Aug 2021 11:33:45 +0200 Subject: [PATCH 42/67] feat: add methods for getting status and killing apps (for testing/debugging) --- .../packages/generic/src/coreHandler.ts | 6 ++ .../packages/generic/src/packageManager.ts | 8 ++- shared/packages/api/src/index.ts | 1 + shared/packages/api/src/methods.ts | 6 ++ shared/packages/api/src/status.ts | 37 +++++++++++ .../src/expectationManager.ts | 66 +++++++++++++++++++ .../expectationManager/src/workforceApi.ts | 10 ++- shared/packages/worker/src/workerAgent.ts | 7 ++ .../packages/workforce/src/workerAgentApi.ts | 3 + shared/packages/workforce/src/workforce.ts | 41 +++++++++++- 10 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 shared/packages/api/src/status.ts diff --git a/apps/package-manager/packages/generic/src/coreHandler.ts b/apps/package-manager/packages/generic/src/coreHandler.ts index 99ee61da..2c985d89 100644 --- a/apps/package-manager/packages/generic/src/coreHandler.ts +++ b/apps/package-manager/packages/generic/src/coreHandler.ts @@ -414,4 +414,10 @@ export class CoreHandler { troubleshoot(): any { return this._packageManagerHandler?.getDataSnapshot() } + async getExpetationManagerStatus(): Promise { + return this._packageManagerHandler?.getExpetationManagerStatus() + } + async debugKillApp(appId: string): Promise { + return this._packageManagerHandler?.debugKillApp(appId) + } } diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 1fc59324..e9cae200 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -297,9 +297,15 @@ export class PackageManagerHandler { // This method can be called from core this.expectationManager.abortExpectation(workId) } - public getDataSnapshot() { + public getDataSnapshot(): any { return this.dataSnapshot } + public async getExpetationManagerStatus(): Promise { + return this.expectationManager.getStatus() + } + public async debugKillApp(appId: string): Promise { + return this.expectationManager.debugKillApp(appId) + } /** Ensures that the packageContainerExpectations containes the mandatory expectations */ private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { diff --git a/shared/packages/api/src/index.ts b/shared/packages/api/src/index.ts index 7742e372..77bda134 100644 --- a/shared/packages/api/src/index.ts +++ b/shared/packages/api/src/index.ts @@ -7,6 +7,7 @@ export * from './lib' export * from './logger' export * from './methods' export * from './packageContainerApi' +export * from './status' export * from './websocketClient' export { MessageBase, MessageReply, MessageIdentifyClient, Hook } from './websocketConnection' export * from './websocketServer' diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 252cfef5..07a33558 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -13,6 +13,7 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, } from './worker' +import { WorkforceStatus } from './status' /** Contains textual descriptions for statuses. */ export type Reason = ExpectedPackageStatusAPI.Reason @@ -25,6 +26,9 @@ export namespace WorkForceExpectationManager { /** Methods on WorkForce, called by ExpectationManager */ export interface WorkForce { registerExpectationManager: (managerId: string, url: string) => Promise + + getStatus: () => Promise + _debugKillApp(appId: string): Promise } /** Methods on ExpectationManager, called by WorkForce */ // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -37,6 +41,8 @@ export namespace WorkForceWorkerAgent { export interface WorkerAgent { expectationManagerAvailable: (id: string, url: string) => Promise expectationManagerGone: (id: string) => Promise + + _debugKill: () => Promise } /** Methods on WorkForce, called by WorkerAgent */ export interface WorkForce { diff --git a/shared/packages/api/src/status.ts b/shared/packages/api/src/status.ts new file mode 100644 index 00000000..23f13ca1 --- /dev/null +++ b/shared/packages/api/src/status.ts @@ -0,0 +1,37 @@ +export interface WorkforceStatus { + workerAgents: { + id: string + }[] + expectationManagers: { + id: string + url?: string + }[] + appContainers: { + id: string + initialized: boolean + + availableApps: { + appType: string + }[] + }[] +} +export interface ExpectationManagerStatus { + expectationStatistics: { + countTotal: number + + countNew: number + countWaiting: number + countReady: number + countWorking: number + countFulfilled: number + countRemoved: number + countRestarted: number + countAborted: number + + countNoAvailableWorkers: number + countError: number + } + workerAgents: { + workerId: string + }[] +} diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index df18f4b6..b8b50f8f 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -11,6 +11,7 @@ import { LoggerInstance, PackageContainerExpectation, Reason, + ExpectationManagerStatus, } from '@shared/api' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { WorkforceAPI } from './workforceApi' @@ -92,6 +93,8 @@ export class ExpectationManager { } = {} private terminating = false + private status: ExpectationManagerStatus + constructor( private logger: LoggerInstance, public readonly managerId: string, @@ -102,6 +105,7 @@ export class ExpectationManager { private callbacks: ExpectationManagerCallbacks ) { this.workforceAPI = new WorkforceAPI(this.logger) + this.status = this.updateStatus() if (this.serverOptions.type === 'websocket') { this.websocketServer = new WebsocketServer(this.serverOptions.port, (client: ClientConnection) => { // A new client has connected @@ -225,6 +229,15 @@ export class ExpectationManager { this.receivedUpdates.expectationsHasBeenUpdated = true this._triggerEvaluateExpectations(true) } + async getStatus(): Promise { + return { + workforce: await this.workforceAPI.getStatus(), + expectationManager: this.status, + } + } + async debugKillApp(appId: string): Promise { + return this.workforceAPI._debugKillApp(appId) + } /** * Schedule the evaluateExpectations() to run * @param asap If true, will re-schedule evaluateExpectations() to run as soon as possible @@ -382,6 +395,8 @@ export class ExpectationManager { // Iterate through all Expectations: const runAgainASAP = await this._evaluateAllExpectations() + this.updateStatus() + if (runAgainASAP) { this._triggerEvaluateExpectations(true) } @@ -1304,6 +1319,57 @@ export class ExpectationManager { } return session.noAssignedWorkerReason } + private updateStatus(): ExpectationManagerStatus { + this.status = { + expectationStatistics: { + countTotal: 0, + + countNew: 0, + countWaiting: 0, + countReady: 0, + countWorking: 0, + countFulfilled: 0, + countRemoved: 0, + countRestarted: 0, + countAborted: 0, + + countNoAvailableWorkers: 0, + countError: 0, + }, + workerAgents: Object.entries(this.workerAgents).map(([id, _workerAgent]) => { + return { + workerId: id, + } + }), + } + const expectationStatistics = this.status.expectationStatistics + for (const exp of Object.values(this.trackedExpectations)) { + expectationStatistics.countTotal++ + + if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW) { + expectationStatistics.countNew++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { + expectationStatistics.countWaiting++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) { + expectationStatistics.countReady++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { + expectationStatistics.countWorking++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { + expectationStatistics.countFulfilled++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.REMOVED) { + expectationStatistics.countRemoved++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.RESTARTED) { + expectationStatistics.countRestarted++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.ABORTED) { + expectationStatistics.countAborted++ + } else assertNever(exp.state) + + if (!exp.availableWorkers.length) expectationStatistics.countNoAvailableWorkers + if (exp.errorCount > 0 && exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) + expectationStatistics.countError++ + } + return this.status + } } export type ExpectationManagerServerOptions = | { diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index 859bd141..0ceaf73e 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, LoggerInstance, WorkForceExpectationManager } from '@shared/api' +import { AdapterClient, LoggerInstance, WorkForceExpectationManager, WorkforceStatus } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the ExpectationManager @@ -15,4 +15,12 @@ export class WorkforceAPI // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('registerExpectationManager', managerId, url) } + async getStatus(): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('getStatus') + } + async _debugKillApp(appId: string): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('_debugKillApp', appId) + } } diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 3d53fe59..0d442348 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -135,6 +135,13 @@ export class WorkerAgent { async expectationManagerGone(id: string): Promise { delete this.expectationManagers[id] } + async _debugKill(): Promise { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 500) + } private async connectToExpectationManager(id: string, url: string): Promise { this.logger.info(`Worker: Connecting to Expectation Manager "${id}" at url "${url}"`) diff --git a/shared/packages/workforce/src/workerAgentApi.ts b/shared/packages/workforce/src/workerAgentApi.ts index 1463cdbf..05714356 100644 --- a/shared/packages/workforce/src/workerAgentApi.ts +++ b/shared/packages/workforce/src/workerAgentApi.ts @@ -21,4 +21,7 @@ export class WorkerAgentAPI async expectationManagerGone(id: string): Promise { return this._sendMessage('expectationManagerGone', id) } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } } diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index 6a9b91cc..f8fca3eb 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -167,12 +167,19 @@ export class Workforce { registerExpectationManager: async (managerId: string, url: string): Promise => { await this.registerExpectationManager(managerId, url) }, + + getStatus: async (): Promise => { + return this.getStatus() + }, + _debugKillApp: async (appId: string): Promise => { + return this._debugKillApp(appId) + }, } } /** Return the API-methods that the Workforce exposes to the AppContainer */ private getAppContainerAPI(clientId: string): WorkForceAppContainer.WorkForce { return { - registerAvailableApps: async (availableApps: { appType: AppContainer.AppType }[]): Promise => { + registerAvailableApps: async (availableApps: { appType: string }[]): Promise => { await this.registerAvailableApps(clientId, availableApps) }, } @@ -192,6 +199,38 @@ export class Workforce { } this.expectationManagers[managerId].url = url } + public async getStatus(): Promise { + return { + workerAgents: Object.entries(this.workerAgents).map(([workerId, _workerAgent]) => { + return { + id: workerId, + } + }), + expectationManagers: Object.entries(this.expectationManagers).map(([id, expMan]) => { + return { + id: id, + url: expMan.url, + } + }), + appContainers: Object.entries(this.appContainers).map(([id, appContainer]) => { + return { + id: id, + initialized: appContainer.initialized, + availableApps: appContainer.availableApps.map((availableApp) => { + return { + appType: availableApp.appType, + } + }), + } + }), + } + } + public async _debugKillApp(appId: string): Promise { + const workerAgent = this.workerAgents[appId] + if (!workerAgent) throw new Error(`Worker "${appId}" not found`) + + return workerAgent.api._debugKill() + } public async removeExpectationManager(managerId: string): Promise { const em = this.expectationManagers[managerId] if (em) { From 4600da0a2d84f88cbfc99ef6a6c86d2116adf977 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 24 Aug 2021 11:34:41 +0200 Subject: [PATCH 43/67] fix: change appType to be a generic string, and moved assertNever to a better place --- .../packages/generic/src/appContainer.ts | 15 +++++---------- .../packages/generic/src/workforceApi.ts | 4 ++-- shared/packages/api/src/lib.ts | 5 +++-- shared/packages/api/src/methods.ts | 5 ++--- shared/packages/api/src/websocketServer.ts | 2 +- .../expectationManager/src/expectationManager.ts | 4 +--- .../src/worker/accessorHandlers/accessor.ts | 4 +--- .../src/worker/accessorHandlers/fileShare.ts | 3 +-- .../worker/src/worker/accessorHandlers/http.ts | 3 +-- .../src/worker/accessorHandlers/httpProxy.ts | 3 +-- .../worker/accessorHandlers/lib/FileHandler.ts | 3 +-- .../src/worker/accessorHandlers/localFolder.ts | 3 +-- shared/packages/worker/src/worker/lib/lib.ts | 5 ----- .../expectationHandlers/lib/ffmpeg.ts | 2 +- .../windowsWorker/expectationHandlers/lib/scan.ts | 3 +-- .../expectationHandlers/mediaFileThumbnail.ts | 2 +- .../worker/workers/windowsWorker/windowsWorker.ts | 2 +- shared/packages/workforce/src/appContainerApi.ts | 6 +++--- shared/packages/workforce/src/workerHandler.ts | 7 +++---- shared/packages/workforce/src/workforce.ts | 11 ++++------- 20 files changed, 34 insertions(+), 58 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index de76c53e..5cd154ab 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -1,12 +1,7 @@ import * as ChildProcess from 'child_process' import * as path from 'path' import * as fs from 'fs' -import { - LoggerInstance, - AppContainerProcessConfig, - ClientConnectionOptions, - AppContainer as NSAppContainer, -} from '@shared/api' +import { LoggerInstance, AppContainerProcessConfig, ClientConnectionOptions } from '@shared/api' import { WorkforceAPI } from './workforceApi' /** Mimimum time between app restarts */ @@ -21,7 +16,7 @@ export class AppContainer { private apps: { [appId: string]: { process: ChildProcess.ChildProcess - appType: NSAppContainer.AppType + appType: string toBeKilled: boolean restarts: number lastRestart: number @@ -55,7 +50,7 @@ export class AppContainer { await this.workforceAPI.registerAvailableApps( Object.entries(this.availableApps).map((o) => { - const appType = o[0] as NSAppContainer.AppType + const appType = o[0] as string return { appType: appType, } @@ -100,7 +95,7 @@ export class AppContainer { ;(await fs.promises.readdir(dirPath)).forEach((fileName) => { if (fileName.match(/worker/i)) { - this.availableApps['worker'] = { + this.availableApps[fileName] = { file: path.join(dirPath, fileName), args: (appId: string) => { return [...getWorkerArgs(appId)] @@ -143,7 +138,7 @@ export class AppContainer { app.process.removeAllListeners() delete this.apps[appId] } - async getRunningApps(): Promise<{ appId: string; appType: NSAppContainer.AppType }[]> { + async getRunningApps(): Promise<{ appId: string; appType: string }[]> { return Object.entries(this.apps).map((o) => { const [appId, app] = o diff --git a/apps/appcontainer-node/packages/generic/src/workforceApi.ts b/apps/appcontainer-node/packages/generic/src/workforceApi.ts index aabca95a..6a810595 100644 --- a/apps/appcontainer-node/packages/generic/src/workforceApi.ts +++ b/apps/appcontainer-node/packages/generic/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, LoggerInstance, WorkForceAppContainer, AppContainer } from '@shared/api' +import { AdapterClient, LoggerInstance, WorkForceAppContainer } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the AppContainer @@ -11,7 +11,7 @@ export class WorkforceAPI constructor(logger: LoggerInstance) { super(logger, 'appContainer') } - async registerAvailableApps(availableApps: { appType: AppContainer.AppType }[]): Promise { + async registerAvailableApps(availableApps: { appType: string }[]): Promise { return this._sendMessage('registerAvailableApps', availableApps) } } diff --git a/shared/packages/api/src/lib.ts b/shared/packages/api/src/lib.ts index 201903e0..3cee0b5b 100644 --- a/shared/packages/api/src/lib.ts +++ b/shared/packages/api/src/lib.ts @@ -40,6 +40,7 @@ export function hash(str: string): string { const hash0 = crypto.createHash('sha1') return hash0.update(str).digest('hex') } -export function assertNever(_never: never): void { - // Do nothing. This is a type guard +/** Helper function to simply assert that the value is of the type never */ +export function assertNever(_value: never): void { + // does nothing } diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 07a33558..8f12c874 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -2,7 +2,6 @@ import { ExpectedPackage, ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { Expectation } from './expectationApi' import { PackageContainerExpectation } from './packageContainerApi' -import { AppContainer as NSAppContainer } from './appContainer' import { ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -164,10 +163,10 @@ export namespace WorkForceAppContainer { appType: 'worker' // | other ) => Promise spinDown: (appId: string) => Promise - getRunningApps: () => Promise<{ appId: string; appType: NSAppContainer.AppType }[]> + getRunningApps: () => Promise<{ appId: string; appType: string }[]> } /** Methods on WorkForce, called by AppContainer */ export interface WorkForce { - registerAvailableApps: (availableApps: { appType: NSAppContainer.AppType }[]) => Promise + registerAvailableApps: (availableApps: { appType: string }[]) => Promise } } diff --git a/shared/packages/api/src/websocketServer.ts b/shared/packages/api/src/websocketServer.ts index eea12fab..f5483924 100644 --- a/shared/packages/api/src/websocketServer.ts +++ b/shared/packages/api/src/websocketServer.ts @@ -103,4 +103,4 @@ export class ClientConnection extends WebsocketConnection { this.ws?.close() } } -type ClientType = MessageIdentifyClient['clientType'] +export type ClientType = MessageIdentifyClient['clientType'] diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index b8b50f8f..479c4c6d 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -11,6 +11,7 @@ import { LoggerInstance, PackageContainerExpectation, Reason, + assertNever, ExpectationManagerStatus, } from '@shared/api' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' @@ -1491,6 +1492,3 @@ interface TrackedPackageContainerExpectation { } } } -function assertNever(_shouldBeNever: never) { - // Nothing -} diff --git a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts index 4af2b4b1..1f1347c0 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts @@ -1,4 +1,5 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' +import { assertNever } from '@shared/api' import { GenericWorker } from '../worker' import { CorePackageInfoAccessorHandle } from './corePackageInfo' import { FileShareAccessorHandle } from './fileShare' @@ -42,9 +43,6 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { throw new Error(`Unsupported Accessor type "${accessor.type}"`) } } -function assertNever(_shouldBeNever: never) { - // Nothing -} export function isLocalFolderAccessorHandle( accessorHandler: GenericAccessorHandle diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 8d5d7083..a5f8650a 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -2,12 +2,11 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' import { GenericWorker } from '../worker' import { WindowsWorker } from '../workers/windowsWorker/windowsWorker' import networkDrive from 'windows-network-drive' import { exec } from 'child_process' -import { assertNever } from '../lib/lib' import { FileShareAccessorHandleType, GenericFileAccessorHandle } from './lib/FileHandler' const fsStat = promisify(fs.stat) diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index bdeb496b..50f61d35 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -6,12 +6,11 @@ import { PutPackageHandler, AccessorHandlerResult, } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' import FormData from 'form-data' import AbortController from 'abort-controller' -import { assertNever } from '../lib/lib' /** Accessor handle for accessing files in a local folder */ export class HTTPAccessorHandle extends GenericAccessorHandle { diff --git a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts index 540bfb05..d312ba21 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts @@ -6,12 +6,11 @@ import { PutPackageHandler, AccessorHandlerResult, } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' import FormData from 'form-data' import AbortController from 'abort-controller' -import { assertNever } from '../lib/lib' /** Accessor handle for accessing files in HTTP- */ export class HTTPProxyAccessorHandle extends GenericAccessorHandle { diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index 94b7c125..7871ec9e 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -1,12 +1,11 @@ import path from 'path' import { promisify } from 'util' import fs from 'fs' -import { Expectation, hashObj, literal, PackageContainerExpectation } from '@shared/api' +import { Expectation, hashObj, literal, PackageContainerExpectation, assertNever } from '@shared/api' import chokidar from 'chokidar' import { GenericWorker } from '../../worker' import { Accessor, AccessorOnPackage, ExpectedPackage } from '@sofie-automation/blueprints-integration' import { GenericAccessorHandle } from '../genericHandle' -import { assertNever } from '../../lib/lib' export const LocalFolderAccessorHandleType = 'localFolder' export const FileShareAccessorHandleType = 'fileShare' diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index 35371374..242bc116 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -3,9 +3,8 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' import { GenericWorker } from '../worker' -import { assertNever } from '../lib/lib' import { GenericFileAccessorHandle, LocalFolderAccessorHandleType } from './lib/FileHandler' const fsStat = promisify(fs.stat) diff --git a/shared/packages/worker/src/worker/lib/lib.ts b/shared/packages/worker/src/worker/lib/lib.ts index 27f3a116..fe81224d 100644 --- a/shared/packages/worker/src/worker/lib/lib.ts +++ b/shared/packages/worker/src/worker/lib/lib.ts @@ -38,8 +38,3 @@ export interface AccessorWithPackageContainer { + async spinUp(appType: string): Promise { return this._sendMessage('spinUp', appType) } async spinDown(appId: string): Promise { return this._sendMessage('spinDown', appId) } - async getRunningApps(): Promise<{ appId: string; appType: AppContainer.AppType }[]> { + async getRunningApps(): Promise<{ appId: string; appType: string }[]> { return this._sendMessage('getRunningApps') } } diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts index d5d91c14..44391204 100644 --- a/shared/packages/workforce/src/workerHandler.ts +++ b/shared/packages/workforce/src/workerHandler.ts @@ -1,9 +1,8 @@ -import { AppContainer } from '@shared/api' import { Workforce } from './workforce' const UPDATE_INTERVAL = 10 * 1000 -/** Is in charge of spinning up/down Workers */ +/** The WorkerHandler is in charge of spinning up/down Workers */ export class WorkerHandler { private updateTimeout: NodeJS.Timer | null = null private updateAgain = false @@ -104,11 +103,11 @@ export class WorkerHandler { } } interface PlannedWorker { - appType: AppContainer.AppType + appType: string appContainerId: string appId?: string } interface AppTarget { - appType: AppContainer.AppType + appType: string fulfilled?: boolean } diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index f8fca3eb..1ad8fb61 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -8,7 +8,7 @@ import { WorkforceConfig, assertNever, WorkForceAppContainer, - AppContainer, + WorkforceStatus, } from '@shared/api' import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' @@ -38,10 +38,10 @@ export class Workforce { initialized: boolean runningApps: { appId: string - appType: AppContainer.AppType + appType: string }[] availableApps: { - appType: AppContainer.AppType + appType: string }[] } } = {} @@ -241,10 +241,7 @@ export class Workforce { } } } - public async registerAvailableApps( - clientId: string, - availableApps: { appType: AppContainer.AppType }[] - ): Promise { + public async registerAvailableApps(clientId: string, availableApps: { appType: string }[]): Promise { this.appContainers[clientId].availableApps = availableApps // Ask the AppContainer for a list of its running apps: From 1030c27aa13ab4ed5ef49185ac0332b474eeb867 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 24 Aug 2021 11:42:23 +0200 Subject: [PATCH 44/67] fix: updates in workerHandler --- .../packages/workforce/src/workerHandler.ts | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts index 44391204..2201a2bf 100644 --- a/shared/packages/workforce/src/workerHandler.ts +++ b/shared/packages/workforce/src/workerHandler.ts @@ -9,7 +9,7 @@ export class WorkerHandler { private updateInterval: NodeJS.Timeout private terminated = false - private workers: PlannedWorker[] = [] + private plannedWorkers: PlannedWorker[] = [] constructor(private workForce: Workforce) { this.updateInterval = setInterval(() => { @@ -40,6 +40,21 @@ export class WorkerHandler { } } private async update(): Promise { + // Update this.plannedWorkers + for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + for (const runningApp of appContainer.runningApps) { + const plannedWorker = this.plannedWorkers.find((pw) => pw.appId === runningApp.appId) + if (!plannedWorker) { + this.plannedWorkers.push({ + appContainerId: appContainerId, + appType: runningApp.appType, + appId: runningApp.appId, + isInUse: false, + }) + } + } + } + // This is a temporary stupid implementation, // to be reworked later.. const needs: AppTarget[] = [ @@ -53,17 +68,21 @@ export class WorkerHandler { appType: 'worker', }, ] - const haves: PlannedWorker[] = this.workers.map((worker) => { - return Object.assign({}, worker) - }) + // Reset plannedWorkers: + for (const plannedWorker of this.plannedWorkers) { + plannedWorker.isInUse = false + } // Initial check to see which needs are already fulfilled: for (const need of needs) { // Do we have anything that fullfills the need? - for (const have of haves) { - if (have.appType === need.appType) { + for (const plannedWorker of this.plannedWorkers) { + if (plannedWorker.isInUse) continue + + if (plannedWorker.appType === need.appType) { // ^ Later, we'll add more checks here ^ need.fulfilled = true + plannedWorker.isInUse = true } } } @@ -87,8 +106,9 @@ export class WorkerHandler { const newPlannedWorker: PlannedWorker = { appContainerId: appContainerId, appType: availableApp.appType, + isInUse: true, } - this.workers.push(newPlannedWorker) + this.plannedWorkers.push(newPlannedWorker) const appId = await appContainer.api.spinUp(availableApp.appType) @@ -106,6 +126,8 @@ interface PlannedWorker { appType: string appContainerId: string appId?: string + + isInUse: boolean } interface AppTarget { appType: string From 75745c11b62f4be3ac9c899df5dff2a4f1f6e93d Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 07:10:39 +0200 Subject: [PATCH 45/67] chore: fix test --- tests/internal-tests/src/__tests__/lib/setupEnv.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index 17817820..b565d9e9 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -50,6 +50,13 @@ const defaultTestConfig: SingleAppConfig = { port: 0, transformerURL: '', }, + appContainer: { + appContainerId: 'appContainer0', + workforceURL: null, + resourceId: '', + networkIds: [], + windowsDriveLetters: ['X', 'Y', 'Z'], + }, } export async function setupExpectationManager( @@ -138,7 +145,7 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise | null ) => { if (!containerStatuses[containerId]) { containerStatuses[containerId] = { @@ -210,7 +217,7 @@ export interface ContainerStatuses { [containerId: string]: { packages: { [packageId: string]: { - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + packageStatus: Omit | null } } } From ed251f25c857162f2da83c829cc7e85076037615 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 07:37:02 +0200 Subject: [PATCH 46/67] fix: bug in workforce --- shared/packages/workforce/src/workerHandler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts index 2201a2bf..5a982d05 100644 --- a/shared/packages/workforce/src/workerHandler.ts +++ b/shared/packages/workforce/src/workerHandler.ts @@ -83,6 +83,7 @@ export class WorkerHandler { // ^ Later, we'll add more checks here ^ need.fulfilled = true plannedWorker.isInUse = true + break } } } From b486ecadec5f7775fb7a0497d4a683fab5fdadf2 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 07:37:55 +0200 Subject: [PATCH 47/67] fix: handle lost connections (server lost connection to a client) --- .../expectationManager/src/expectationManager.ts | 4 +++- shared/packages/workforce/src/workforce.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 479c4c6d..701fc69b 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -121,8 +121,10 @@ export class ExpectationManager { type: 'websocket', clientConnection: client, }) - this.workerAgents[client.clientId] = { api } + client.on('close', () => { + delete this.workerAgents[client.clientId] + }) break } case 'N/A': diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index 1ad8fb61..748e592a 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -64,6 +64,9 @@ export class Workforce { clientConnection: client, }) this.workerAgents[client.clientId] = { api } + client.on('close', () => { + delete this.workerAgents[client.clientId] + }) break } case 'expectationManager': { @@ -73,6 +76,9 @@ export class Workforce { clientConnection: client, }) this.expectationManagers[client.clientId] = { api } + client.on('close', () => { + delete this.expectationManagers[client.clientId] + }) break } case 'appContainer': { @@ -87,6 +93,9 @@ export class Workforce { runningApps: [], initialized: false, } + client.on('close', () => { + delete this.appContainers[client.clientId] + }) break } From 0ec230f6ff3bc00ce9a04f447131bc771617ea38 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 07:39:15 +0200 Subject: [PATCH 48/67] chore: change verbosify of logging --- .../packages/generic/src/packageManager.ts | 4 ++-- .../src/expectationManager.ts | 24 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index e9cae200..f921fd8b 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -144,7 +144,7 @@ export class PackageManagerHandler { }) this._observers = [] } - this.logger.info('Renewing observers') + this.logger.debug('Renewing observers') const expectedPackagesObserver = this.coreHandler.core.observe('deviceExpectedPackages') expectedPackagesObserver.added = () => { @@ -658,7 +658,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks } } - this.logger.info( + this.logger.debug( `reportMonitoredPackages: ${expectedPackages.length} packages, ${expectedPackagesWraps.length} wraps` ) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 701fc69b..3da33e1c 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -297,7 +297,7 @@ export class ExpectationManager { wip.trackedExp.status.actualVersionHash = actualVersionHash wip.trackedExp.status.workProgress = progress - this.logger.info( + this.logger.debug( `Expectation "${JSON.stringify( wip.trackedExp.exp.statusReport.label )}" progress: ${progress}` @@ -380,7 +380,7 @@ export class ExpectationManager { * Evaluates the Expectations and PackageContainerExpectations */ private async _evaluateExpectations(): Promise { - this.logger.info(Date.now() / 1000 + ' _evaluateExpectations ----------') + this.logger.debug(Date.now() / 1000 + ' _evaluateExpectations ----------') // First we're going to see if there is any new incoming data which needs to be pulled in. if (this.receivedUpdates.expectationsHasBeenUpdated) { @@ -423,7 +423,7 @@ export class ExpectationManager { if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to update`) + this.logger.debug(`Cancelling ${trackedExp.id} due to update`) await trackedExp.status.workInProgressCancel() } } @@ -471,7 +471,7 @@ export class ExpectationManager { if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to removed`) + this.logger.debug(`Cancelling ${trackedExp.id} due to removed`) await trackedExp.status.workInProgressCancel() } } @@ -494,7 +494,7 @@ export class ExpectationManager { if (trackedExp) { if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to restart`) + this.logger.debug(`Cancelling ${trackedExp.id} due to restart`) await trackedExp.status.workInProgressCancel() } } @@ -511,7 +511,7 @@ export class ExpectationManager { if (trackedExp) { if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to abort`) + this.logger.debug(`Cancelling ${trackedExp.id} due to abort`) await trackedExp.status.workInProgressCancel() } } @@ -584,7 +584,7 @@ export class ExpectationManager { const trackedWithState = tracked.filter((trackedExp) => trackedExp.state === handleState) if (trackedWithState.length) { - this.logger.info(`Handle state ${handleState}, ${trackedWithState.length} expectations..`) + this.logger.debug(`Handle state ${handleState}, ${trackedWithState.length} expectations..`) } if (trackedWithState.length) { @@ -606,7 +606,7 @@ export class ExpectationManager { } } - this.logger.info(`Handle other states..`) + this.logger.debug(`Handle other states..`) // Step 1.5: Reset the session: // Because during the next iteration, the worker-assignment need to be done in series @@ -760,7 +760,7 @@ export class ExpectationManager { if (trackedExp.session.assignedWorker) { const assignedWorker = trackedExp.session.assignedWorker - this.logger.info(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) + this.logger.debug(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) // Start working on the Expectation: const wipInfo = await assignedWorker.worker.workOnExpectation(trackedExp.exp, assignedWorker.cost) @@ -964,11 +964,11 @@ export class ExpectationManager { } // Log and report new states an reasons: if (updatedState) { - this.logger.info( + this.logger.debug( `${trackedExp.exp.statusReport.label}: New state: "${prevState}"->"${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } else if (updatedReason) { - this.logger.info( + this.logger.debug( `${trackedExp.exp.statusReport.label}: State: "${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } @@ -1295,7 +1295,7 @@ export class ExpectationManager { } if (updatedReason) { - this.logger.info( + this.logger.debug( `PackageContainerStatus "${trackedPackageContainer.packageContainer.label}": Reason: "${trackedPackageContainer.status.reason.tech}"` ) } From f7eeab3f47270e54fe879593eb4824a51b00792a Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 13:04:39 +0200 Subject: [PATCH 49/67] feat: add methods for getting statuses and kill process (for testing/debugging) --- .../packages/generic/src/appContainer.ts | 13 ++++- .../packages/generic/src/packageManager.ts | 13 ++++- shared/packages/api/src/logger.ts | 4 ++ shared/packages/api/src/methods.ts | 25 ++++++--- shared/packages/api/src/status.ts | 2 +- .../src/expectationManager.ts | 15 ++++++ .../expectationManager/src/workforceApi.ts | 15 +++++- .../worker/src/expectationManagerApi.ts | 1 + shared/packages/worker/src/workerAgent.ts | 6 ++- .../packages/workforce/src/appContainerApi.ts | 9 +++- .../workforce/src/expectationManagerApi.ts | 9 +++- .../packages/workforce/src/workerAgentApi.ts | 12 +++-- shared/packages/workforce/src/workforce.ts | 51 ++++++++++++++++--- 13 files changed, 151 insertions(+), 24 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 5cd154ab..1dcc0cbb 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -1,7 +1,7 @@ import * as ChildProcess from 'child_process' import * as path from 'path' import * as fs from 'fs' -import { LoggerInstance, AppContainerProcessConfig, ClientConnectionOptions } from '@shared/api' +import { LoggerInstance, AppContainerProcessConfig, ClientConnectionOptions, LogLevel } from '@shared/api' import { WorkforceAPI } from './workforceApi' /** Mimimum time between app restarts */ @@ -110,6 +110,16 @@ export class AppContainer { // kill child processes } + async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } + async _debugKill(): Promise { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } async spinUp(appType: AppType): Promise { const availableApp = this.availableApps[appType] @@ -153,6 +163,7 @@ export class AppContainer { appId: string, availableApp: AvailableAppInfo ): ChildProcess.ChildProcess { + this.logger.info(`Starting process "${appId}" (${appType}): "${availableApp.file}"`) const cwd = process.execPath.match(/node.exe$/) ? undefined // Process runs as a node process, we're probably in development mode. : path.dirname(process.execPath) // Process runs as a node process, we're probably in development mode. diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index f921fd8b..cf145c6d 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -301,7 +301,18 @@ export class PackageManagerHandler { return this.dataSnapshot } public async getExpetationManagerStatus(): Promise { - return this.expectationManager.getStatus() + return { + ...(await this.expectationManager.getStatus()), + packageManager: { + workforceURL: + this.workForceConnectionOptions.type === 'websocket' ? this.workForceConnectionOptions.url : null, + lastUpdated: this.dataSnapshot.updated, + countExpectedPackages: this.dataSnapshot.expectedPackages.length, + countPackageContainers: Object.keys(this.dataSnapshot.packageContainers).length, + countExpectations: Object.keys(this.dataSnapshot.expectations).length, + countPackageContainerExpectations: Object.keys(this.dataSnapshot.packageContainerExpectations).length, + }, + } } public async debugKillApp(appId: string): Promise { return this.expectationManager.debugKillApp(appId) diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts index 7ea11cef..482c87a3 100644 --- a/shared/packages/api/src/logger.ts +++ b/shared/packages/api/src/logger.ts @@ -79,3 +79,7 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance }) return logger } +export enum LogLevel { + INFO = 'info', + DEBUG = 'debug', +} diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 8f12c874..973e0a1c 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -13,6 +13,7 @@ import { ReturnTypeSetupPackageContainerMonitors, } from './worker' import { WorkforceStatus } from './status' +import { LogLevel } from './logger' /** Contains textual descriptions for statuses. */ export type Reason = ExpectedPackageStatusAPI.Reason @@ -24,24 +25,33 @@ export type Reason = ExpectedPackageStatusAPI.Reason export namespace WorkForceExpectationManager { /** Methods on WorkForce, called by ExpectationManager */ export interface WorkForce { - registerExpectationManager: (managerId: string, url: string) => Promise - - getStatus: () => Promise + setLogLevel: (logLevel: LogLevel) => Promise + setLogLevelOfApp: (appId: string, logLevel: LogLevel) => Promise _debugKillApp(appId: string): Promise + getStatus: () => Promise + + // This is a temporary function: + setWorkerCount: (count: number) => Promise + + registerExpectationManager: (managerId: string, url: string) => Promise } /** Methods on ExpectationManager, called by WorkForce */ // eslint-disable-next-line @typescript-eslint/no-empty-interface - export interface ExpectationManager {} + export interface ExpectationManager { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + } } /** Methods used by WorkForce and WorkerAgent */ export namespace WorkForceWorkerAgent { /** Methods on WorkerAgent, called by WorkForce */ export interface WorkerAgent { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + expectationManagerAvailable: (id: string, url: string) => Promise expectationManagerGone: (id: string) => Promise - - _debugKill: () => Promise } /** Methods on WorkForce, called by WorkerAgent */ export interface WorkForce { @@ -159,6 +169,9 @@ export namespace ExpectationManagerWorkerAgent { export namespace WorkForceAppContainer { /** Methods on AppContainer, called by WorkForce */ export interface AppContainer { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + spinUp: ( appType: 'worker' // | other ) => Promise diff --git a/shared/packages/api/src/status.ts b/shared/packages/api/src/status.ts index 23f13ca1..9fdcd86d 100644 --- a/shared/packages/api/src/status.ts +++ b/shared/packages/api/src/status.ts @@ -16,6 +16,7 @@ export interface WorkforceStatus { }[] } export interface ExpectationManagerStatus { + id: string expectationStatistics: { countTotal: number @@ -27,7 +28,6 @@ export interface ExpectationManagerStatus { countRemoved: number countRestarted: number countAborted: number - countNoAvailableWorkers: number countError: number } diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 3da33e1c..03cc737f 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -13,6 +13,7 @@ import { Reason, assertNever, ExpectationManagerStatus, + LogLevel, } from '@shared/api' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { WorkforceAPI } from './workforceApi' @@ -232,12 +233,25 @@ export class ExpectationManager { this.receivedUpdates.expectationsHasBeenUpdated = true this._triggerEvaluateExpectations(true) } + async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } + async _debugKill(): Promise { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } async getStatus(): Promise { return { workforce: await this.workforceAPI.getStatus(), expectationManager: this.status, } } + async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + return this.workforceAPI.setLogLevelOfApp(appId, logLevel) + } async debugKillApp(appId: string): Promise { return this.workforceAPI._debugKillApp(appId) } @@ -1324,6 +1338,7 @@ export class ExpectationManager { } private updateStatus(): ExpectationManagerStatus { this.status = { + id: this.managerId, expectationStatistics: { countTotal: 0, diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index 0ceaf73e..a066c501 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, LoggerInstance, WorkForceExpectationManager, WorkforceStatus } from '@shared/api' +import { WorkForceExpectationManager, AdapterClient, LoggerInstance, LogLevel, WorkforceStatus } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the ExpectationManager @@ -11,6 +11,11 @@ export class WorkforceAPI constructor(logger: LoggerInstance) { super(logger, 'expectationManager') } + + async setWorkerCount(count: number): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('setWorkerCount', count) + } async registerExpectationManager(managerId: string, url: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('registerExpectationManager', managerId, url) @@ -19,6 +24,14 @@ export class WorkforceAPI // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('getStatus') } + async setLogLevel(logLevel: LogLevel): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('setLogLevel', logLevel) + } + async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('setLogLevelOfApp', appId, logLevel) + } async _debugKillApp(appId: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('_debugKillApp', appId) diff --git a/shared/packages/worker/src/expectationManagerApi.ts b/shared/packages/worker/src/expectationManagerApi.ts index d9495d0c..5f2e62d2 100644 --- a/shared/packages/worker/src/expectationManagerApi.ts +++ b/shared/packages/worker/src/expectationManagerApi.ts @@ -11,6 +11,7 @@ export class ExpectationManagerAPI constructor(logger: LoggerInstance) { super(logger, 'workerAgent') } + async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts return this._sendMessage('messageFromWorker', message) diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 0d442348..44d66b41 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -16,6 +16,7 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, ReturnTypeDisposePackageContainerMonitors, + LogLevel, } from '@shared/api' import { ExpectationManagerAPI } from './expectationManagerApi' import { IWorkInProgress } from './worker/lib/workInProgress' @@ -135,12 +136,15 @@ export class WorkerAgent { async expectationManagerGone(id: string): Promise { delete this.expectationManagers[id] } + public async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } async _debugKill(): Promise { // This is for testing purposes only setTimeout(() => { // eslint-disable-next-line no-process-exit process.exit(42) - }, 500) + }, 1) } private async connectToExpectationManager(id: string, url: string): Promise { diff --git a/shared/packages/workforce/src/appContainerApi.ts b/shared/packages/workforce/src/appContainerApi.ts index 2864ae81..27c652c9 100644 --- a/shared/packages/workforce/src/appContainerApi.ts +++ b/shared/packages/workforce/src/appContainerApi.ts @@ -1,4 +1,4 @@ -import { WorkForceAppContainer, AdapterServer, AdapterServerOptions } from '@shared/api' +import { WorkForceAppContainer, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' /** * Exposes the API-methods of a AppContainer, to be called from the Workforce @@ -15,6 +15,13 @@ export class AppContainerAPI super(methods, options) } + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + async spinUp(appType: string): Promise { return this._sendMessage('spinUp', appType) } diff --git a/shared/packages/workforce/src/expectationManagerApi.ts b/shared/packages/workforce/src/expectationManagerApi.ts index d0a63e11..a02747a6 100644 --- a/shared/packages/workforce/src/expectationManagerApi.ts +++ b/shared/packages/workforce/src/expectationManagerApi.ts @@ -1,4 +1,4 @@ -import { WorkForceExpectationManager, AdapterServer, AdapterServerOptions } from '@shared/api' +import { WorkForceExpectationManager, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' /** * Exposes the API-methods of a ExpectationManager, to be called from the Workforce @@ -15,5 +15,10 @@ export class ExpectationManagerAPI super(methods, options) } - // Note: This side of the API has no methods exposed. + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } } diff --git a/shared/packages/workforce/src/workerAgentApi.ts b/shared/packages/workforce/src/workerAgentApi.ts index 05714356..0e785763 100644 --- a/shared/packages/workforce/src/workerAgentApi.ts +++ b/shared/packages/workforce/src/workerAgentApi.ts @@ -1,4 +1,4 @@ -import { WorkForceWorkerAgent, AdapterServer, AdapterServerOptions } from '@shared/api' +import { WorkForceWorkerAgent, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' /** * Exposes the API-methods of a WorkerAgent, to be called from the Workforce @@ -15,13 +15,17 @@ export class WorkerAgentAPI super(methods, options) } + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + async expectationManagerAvailable(id: string, url: string): Promise { return this._sendMessage('expectationManagerAvailable', id, url) } async expectationManagerGone(id: string): Promise { return this._sendMessage('expectationManagerGone', id) } - async _debugKill(): Promise { - return this._sendMessage('_debugKill') - } } diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index 748e592a..ffe2371e 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -9,6 +9,7 @@ import { assertNever, WorkForceAppContainer, WorkforceStatus, + LogLevel, } from '@shared/api' import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' @@ -47,7 +48,7 @@ export class Workforce { } = {} private websocketServer?: WebsocketServer - private availableApps: WorkerHandler + private workerHandler: WorkerHandler constructor(public logger: LoggerInstance, config: WorkforceConfig) { if (config.workforce.port !== null) { @@ -107,12 +108,12 @@ export class Workforce { } }) } - this.availableApps = new WorkerHandler(this) + this.workerHandler = new WorkerHandler(this) } async init(): Promise { // Nothing to do here at the moment - this.availableApps.triggerUpdate() + this.workerHandler.triggerUpdate() } terminate(): void { this.websocketServer?.terminate() @@ -173,6 +174,12 @@ export class Workforce { /** Return the API-methods that the Workforce exposes to the ExpectationManager */ private getExpectationManagerAPI(): WorkForceExpectationManager.WorkForce { return { + setLogLevel: async (logLevel: LogLevel): Promise => { + return this.setLogLevel(logLevel) + }, + setLogLevelOfApp: async (appId: string, logLevel: LogLevel): Promise => { + return this.setLogLevelOfApp(appId, logLevel) + }, registerExpectationManager: async (managerId: string, url: string): Promise => { await this.registerExpectationManager(managerId, url) }, @@ -193,6 +200,13 @@ export class Workforce { }, } } + private _debugKill(): void { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } public async registerExpectationManager(managerId: string, url: string): Promise { const em = this.expectationManagers[managerId] @@ -234,12 +248,37 @@ export class Workforce { }), } } + + public setLogLevel(logLevel: LogLevel): void { + this.logger.level = logLevel + } + public async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + const workerAgent = this.workerAgents[appId] + if (workerAgent) return workerAgent.api.setLogLevel(logLevel) + + const appContainer = this.appContainers[appId] + if (appContainer) return appContainer.api.setLogLevel(logLevel) + + const expectationManager = this.expectationManagers[appId] + if (expectationManager) return expectationManager.api.setLogLevel(logLevel) + + if (appId === 'workforce') return this.setLogLevel(logLevel) + throw new Error(`App with id "${appId}" not found`) + } public async _debugKillApp(appId: string): Promise { const workerAgent = this.workerAgents[appId] - if (!workerAgent) throw new Error(`Worker "${appId}" not found`) + if (workerAgent) return workerAgent.api._debugKill() - return workerAgent.api._debugKill() + const appContainer = this.appContainers[appId] + if (appContainer) return appContainer.api._debugKill() + + const expectationManager = this.expectationManagers[appId] + if (expectationManager) return expectationManager.api._debugKill() + + if (appId === 'workforce') return this._debugKill() + throw new Error(`App with id "${appId}" not found`) } + public async removeExpectationManager(managerId: string): Promise { const em = this.expectationManagers[managerId] if (em) { @@ -259,7 +298,7 @@ export class Workforce { .then((runningApps) => { this.appContainers[clientId].runningApps = runningApps this.appContainers[clientId].initialized = true - this.availableApps.triggerUpdate() + this.workerHandler.triggerUpdate() }) .catch((error) => { this.logger.error('Workforce: Error in getRunningApps') From 3190639fd4e8a543a836515d8025349ac455b697 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 25 Aug 2021 16:06:51 +0200 Subject: [PATCH 50/67] feat: scale up workers according to demand (wip) --- .../packages/generic/src/appContainer.ts | 124 +++++++- .../packages/generic/src/workerAgentApi.ts | 34 +++ shared/packages/api/src/appContainer.ts | 2 + shared/packages/api/src/config.ts | 22 +- shared/packages/api/src/lib.ts | 5 + shared/packages/api/src/methods.ts | 19 +- .../src/expectationManager.ts | 58 +++- .../expectationManager/src/workforceApi.ts | 17 +- shared/packages/worker/src/appContainerApi.ts | 17 ++ .../expectationHandlers/fileCopy.ts | 2 +- .../windowsWorker/expectationHandlers/lib.ts | 6 +- shared/packages/worker/src/workerAgent.ts | 33 ++- .../packages/workforce/src/appContainerApi.ts | 5 +- .../packages/workforce/src/workerHandler.ts | 266 ++++++++++-------- shared/packages/workforce/src/workforce.ts | 11 +- 15 files changed, 462 insertions(+), 159 deletions(-) create mode 100644 apps/appcontainer-node/packages/generic/src/workerAgentApi.ts create mode 100644 shared/packages/worker/src/appContainerApi.ts diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 1dcc0cbb..4df02ba9 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -1,8 +1,20 @@ import * as ChildProcess from 'child_process' import * as path from 'path' import * as fs from 'fs' -import { LoggerInstance, AppContainerProcessConfig, ClientConnectionOptions, LogLevel } from '@shared/api' +import { + LoggerInstance, + AppContainerProcessConfig, + ClientConnectionOptions, + LogLevel, + WebsocketServer, + ClientConnection, + AppContainerWorkerAgent, + assertNever, + Expectation, + waitTime, +} from '@shared/api' import { WorkforceAPI } from './workforceApi' +import { WorkerAgentAPI } from './workerAgentApi' /** Mimimum time between app restarts */ const RESTART_COOLDOWN = 60 * 1000 // ms @@ -10,7 +22,7 @@ const RESTART_COOLDOWN = 60 * 1000 // ms export class AppContainer { private workforceAPI: WorkforceAPI private id: string - private connectionOptions: ClientConnectionOptions + private workForceConnectionOptions: ClientConnectionOptions private appId = 0 private apps: { @@ -20,17 +32,52 @@ export class AppContainer { toBeKilled: boolean restarts: number lastRestart: number + workerAgentApi?: WorkerAgentAPI } } = {} private availableApps: { [appType: string]: AvailableAppInfo } = {} + private websocketServer?: WebsocketServer constructor(private logger: LoggerInstance, private config: AppContainerProcessConfig) { + if (config.appContainer.port !== null) { + this.websocketServer = new WebsocketServer(config.appContainer.port, (client: ClientConnection) => { + // A new client has connected + + this.logger.info(`AppContainer: New client "${client.clientType}" connected, id "${client.clientId}"`) + + switch (client.clientType) { + case 'workerAgent': { + const workForceMethods = this.getWorkerAgentAPI() + const api = new WorkerAgentAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + if (!this.apps[client.clientId]) { + throw new Error(`Unknown app "${client.clientId}" just connected to the appContainer`) + } + this.apps[client.clientId].workerAgentApi = api + client.on('close', () => { + delete this.apps[client.clientId].workerAgentApi + }) + break + } + case 'expectationManager': + case 'appContainer': + case 'N/A': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: + assertNever(client.clientType) + throw new Error(`Workforce: Unknown clientType "${client.clientType}"`) + } + }) + } + this.workforceAPI = new WorkforceAPI(this.logger) this.id = config.appContainer.appContainerId - this.connectionOptions = this.config.appContainer.workforceURL + this.workForceConnectionOptions = this.config.appContainer.workforceURL ? { type: 'websocket', url: this.config.appContainer.workforceURL, @@ -40,13 +87,13 @@ export class AppContainer { } } async init(): Promise { - if (this.connectionOptions.type === 'websocket') { - this.logger.info(`AppContainer: Connecting to Workforce at "${this.connectionOptions.url}"`) + if (this.workForceConnectionOptions.type === 'websocket') { + this.logger.info(`AppContainer: Connecting to Workforce at "${this.workForceConnectionOptions.url}"`) } await this.setupAvailableApps() - await this.workforceAPI.init(this.id, this.connectionOptions, this) + await this.workforceAPI.init(this.id, this.workForceConnectionOptions, this) await this.workforceAPI.registerAvailableApps( Object.entries(this.availableApps).map((o) => { @@ -64,11 +111,20 @@ export class AppContainer { // * how many can be spun up // * etc... } + /** Return the API-methods that the AppContainer exposes to the WorkerAgent */ + private getWorkerAgentAPI(): AppContainerWorkerAgent.AppContainer { + return { + ping: async (): Promise => { + // todo: Set last seen + }, + } + } private async setupAvailableApps() { const getWorkerArgs = (appId: string): string[] => { return [ `--workerId=${appId}`, `--workforceURL=${this.config.appContainer.workforceURL}`, + `--appContainerURL=${'ws://127.0.0.1:' + this.websocketServer?.port}`, this.config.appContainer.windowsDriveLetters ? `--windowsDriveLetters=${this.config.appContainer.windowsDriveLetters?.join(';')}` : '', @@ -85,6 +141,7 @@ export class AppContainer { args: (appId: string) => { return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId)] }, + cost: 0, } } else { // Process is a compiled executable @@ -100,6 +157,7 @@ export class AppContainer { args: (appId: string) => { return [...getWorkerArgs(appId)] }, + cost: 0, } } }) @@ -107,6 +165,7 @@ export class AppContainer { } terminate(): void { this.workforceAPI.terminate() + this.websocketServer?.terminate() // kill child processes } @@ -121,7 +180,45 @@ export class AppContainer { }, 1) } - async spinUp(appType: AppType): Promise { + async requestAppTypeForExpectation(exp: Expectation.Any): Promise<{ appType: string; cost: number } | null> { + if (Object.keys(this.apps).length >= this.config.appContainer.maxRunningApps) { + // If we're at our limit, we can't possibly run anything else + return null + } + + for (const [appType, availableApp] of Object.entries(this.availableApps)) { + // Do we already have any instance of the appType running? + let runningApp = Object.values(this.apps).find((app) => { + return app.appType === appType + }) + + if (!runningApp) { + const newAppId = await this.spinUp(appType) // todo: make it not die too soon + + // wait for the app to connect to us: + tryAfewTimes(async () => { + if (this.apps[newAppId].workerAgentApi) { + return true + } + await waitTime(200) + return false + }, 10) + runningApp = this.apps[newAppId] + if (!runningApp) throw new Error(`AppContainer: Worker "${newAppId}" didn't connect in time`) + } + if (runningApp?.workerAgentApi) { + const result = await runningApp.workerAgentApi.doYouSupportExpectation(exp) + if (result.support) { + return { + appType: appType, + cost: availableApp.cost, + } + } + } + } + return null + } + async spinUp(appType: string): Promise { const availableApp = this.availableApps[appType] if (!availableApp) throw new Error(`Unknown appType "${appType}"`) @@ -159,7 +256,7 @@ export class AppContainer { }) } private setupChildProcess( - appType: AppType, + appType: string, appId: string, availableApp: AvailableAppInfo ): ChildProcess.ChildProcess { @@ -204,8 +301,17 @@ export class AppContainer { return child } } -type AppType = 'worker' // | other interface AvailableAppInfo { file: string args: (appId: string) => string[] + /** Some kind of value, how much it costs to run it, per minute */ + cost: number +} + +async function tryAfewTimes(cb: () => Promise, maxTries: number) { + for (let i = 0; i < maxTries; i++) { + if (await cb()) { + break + } + } } diff --git a/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts new file mode 100644 index 00000000..1232288c --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts @@ -0,0 +1,34 @@ +import { + AppContainerWorkerAgent, + AdapterServer, + AdapterServerOptions, + LogLevel, + Expectation, + ReturnTypeDoYouSupportExpectation, +} from '@shared/api' + +/** + * Exposes the API-methods of a WorkerAgent, to be called from the AppContainer + * Note: The WorkerAgent connects to the AppContainer, therefore the AppContainer is the AdapterServer here. + * The corresponding other side is implemented at shared/packages/worker/src/appContainerApi.ts + */ +export class WorkerAgentAPI + extends AdapterServer + implements AppContainerWorkerAgent.WorkerAgent { + constructor( + methods: AppContainerWorkerAgent.AppContainer, + options: AdapterServerOptions + ) { + super(methods, options) + } + + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + async doYouSupportExpectation(exp: Expectation.Any): Promise { + return this._sendMessage('doYouSupportExpectation', exp) + } +} diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts index afce7a06..7ac0523e 100644 --- a/shared/packages/api/src/appContainer.ts +++ b/shared/packages/api/src/appContainer.ts @@ -4,7 +4,9 @@ import { WorkerAgentConfig } from './worker' export interface AppContainerConfig { workforceURL: string | null + port: number | null appContainerId: string + maxRunningApps: number resourceId: string networkIds: string[] diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index aa6b1b83..2ec834ee 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -109,6 +109,11 @@ const workerArguments = defineArguments({ default: process.env.WORKFORCE_URL || 'ws://localhost:8070', describe: 'The URL to the Workforce', }, + appContainerURL: { + type: 'string', + default: process.env.APP_CONTAINER_URL || '', // 'ws://localhost:8090', + describe: 'The URL to the AppContainer', + }, windowsDriveLetters: { type: 'string', default: process.env.WORKER_WINDOWS_DRIVE_LETTERS || 'X;Y;Z', @@ -137,6 +142,16 @@ const appContainerArguments = defineArguments({ default: process.env.WORKFORCE_URL || 'ws://localhost:8070', describe: 'The URL to the Workforce', }, + port: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_PORT || '', 10) || 8090, + describe: 'The port number to start the App Container websocket server on', + }, + maxRunningApps: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_MAX_RUNNING_APPS || '', 10) || 3, + describe: 'How many apps the appContainer can run at the same time', + }, resourceId: { type: 'string', @@ -287,6 +302,7 @@ export interface WorkerConfig { process: ProcessConfig worker: { workforceURL: string | null + appContainerURL: string | null resourceId: string networkIds: string[] } & WorkerAgentConfig @@ -302,6 +318,7 @@ export function getWorkerConfig(): WorkerConfig { worker: { workerId: argv.workerId, workforceURL: argv.workforceURL, + appContainerURL: argv.appContainerURL, windowsDriveLetters: argv.windowsDriveLetters ? argv.windowsDriveLetters.split(';') : [], resourceId: argv.resourceId, networkIds: argv.networkIds ? argv.networkIds.split(';') : [], @@ -322,8 +339,11 @@ export function getAppContainerConfig(): AppContainerProcessConfig { return { process: getProcessConfig(argv), appContainer: { - appContainerId: argv.appContainerId, workforceURL: argv.workforceURL, + port: argv.port, + appContainerId: argv.appContainerId, + maxRunningApps: argv.maxRunningApps, + resourceId: argv.resourceId, networkIds: argv.networkIds ? argv.networkIds.split(';') : [], windowsDriveLetters: argv.windowsDriveLetters ? argv.windowsDriveLetters.split(';') : [], diff --git a/shared/packages/api/src/lib.ts b/shared/packages/api/src/lib.ts index 3cee0b5b..ec5ff31b 100644 --- a/shared/packages/api/src/lib.ts +++ b/shared/packages/api/src/lib.ts @@ -44,3 +44,8 @@ export function hash(str: string): string { export function assertNever(_value: never): void { // does nothing } +export function waitTime(duration: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, duration) + }) +} diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 973e0a1c..52afd610 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -30,8 +30,7 @@ export namespace WorkForceExpectationManager { _debugKillApp(appId: string): Promise getStatus: () => Promise - // This is a temporary function: - setWorkerCount: (count: number) => Promise + requestResources: (exp: Expectation.Any) => Promise registerExpectationManager: (managerId: string, url: string) => Promise } @@ -172,6 +171,7 @@ export namespace WorkForceAppContainer { setLogLevel: (logLevel: LogLevel) => Promise _debugKill: () => Promise + requestAppTypeForExpectation: (exp: Expectation.Any) => Promise<{ appType: string; cost: number } | null> spinUp: ( appType: 'worker' // | other ) => Promise @@ -183,3 +183,18 @@ export namespace WorkForceAppContainer { registerAvailableApps: (availableApps: { appType: string }[]) => Promise } } + +/** Methods used by AppContainer and WorkerAgent */ +export namespace AppContainerWorkerAgent { + /** Methods on WorkerAgent, called by AppContainer */ + export interface WorkerAgent { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + + doYouSupportExpectation: (exp: Expectation.Any) => Promise + } + /** Methods on AppContainer, called by WorkerAgent */ + export interface AppContainer { + ping: () => Promise + } +} diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 03cc737f..19557491 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -37,6 +37,11 @@ export class ExpectationManager { */ private readonly ALLOW_SKIPPING_QUEUE_TIME = 30 * 1000 // ms + /** How long to wait before requesting more resources (workers) */ + private readonly SCALE_UP_TIME = 5 * 1000 // ms + /** How many resources to request at a time */ + private readonly SCALE_UP_COUNT = 1 + private workforceAPI: WorkforceAPI /** Store for various incoming data, to be processed on next iteration round */ @@ -414,6 +419,8 @@ export class ExpectationManager { this.updateStatus() + this.checkIfNeedToScaleUp() + if (runAgainASAP) { this._triggerEvaluateExpectations(true) } @@ -450,6 +457,7 @@ export class ExpectationManager { state: ExpectedPackageStatusAPI.WorkStatusState.NEW, availableWorkers: [], lastEvaluationTime: 0, + waitingForWorkerTime: null, errorCount: 0, reason: { user: '', @@ -709,7 +717,7 @@ export class ExpectationManager { } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { // Check if the expectation is ready to start: - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // First, check if it is already fulfilled: @@ -770,7 +778,7 @@ export class ExpectationManager { } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) { // Start working on it: - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { const assignedWorker = trackedExp.session.assignedWorker @@ -813,7 +821,7 @@ export class ExpectationManager { // TODO: Some monitor that is able to invalidate if it isn't fullfilled anymore? if (timeSinceLastEvaluation > this.getFullfilledWaitTime()) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // Check if it is still fulfilled: const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( @@ -848,7 +856,7 @@ export class ExpectationManager { // Do nothing } } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.REMOVED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { @@ -872,7 +880,7 @@ export class ExpectationManager { ) } } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.RESTARTED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) @@ -899,7 +907,7 @@ export class ExpectationManager { ) } } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.ABORTED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) @@ -1031,10 +1039,9 @@ export class ExpectationManager { } } /** Do a bidding between the available Workers and assign the cheapest one to use for the evaulation-session. */ - private async assignWorkerToSession( - session: ExpectationStateHandlerSession, - trackedExp: TrackedExpectation - ): Promise { + private async assignWorkerToSession(trackedExp: TrackedExpectation): Promise { + const session: ExpectationStateHandlerSession | null = trackedExp.session + if (!session) throw new Error('ExpectationManager: INternal error: Session not set') if (session.assignedWorker) return // A worker has already been assigned /** How many requests to send out simultaneously */ @@ -1388,6 +1395,35 @@ export class ExpectationManager { } return this.status } + private async checkIfNeedToScaleUp(): Promise { + const waitingExpectations: TrackedExpectation[] = [] + + for (const exp of Object.values(this.trackedExpectations)) { + if ( + (exp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW || + exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) && + !exp.availableWorkers.length && // No workers supports it + !exp.session?.assignedWorker // No worker has time to work on it + ) { + if (!exp.waitingForWorkerTime) { + exp.waitingForWorkerTime = Date.now() + } + } else { + exp.waitingForWorkerTime = null + } + if (exp.waitingForWorkerTime) + if (exp.waitingForWorkerTime && Date.now() - exp.waitingForWorkerTime > this.SCALE_UP_TIME) { + if (waitingExpectations.length < this.SCALE_UP_COUNT) { + waitingExpectations.push(exp) + } + } + } + + for (const exp of waitingExpectations) { + this.logger.info(`Requesting more resources to handle expectation "${exp.id}"`) + await this.workforceAPI.requestResources(exp.exp) + } + } } export type ExpectationManagerServerOptions = | { @@ -1414,6 +1450,8 @@ interface TrackedExpectation { availableWorkers: string[] /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number + /** Timestamp to be track how long the expectation has been awiting for a worker (can't start working) */ + waitingForWorkerTime: number | null /** The number of times the expectation has failed */ errorCount: number diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index a066c501..aa8a473b 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -1,4 +1,11 @@ -import { WorkForceExpectationManager, AdapterClient, LoggerInstance, LogLevel, WorkforceStatus } from '@shared/api' +import { + WorkForceExpectationManager, + AdapterClient, + LoggerInstance, + LogLevel, + WorkforceStatus, + Expectation, +} from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the ExpectationManager @@ -12,10 +19,6 @@ export class WorkforceAPI super(logger, 'expectationManager') } - async setWorkerCount(count: number): Promise { - // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts - return this._sendMessage('setWorkerCount', count) - } async registerExpectationManager(managerId: string, url: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('registerExpectationManager', managerId, url) @@ -36,4 +39,8 @@ export class WorkforceAPI // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts return this._sendMessage('_debugKillApp', appId) } + async requestResources(exp: Expectation.Any): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('requestResources', exp) + } } diff --git a/shared/packages/worker/src/appContainerApi.ts b/shared/packages/worker/src/appContainerApi.ts new file mode 100644 index 00000000..b6f159cc --- /dev/null +++ b/shared/packages/worker/src/appContainerApi.ts @@ -0,0 +1,17 @@ +import { AdapterClient, LoggerInstance, AppContainerWorkerAgent } from '@shared/api' + +/** + * Exposes the API-methods of a Workforce, to be called from the WorkerAgent + * Note: The WorkerAgent connects to the Workforce, therefore the WorkerAgent is the AdapterClient here. + * The corresponding other side is implemented at shared/packages/workforce/src/workerAgentApi.ts + */ +export class AppContainerAPI + extends AdapterClient + implements AppContainerWorkerAgent.AppContainer { + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') + } + async ping(): Promise { + return this._sendMessage('ping') + } +} diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index b04af1eb..5ce06e22 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -6,6 +6,7 @@ import { UniversalVersion, compareUniversalVersions, makeUniversalVersion, getSt import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { hashObj, + waitTime, Expectation, ReturnTypeDoYouSupportExpectation, ReturnTypeGetCostFortExpectation, @@ -25,7 +26,6 @@ import { lookupAccessorHandles, LookupPackageContainer, userReadableDiff, - waitTime, } from './lib' import { CancelablePromise } from '../../../lib/cancelablePromise' import { PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index 383ce1bd..c580c582 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -212,11 +212,7 @@ export async function lookupAccessorHandles( reason: errorReason, } } -export function waitTime(duration: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, duration) - }) -} + /** Converts a diff to some kind of user-readable string */ export function userReadableDiff(diffs: Diff[]): string { const strs: string[] = [] diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 44d66b41..3682ad88 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -18,6 +18,7 @@ import { ReturnTypeDisposePackageContainerMonitors, LogLevel, } from '@shared/api' +import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' import { IWorkInProgress } from './worker/lib/workInProgress' import { GenericWorker } from './worker/worker' @@ -32,11 +33,13 @@ export class WorkerAgent { // private _busyMethodCount = 0 private currentJobs: { cost: ExpectationManagerWorkerAgent.ExpectationCost; progress: number }[] = [] private workforceAPI: WorkforceAPI + private appContainerAPI: AppContainerAPI private wipI = 0 private worksInProgress: { [wipId: string]: IWorkInProgress } = {} private id: string - private connectionOptions: ClientConnectionOptions + private workForceConnectionOptions: ClientConnectionOptions + private appContainerConnectionOptions: ClientConnectionOptions | null private expectationManagers: { [id: string]: { @@ -53,9 +56,10 @@ export class WorkerAgent { constructor(private logger: LoggerInstance, private config: WorkerConfig) { this.workforceAPI = new WorkforceAPI(this.logger) + this.appContainerAPI = new AppContainerAPI(this.logger) this.id = config.worker.workerId - this.connectionOptions = this.config.worker.workforceURL + this.workForceConnectionOptions = this.config.worker.workforceURL ? { type: 'websocket', url: this.config.worker.workforceURL, @@ -63,7 +67,12 @@ export class WorkerAgent { : { type: 'internal', } - + this.appContainerConnectionOptions = this.config.worker.appContainerURL + ? { + type: 'websocket', + url: this.config.worker.appContainerURL, + } + : null // Todo: Different types of workers: this._worker = new WindowsWorker( this.logger, @@ -84,10 +93,19 @@ export class WorkerAgent { ) } async init(): Promise { - if (this.connectionOptions.type === 'websocket') { - this.logger.info(`Worker: Connecting to Workforce at "${this.connectionOptions.url}"`) + // Connect to WorkForce: + if (this.workForceConnectionOptions.type === 'websocket') { + this.logger.info(`Worker: Connecting to Workforce at "${this.workForceConnectionOptions.url}"`) + } + await this.workforceAPI.init(this.id, this.workForceConnectionOptions, this) + + // Connect to AppContainer (if applicable) + if (this.appContainerConnectionOptions) { + if (this.appContainerConnectionOptions.type === 'websocket') { + this.logger.info(`Worker: Connecting to AppContainer at "${this.appContainerConnectionOptions.url}"`) + } + await this.appContainerAPI.init(this.id, this.appContainerConnectionOptions, this) } - await this.workforceAPI.init(this.id, this.connectionOptions, this) const list = await this.workforceAPI.getExpectationManagerList() await this.updateListOfExpectationManagers(list) @@ -125,6 +143,9 @@ export class WorkerAgent { // isFree(): boolean { // return this._busyMethodCount === 0 // } + async doYouSupportExpectation(exp: Expectation.Any): Promise { + return this._worker.doYouSupportExpectation(exp) + } async expectationManagerAvailable(id: string, url: string): Promise { const existing = this.expectationManagers[id] if (existing) { diff --git a/shared/packages/workforce/src/appContainerApi.ts b/shared/packages/workforce/src/appContainerApi.ts index 27c652c9..fc64bbd8 100644 --- a/shared/packages/workforce/src/appContainerApi.ts +++ b/shared/packages/workforce/src/appContainerApi.ts @@ -1,4 +1,4 @@ -import { WorkForceAppContainer, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' +import { WorkForceAppContainer, AdapterServer, AdapterServerOptions, LogLevel, Expectation } from '@shared/api' /** * Exposes the API-methods of a AppContainer, to be called from the Workforce @@ -22,6 +22,9 @@ export class AppContainerAPI return this._sendMessage('_debugKill') } + async requestAppTypeForExpectation(exp: Expectation.Any): Promise<{ appType: string; cost: number } | null> { + return this._sendMessage('requestAppTypeForExpectation', exp) + } async spinUp(appType: string): Promise { return this._sendMessage('spinUp', appType) } diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts index 5a982d05..1b30687a 100644 --- a/shared/packages/workforce/src/workerHandler.ts +++ b/shared/packages/workforce/src/workerHandler.ts @@ -1,136 +1,168 @@ +import { Expectation } from '@shared/api' import { Workforce } from './workforce' -const UPDATE_INTERVAL = 10 * 1000 +// const UPDATE_INTERVAL = 10 * 1000 /** The WorkerHandler is in charge of spinning up/down Workers */ export class WorkerHandler { - private updateTimeout: NodeJS.Timer | null = null - private updateAgain = false - private updateInterval: NodeJS.Timeout - private terminated = false + // private updateTimeout: NodeJS.Timer | null = null + // private updateAgain = false + // private updateInterval: NodeJS.Timeout + // private terminated = false - private plannedWorkers: PlannedWorker[] = [] + // private plannedWorkers: PlannedWorker[] = [] + // private workerCount = 0 constructor(private workForce: Workforce) { - this.updateInterval = setInterval(() => { - this.triggerUpdate() - }, UPDATE_INTERVAL) + // this.updateInterval = setInterval(() => { + // this.triggerUpdate() + // }, UPDATE_INTERVAL) } public terminate(): void { - clearInterval(this.updateInterval) - this.terminated = true + // clearInterval(this.updateInterval) + // this.terminated = true } - public triggerUpdate(): void { - if (this.terminated) return - - if (!this.updateTimeout) { - this.updateAgain = false - this.updateTimeout = setTimeout(() => { - this.update() - .catch((error) => { - this.workForce.logger.error(error) - }) - .finally(() => { - this.updateTimeout = null - if (this.updateAgain) this.triggerUpdate() - }) - }, 500) - } else { - this.updateAgain = true - } - } - private async update(): Promise { - // Update this.plannedWorkers + public async requestResources(exp: Expectation.Any): Promise { + let best: { appContainerId: string; appType: string; cost: number } | null = null for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { - for (const runningApp of appContainer.runningApps) { - const plannedWorker = this.plannedWorkers.find((pw) => pw.appId === runningApp.appId) - if (!plannedWorker) { - this.plannedWorkers.push({ + const proposal = await appContainer.api.requestAppTypeForExpectation(exp) + if (proposal) { + if (!best || proposal.cost < best.cost) { + best = { appContainerId: appContainerId, - appType: runningApp.appType, - appId: runningApp.appId, - isInUse: false, - }) - } - } - } - - // This is a temporary stupid implementation, - // to be reworked later.. - const needs: AppTarget[] = [ - { - appType: 'worker', - }, - { - appType: 'worker', - }, - { - appType: 'worker', - }, - ] - // Reset plannedWorkers: - for (const plannedWorker of this.plannedWorkers) { - plannedWorker.isInUse = false - } - - // Initial check to see which needs are already fulfilled: - for (const need of needs) { - // Do we have anything that fullfills the need? - for (const plannedWorker of this.plannedWorkers) { - if (plannedWorker.isInUse) continue - - if (plannedWorker.appType === need.appType) { - // ^ Later, we'll add more checks here ^ - need.fulfilled = true - plannedWorker.isInUse = true - break - } - } - } - for (const need of needs) { - if (need.fulfilled) continue - - // See which AppContainers can fullfill our need: - let found = false - for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { - if (found) break - if (!appContainer.initialized) continue - - for (const availableApp of appContainer.availableApps) { - if (availableApp.appType === need.appType) { - // Spin up that worker: - - this.workForce.logger.info( - `Workforce: Spinning up another worker (${availableApp.appType}) on "${appContainerId}"` - ) - - const newPlannedWorker: PlannedWorker = { - appContainerId: appContainerId, - appType: availableApp.appType, - isInUse: true, - } - this.plannedWorkers.push(newPlannedWorker) - - const appId = await appContainer.api.spinUp(availableApp.appType) - - newPlannedWorker.appId = appId - - found = true - break + appType: proposal.appType, + cost: proposal.cost, } } } } + if (best) { + const appContainer = this.workForce.appContainers[best.appContainerId] + if (!appContainer) throw new Error(`WorkerHandler: AppContainer "${best.appContainerId}" not found`) + + this.workForce.logger.info( + `Workforce: Spinning up another worker (${best.appType}) on "${best.appContainerId}"` + ) + + // const newPlannedWorker: PlannedWorker = { + // appContainerId: best.appContainerId, + // appType: best.appType, + // } + // this.plannedWorkers.push(newPlannedWorker) + + await appContainer.api.spinUp(best.appType) + // newPlannedWorker.appId = appId + return true + } else { + return false + } } + // public triggerUpdate(): void { + // if (this.terminated) return + + // if (!this.updateTimeout) { + // this.updateAgain = false + // this.updateTimeout = setTimeout(() => { + // this.update() + // .catch((error) => { + // this.workForce.logger.error(error) + // }) + // .finally(() => { + // this.updateTimeout = null + // if (this.updateAgain) this.triggerUpdate() + // }) + // }, 500) + // } else { + // this.updateAgain = true + // } + // } + // private async update(): Promise { + // // Update this.plannedWorkers + // for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + // for (const runningApp of appContainer.runningApps) { + // const plannedWorker = this.plannedWorkers.find((pw) => pw.appId === runningApp.appId) + // if (!plannedWorker) { + // this.plannedWorkers.push({ + // appContainerId: appContainerId, + // appType: runningApp.appType, + // appId: runningApp.appId, + // isInUse: false, + // }) + // } + // } + // } + + // // This is a temporary stupid implementation, + // // to be reworked later.. + // const needs: AppTarget[] = [] + // for (let i = 0; i < this.workerCount; i++) { + // needs.push({ + // appType: 'worker', + // }) + // } + // // Reset plannedWorkers: + // for (const plannedWorker of this.plannedWorkers) { + // plannedWorker.isInUse = false + // } + + // // Initial check to see which needs are already fulfilled: + // for (const need of needs) { + // // Do we have anything that fullfills the need? + // for (const plannedWorker of this.plannedWorkers) { + // if (plannedWorker.isInUse) continue + + // if (plannedWorker.appType === need.appType) { + // // ^ Later, we'll add more checks here ^ + // need.fulfilled = true + // plannedWorker.isInUse = true + // break + // } + // } + // } + // for (const need of needs) { + // if (need.fulfilled) continue + + // // See which AppContainers can fullfill our need: + // let found = false + // for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + // if (found) break + // if (!appContainer.initialized) continue + + // for (const availableApp of appContainer.availableApps) { + // if (availableApp.appType === need.appType) { + // // Spin up that worker: + + // this.workForce.logger.info( + // `Workforce: Spinning up another worker (${availableApp.appType}) on "${appContainerId}"` + // ) + + // const newPlannedWorker: PlannedWorker = { + // appContainerId: appContainerId, + // appType: availableApp.appType, + // isInUse: true, + // } + // this.plannedWorkers.push(newPlannedWorker) + + // const appId = await appContainer.api.spinUp(availableApp.appType) + + // newPlannedWorker.appId = appId + + // found = true + // break + // } + // } + // } + // } + // } } -interface PlannedWorker { - appType: string - appContainerId: string - appId?: string - - isInUse: boolean -} -interface AppTarget { - appType: string - fulfilled?: boolean -} +// interface PlannedWorker { +// appType: string +// appContainerId: string +// appId?: string + +// isInUse: boolean +// } +// interface AppTarget { +// appType: string +// fulfilled?: boolean +// } diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index ffe2371e..72b2dbb5 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -10,6 +10,7 @@ import { WorkForceAppContainer, WorkforceStatus, LogLevel, + Expectation, } from '@shared/api' import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' @@ -113,7 +114,7 @@ export class Workforce { async init(): Promise { // Nothing to do here at the moment - this.workerHandler.triggerUpdate() + // this.workerHandler.triggerUpdate() } terminate(): void { this.websocketServer?.terminate() @@ -183,6 +184,9 @@ export class Workforce { registerExpectationManager: async (managerId: string, url: string): Promise => { await this.registerExpectationManager(managerId, url) }, + requestResources: async (exp: Expectation.Any): Promise => { + return this.requestResources(exp) + }, getStatus: async (): Promise => { return this.getStatus() @@ -222,6 +226,9 @@ export class Workforce { } this.expectationManagers[managerId].url = url } + public async requestResources(exp: Expectation.Any): Promise { + return this.workerHandler.requestResources(exp) + } public async getStatus(): Promise { return { workerAgents: Object.entries(this.workerAgents).map(([workerId, _workerAgent]) => { @@ -298,7 +305,7 @@ export class Workforce { .then((runningApps) => { this.appContainers[clientId].runningApps = runningApps this.appContainers[clientId].initialized = true - this.workerHandler.triggerUpdate() + // this.workerHandler.triggerUpdate() }) .catch((error) => { this.logger.error('Workforce: Error in getRunningApps') From 49fd07560888cf5400019fb2b48491ab3a6c7d1a Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 26 Aug 2021 10:02:55 +0200 Subject: [PATCH 51/67] chore: move ExpectationManager constants, so that they can be configured (used in tests) --- .../src/expectationManager.ts | 84 ++++++++++++++----- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 19557491..958c23b4 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -27,20 +27,7 @@ import PromisePool from '@supercharge/promise-pool' */ export class ExpectationManager { - /** Time between iterations of the expectation queue */ - private readonly EVALUATE_INTERVAL = 10 * 1000 // ms - /** Minimum time between re-evaluating fulfilled expectations */ - private readonly FULLFILLED_MONITOR_TIME = 10 * 1000 // ms - /** - * If the iteration of the queue has been going for this time - * allow skipping the rest of the queue in order to reiterate the high-prio expectations - */ - private readonly ALLOW_SKIPPING_QUEUE_TIME = 30 * 1000 // ms - - /** How long to wait before requesting more resources (workers) */ - private readonly SCALE_UP_TIME = 5 * 1000 // ms - /** How many resources to request at a time */ - private readonly SCALE_UP_COUNT = 1 + private constants: ExpectationManagerConstants private workforceAPI: WorkforceAPI @@ -109,8 +96,17 @@ export class ExpectationManager { /** At what url the ExpectationManager can be reached on */ private serverAccessBaseUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, - private callbacks: ExpectationManagerCallbacks + private callbacks: ExpectationManagerCallbacks, + options?: ExpectationManagerOptions ) { + this.constants = { + EVALUATE_INTERVAL: 10 * 1000, + FULLFILLED_MONITOR_TIME: 10 * 1000, + ALLOW_SKIPPING_QUEUE_TIME: 30 * 1000, + SCALE_UP_TIME: 5 * 1000, + SCALE_UP_COUNT: 1, + ...options?.constants, + } this.workforceAPI = new WorkforceAPI(this.logger) this.status = this.updateStatus() if (this.serverOptions.type === 'websocket') { @@ -182,6 +178,24 @@ export class ExpectationManager { this.websocketServer.terminate() } } + /** USED IN TESTS ONLY. Quickly reset the tracked work of the expectationManager. */ + resetWork(): void { + this.receivedUpdates = { + expectations: {}, + expectationsHasBeenUpdated: false, + packageContainers: {}, + packageContainersHasBeenUpdated: false, + restartExpectations: {}, + abortExpectations: {}, + restartAllExpectations: false, + } + this.trackedExpectations = {} + this.trackedExpectationsCount = 0 + this.trackedPackageContainers = {} + // this.worksInProgress + + this._triggerEvaluateExpectations(true) + } /** Returns a Hook used to hook up a WorkerAgent to our API-methods. */ getWorkerAgentHook(): Hook< ExpectationManagerWorkerAgent.ExpectationManager, @@ -293,7 +307,7 @@ export class ExpectationManager { this._triggerEvaluateExpectations() }) }, - this._evaluateExpectationsRunAsap ? 1 : this.EVALUATE_INTERVAL + this._evaluateExpectationsRunAsap ? 1 : this.constants.EVALUATE_INTERVAL ) } /** Return the API-methods that the ExpectationManager exposes to the WorkerAgent */ @@ -657,7 +671,7 @@ export class ExpectationManager { removeIds.push(trackedExp.id) } } - if (runAgainASAP && Date.now() - startTime > this.ALLOW_SKIPPING_QUEUE_TIME) { + if (runAgainASAP && Date.now() - startTime > this.constants.ALLOW_SKIPPING_QUEUE_TIME) { // Skip the rest of the queue, so that we don't get stuck on evaluating low-prio expectations. break } @@ -949,7 +963,7 @@ export class ExpectationManager { private getFullfilledWaitTime(): number { return ( // Default minimum time to wait: - this.FULLFILLED_MONITOR_TIME + + this.constants.FULLFILLED_MONITOR_TIME + // Also add some more time, so that we don't check too often when we have a lot of expectations: this.trackedExpectationsCount * 0.02 ) @@ -1389,9 +1403,16 @@ export class ExpectationManager { expectationStatistics.countAborted++ } else assertNever(exp.state) - if (!exp.availableWorkers.length) expectationStatistics.countNoAvailableWorkers - if (exp.errorCount > 0 && exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) + if (!exp.availableWorkers.length) { + expectationStatistics.countNoAvailableWorkers++ + } + if ( + exp.errorCount > 0 && + exp.state !== ExpectedPackageStatusAPI.WorkStatusState.WORKING && + exp.state !== ExpectedPackageStatusAPI.WorkStatusState.FULFILLED + ) { expectationStatistics.countError++ + } } return this.status } @@ -1412,8 +1433,8 @@ export class ExpectationManager { exp.waitingForWorkerTime = null } if (exp.waitingForWorkerTime) - if (exp.waitingForWorkerTime && Date.now() - exp.waitingForWorkerTime > this.SCALE_UP_TIME) { - if (waitingExpectations.length < this.SCALE_UP_COUNT) { + if (exp.waitingForWorkerTime && Date.now() - exp.waitingForWorkerTime > this.constants.SCALE_UP_TIME) { + if (waitingExpectations.length < this.constants.SCALE_UP_COUNT) { waitingExpectations.push(exp) } } @@ -1425,6 +1446,25 @@ export class ExpectationManager { } } } +export interface ExpectationManagerOptions { + constants: Partial +} +export interface ExpectationManagerConstants { + /** Time between iterations of the expectation queue [ms] */ + EVALUATE_INTERVAL: number + /** Minimum time between re-evaluating fulfilled expectations [ms] */ + FULLFILLED_MONITOR_TIME: number + /** + * If the iteration of the queue has been going for this time + * allow skipping the rest of the queue in order to reiterate the high-prio expectations [ms] + */ + ALLOW_SKIPPING_QUEUE_TIME: number + + /** How long to wait before requesting more resources (workers) [ms] */ + SCALE_UP_TIME: number + /** How many resources to request at a time */ + SCALE_UP_COUNT: number +} export type ExpectationManagerServerOptions = | { type: 'websocket' From 64c2416b4e2da7b52ce4f4f8a439991f2f8a6814 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 26 Aug 2021 10:07:41 +0200 Subject: [PATCH 52/67] chore: fix and add tests --- .../src/__mocks__/child_process.ts | 9 +- tests/internal-tests/src/__mocks__/fs.ts | 53 ++- .../src/__tests__/basic.spec.ts | 374 ++++++++---------- .../src/__tests__/issues.spec.ts | 306 +++++++++----- tests/internal-tests/src/__tests__/lib/lib.ts | 4 +- .../src/__tests__/lib/setupEnv.ts | 147 ++++--- 6 files changed, 511 insertions(+), 382 deletions(-) diff --git a/tests/internal-tests/src/__mocks__/child_process.ts b/tests/internal-tests/src/__mocks__/child_process.ts index eec8fe6e..200dc7cc 100644 --- a/tests/internal-tests/src/__mocks__/child_process.ts +++ b/tests/internal-tests/src/__mocks__/child_process.ts @@ -6,8 +6,7 @@ import path from 'path' /* eslint-disable no-console */ const fsCopyFile = promisify(fs.copyFile) -// @ts-expect-error mock -const fs__mockSetDirectory = fs.__mockSetDirectory +const fsMkdir = promisify(fs.mkdir) const child_process: any = jest.createMockFromModule('child_process') @@ -123,14 +122,14 @@ async function robocopy(spawned: SpawnedProcess, args: string[]) { const source = path.join(sourceFolder, file) const destination = path.join(destinationFolder, file) - fs__mockSetDirectory(destinationFolder) // robocopy automatically creates the destination folder + await fsMkdir(destinationFolder) // robocopy automatically creates the destination folder await fsCopyFile(source, destination) } spawned.emit('close', 1) // OK } catch (err) { - console.log(err) - spawned.emit('close', 9999) + // console.log(err) + spawned.emit('close', 16) // Serious error. Robocopy did not copy any files. } } async function ffmpeg(spawned: SpawnedProcess, args: string[]) { diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index dca213c3..ae56df7b 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -1,5 +1,6 @@ // eslint-disable-next-line node/no-unpublished-import import fsMockType from 'windows-network-drive' // Note: this is a mocked module +// import * as Path from 'path' /* eslint-disable no-console */ const DEBUG_LOG = false @@ -9,7 +10,7 @@ enum fsConstants { W_OK = 4, } -const fs = jest.createMockFromModule('fs') as any +const fs: any = jest.createMockFromModule('fs') type MockAny = MockDirectory | MockFile interface MockBase { @@ -34,7 +35,7 @@ const mockRoot: MockDirectory = { content: {}, } const openFileDescriptors: { [fd: string]: MockAny } = {} -let fd = 0 +let fdId = 0 function getMock(path: string, orgPath?: string, dir?: MockDirectory): MockAny { dir = dir || mockRoot @@ -75,7 +76,7 @@ function getMock(path: string, orgPath?: string, dir?: MockDirectory): MockAny { path: path, }) } -function setMock(path: string, create: MockAny, autoCreateTree: boolean, dir?: MockDirectory): void { +function setMock(path: string, create: MockAny, autoCreateTree: boolean, force = false, dir?: MockDirectory): void { dir = dir || mockRoot const m = path.match(/([^/]+)\/(.*)/) @@ -119,10 +120,20 @@ function setMock(path: string, create: MockAny, autoCreateTree: boolean, dir?: M } if (nextDir) { - return setMock(nextPath, create, autoCreateTree, nextDir) + return setMock(nextPath, create, autoCreateTree, force, nextDir) } } else { const fileName = path + if (dir.content[fileName]) { + if (!dir.content[fileName].accessWrite && !force) { + throw Object.assign(new Error(`EACCESS: Not able to write to parent folder "${path}"`), { + errno: 0, + code: 'EACCESS', + syscall: 'mock', + path: path, + }) + } + } dir.content[fileName] = create if (DEBUG_LOG) console.log('setMock', path, create) @@ -175,8 +186,6 @@ export function __printAllFiles(): string { const getPaths = (dir: MockDirectory, indent: string): string => { const strs: any[] = [] for (const [name, file] of Object.entries(dir.content)) { - // const path = `${prevPath}/${name}` - if (file.isDirectory) { strs.push(`${indent}${name}/`) strs.push(getPaths(file, indent + ' ')) @@ -215,41 +224,40 @@ export function __mockReset(): void { } fs.__mockReset = __mockReset -export function __mockSetFile(path: string, size: number): void { +export function __mockSetFile(path: string, size: number, accessOptions?: FileAccess): void { path = fixPath(path) setMock( path, { - accessRead: true, - accessWrite: true, + accessRead: accessOptions?.accessRead ?? true, + accessWrite: accessOptions?.accessWrite ?? true, + isDirectory: false, content: 'mockContent', size: size, }, + true, true ) } fs.__mockSetFile = __mockSetFile -export function __mockSetDirectory(path: string): void { +export function __mockSetDirectory(path: string, accessOptions?: FileAccess): void { path = fixPath(path) setMock( path, { - accessRead: true, - accessWrite: true, + accessRead: accessOptions?.accessRead ?? true, + accessWrite: accessOptions?.accessWrite ?? true, + isDirectory: true, content: {}, }, + true, true ) } fs.__mockSetDirectory = __mockSetDirectory -// export function readdirSync(directoryPath: string) { -// return mockFiles[directoryPath] || [] -// } -// fs.readdirSync = readdirSync - export function stat(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.stat', path) @@ -391,10 +399,10 @@ export function open(path: string, _options: string, callback: (error: any, resu if (DEBUG_LOG) console.log('fs.open', path) try { const file = getMock(path) - fd++ - openFileDescriptors[fd + ''] = file + fdId++ + openFileDescriptors[fdId + ''] = file - return callback(undefined, fd) + return callback(undefined, fdId) } catch (err) { callback(err) } @@ -441,4 +449,9 @@ export function rename(source: string, destination: string, callback: (error: an } fs.rename = rename +interface FileAccess { + accessRead: boolean + accessWrite: boolean +} + module.exports = fs diff --git a/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts index 11a83779..179f305d 100644 --- a/tests/internal-tests/src/__tests__/basic.spec.ts +++ b/tests/internal-tests/src/__tests__/basic.spec.ts @@ -1,14 +1,14 @@ +import fsOrg from 'fs' import { promisify } from 'util' +import WNDOrg from 'windows-network-drive' +import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' +import * as QGatewayClientOrg from 'tv-automation-quantel-gateway-client' import { Expectation, literal } from '@shared/api' -import fsOrg from 'fs' import type * as fsMockType from '../__mocks__/fs' -import WNDOrg from 'windows-network-drive' import type * as WNDType from '../__mocks__/windows-network-drive' -import * as QGatewayClientOrg from 'tv-automation-quantel-gateway-client' import type * as QGatewayClientType from '../__mocks__/tv-automation-quantel-gateway-client' import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' -import { waitSeconds } from './lib/lib' -import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' +import { waitTime } from './lib/lib' import { getFileShareSource, getLocalSource, @@ -27,8 +27,6 @@ const QGatewayClient = (QGatewayClientOrg as any) as typeof QGatewayClientType const fsStat = promisify(fs.stat) -const WAIT_JOB_TIME = 1 // seconds - describe('Basic', () => { let env: TestEnviromnent @@ -46,210 +44,182 @@ describe('Basic', () => { env.reset() QGatewayClient.resetMock() }) - test( - 'Be able to copy local file', - async () => { - fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) - fs.__mockSetDirectory('/targets/target0') - // console.log(fs.__printAllFiles()) - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.FILE_COPY, - statusReport: { - label: `Copy file0`, - description: `Copy file0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getLocalSource('source0', 'file0Source.mp4')], - }, - endRequirement: { - targets: [getLocalTarget('target0', 'file0Target.mp4')], - content: { - filePath: 'file0Target.mp4', - }, - version: { type: Expectation.Version.Type.FILE_ON_DISK }, - }, - workOptions: {}, - }), - }) - - await waitSeconds(WAIT_JOB_TIME) - - // Expect the copy to have completed by now: - - expect(env.containerStatuses['target0']).toBeTruthy() - expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) - - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - - expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ - size: 1234, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test( - 'Be able to copy Networked file to local', - async () => { - fs.__mockSetFile('\\\\networkShare/sources/source1/file0Source.mp4', 1234) - fs.__mockSetDirectory('/targets/target1') - // console.log(fs.__printAllFiles()) - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.FILE_COPY, - statusReport: { - label: `Copy file0`, - description: `Copy file0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getFileShareSource('source1', 'file0Source.mp4')], - }, - endRequirement: { - targets: [getLocalTarget('target1', 'subFolder0/file0Target.mp4')], - content: { - filePath: 'subFolder0/file0Target.mp4', - }, - version: { type: Expectation.Version.Type.FILE_ON_DISK }, - }, - workOptions: {}, - }), - }) - - await waitSeconds(WAIT_JOB_TIME) - - // Expect the copy to have completed by now: - - // console.log(fs.__printAllFiles()) - - expect(env.containerStatuses['target1']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) - - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - - expect(await WND.list()).toEqual({ - X: '\\\\networkShare\\sources\\source1\\', - }) - - expect(await fsStat('/targets/target1/subFolder0/file0Target.mp4')).toMatchObject({ - size: 1234, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test( - 'Be able to copy Quantel clips', - async () => { - const orgClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123')[0] - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.QUANTEL_CLIP_COPY, - statusReport: { - label: `Copy quantel clip0`, - description: `Copy clip0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getQuantelSource('source0')], + test('Be able to copy local file', async () => { + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target0') + // console.log(fs.__printAllFiles()) + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: [getLocalSource('source0', 'file0Source.mp4')], + }, + endRequirement: { + targets: [getLocalTarget('target0', 'file0Target.mp4')], + content: { + filePath: 'file0Target.mp4', }, - endRequirement: { - targets: [getQuantelTarget('target1', 1001)], - content: { - guid: 'abc123', - }, - version: { type: Expectation.Version.Type.QUANTEL_CLIP }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test('Be able to copy Networked file to local', async () => { + fs.__mockSetFile('\\\\networkShare/sources/source1/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target1') + // console.log(fs.__printAllFiles()) + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: [getFileShareSource('source1', 'file0Source.mp4')], + }, + endRequirement: { + targets: [getLocalTarget('target1', 'subFolder0/file0Target.mp4')], + content: { + filePath: 'subFolder0/file0Target.mp4', }, - workOptions: {}, - }), - }) + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) - await waitSeconds(WAIT_JOB_TIME) + await waitTime(env.WAIT_JOB_TIME) - // Expect the copy to have completed by now: + // Expect the copy to have completed by now: - expect(env.containerStatuses['target1']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) + // console.log(fs.__printAllFiles()) - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(env.containerStatuses['target1']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) - const newClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123' && clip !== orgClip.clip)[0] - expect(newClip).toBeTruthy() + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - expect(newClip).toMatchObject({ - server: { - ident: 1001, + expect(await WND.list()).toEqual({ + X: '\\\\networkShare\\sources\\source1\\', + }) + + expect(await fsStat('/targets/target1/subFolder0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test('Be able to copy Quantel clips', async () => { + const orgClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123')[0] + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.QUANTEL_CLIP_COPY, + statusReport: { + label: `Copy quantel clip0`, + description: `Copy clip0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, }, - clip: { - ClipGUID: 'abc123', - CloneId: orgClip.clip.ClipID, + startRequirement: { + sources: [getQuantelSource('source0')], }, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Be able to copy local file to http', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Be able to handle 1000 expectations', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Media file preview from local to file share', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Media file preview from local to file share', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) + endRequirement: { + targets: [getQuantelTarget('target1', 1001)], + content: { + guid: 'abc123', + }, + version: { type: Expectation.Version.Type.QUANTEL_CLIP }, + }, + workOptions: {}, + }), + }) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target1']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + + const newClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123' && clip !== orgClip.clip)[0] + expect(newClip).toBeTruthy() + + expect(newClip).toMatchObject({ + server: { + ident: 1001, + }, + clip: { + ClipGUID: 'abc123', + CloneId: orgClip.clip.ClipID, + }, + }) + }) + test.skip('Be able to copy local file to http', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Be able to handle 1000 expectations', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Media file preview from local to file share', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Media file preview from local to file share', async () => { + // To be written + expect(1).toEqual(1) + }) }) -export {} +export {} // Just to get rid of a "not a module" warning diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts index 6a2a7fc0..4bb6d960 100644 --- a/tests/internal-tests/src/__tests__/issues.spec.ts +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -1,22 +1,32 @@ -// import { promisify } from 'util' -import * as fsOrg from 'fs' -import type * as fsMock from '../__mocks__/fs' +import fsOrg from 'fs' +import { promisify } from 'util' +import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' +import { Expectation, literal } from '@shared/api' +import type * as fsMockType from '../__mocks__/fs' import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' +import { waitTime } from './lib/lib' +import { getLocalSource, getLocalTarget } from './lib/containers' jest.mock('fs') jest.mock('child_process') jest.mock('windows-network-drive') +jest.mock('tv-automation-quantel-gateway-client') -const fs = (fsOrg as any) as typeof fsMock +const fs = (fsOrg as any) as typeof fsMockType -// const fsStat = promisify(fs.stat) +const fsStat = promisify(fs.stat) -const WAIT_JOB_TIME = 1 // seconds +// const fsStat = promisify(fs.stat) describe('Handle unhappy paths', () => { let env: TestEnviromnent beforeAll(async () => { - env = await prepareTestEnviromnent(true) + env = await prepareTestEnviromnent(false) // set to true to enable debug-logging + // Verify that the fs mock works: + expect(fs.lstat).toBeTruthy() + expect(fs.__mockReset).toBeTruthy() + + jest.setTimeout(env.WAIT_JOB_TIME * 10 + env.WAIT_SCAN_TIME * 2) }) afterAll(() => { env.terminate() @@ -26,93 +36,199 @@ describe('Handle unhappy paths', () => { fs.__mockReset() env.reset() }) - test.skip( - 'Wait for non-existing local file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for non-existing network-shared, file', - async () => { - // To be written - - // Handle Drive-letters: Issue when when files aren't found? (Johan knows) - - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for growing file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for non-existing file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for read access on source', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for write access on target', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Expectation changed', - async () => { - // Expectation is changed while waiting for a file - // Expectation is changed while work-in-progress - // Expectation is changed after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Aborting a job', - async () => { - // Expectation is aborted while waiting for a file - // Expectation is aborted while work-in-progress - // Expectation is aborted after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Restarting a job', - async () => { - // Expectation is restarted while waiting for a file - // Expectation is restarted while work-in-progress - // Expectation is restarted after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) + + test('Wait for non-existing local file', async () => { + fs.__mockSetDirectory('/sources/source0/') + fs.__mockSetDirectory('/targets/target0') + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'waiting', + statusReason: { + tech: /not able to access file/i, + }, + }, + }) + + // Now the file suddenly pops up: + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + + await waitTime(env.WAIT_SCAN_TIME) + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test.skip('Wait for non-existing network-shared, file', async () => { + // To be written + + // Handle Drive-letters: Issue when when files aren't found? (Johan knows) + + expect(1).toEqual(1) + }) + test.skip('Wait for growing file', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Wait for non-existing file', async () => { + // To be written + expect(1).toEqual(1) + }) + test('Wait for read access on source', async () => { + // To be written + expect(1).toEqual(1) + + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234, { + accessRead: false, + accessWrite: false, + }) + fs.__mockSetDirectory('/targets/target0', { + accessRead: true, + accessWrite: false, + }) + // fs.__printAllFiles() + + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'waiting', + statusReason: { + tech: /not able to access file/i, + }, + }, + }) + + // Now the file can be read from: + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'waiting', + statusReason: { + tech: /not able to access target/i, + }, + }, + }) + + // Now the target can be written to: + fs.__mockSetDirectory('/targets/target0', { + accessRead: true, + accessWrite: true, + }) + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test.skip('Wait for write access on target', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Expectation changed', async () => { + // Expectation is changed while waiting for a file + // Expectation is changed while work-in-progress + // Expectation is changed after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test.skip('Aborting a job', async () => { + // Expectation is aborted while waiting for a file + // Expectation is aborted while work-in-progress + // Expectation is aborted after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test.skip('Restarting a job', async () => { + // Expectation is restarted while waiting for a file + // Expectation is restarted while work-in-progress + // Expectation is restarted after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test.skip('A worker crashes', async () => { + // A worker crashes while expectation waiting for a file + // A worker crashes while expectation work-in-progress + + // To be written + expect(1).toEqual(1) + }) }) +function addCopyFileExpectation( + env: TestEnviromnent, + expectationId: string, + sources: Expectation.SpecificPackageContainerOnPackage.File[], + targets: [Expectation.SpecificPackageContainerOnPackage.File] +) { + env.expectationManager.updateExpectations({ + copy0: literal({ + id: expectationId, + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: sources, + }, + endRequirement: { + targets: targets, + content: { + filePath: 'file0Target.mp4', + }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) +} -export {} +export {} // Just to get rid of a "not a module" warning diff --git a/tests/internal-tests/src/__tests__/lib/lib.ts b/tests/internal-tests/src/__tests__/lib/lib.ts index 8b80ba92..872de6c3 100644 --- a/tests/internal-tests/src/__tests__/lib/lib.ts +++ b/tests/internal-tests/src/__tests__/lib/lib.ts @@ -1,3 +1,3 @@ -export function waitSeconds(seconds: number) { - return new Promise((resolve) => setTimeout(resolve, seconds * 1000)) +export function waitTime(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) } diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index b565d9e9..da593683 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -5,7 +5,7 @@ import * as Worker from '@shared/worker' import * as Winston from 'winston' import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, Reason, SingleAppConfig } from '@shared/api' // import deepExtend from 'deep-extend' -import { ExpectationManager, ExpectationManagerCallbacks } from '@shared/expectation-manager' +import { ExpectationManager, ExpectationManagerCallbacks, ExpectationManagerOptions } from '@shared/expectation-manager' import { CoreMockAPI } from './coreMockAPI' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' @@ -41,6 +41,7 @@ const defaultTestConfig: SingleAppConfig = { worker: { workerId: 'worker', workforceURL: null, + appContainerURL: null, resourceId: '', networkIds: [], windowsDriveLetters: ['X', 'Y', 'Z'], @@ -53,6 +54,8 @@ const defaultTestConfig: SingleAppConfig = { appContainer: { appContainerId: 'appContainer0', workforceURL: null, + port: 0, + maxRunningApps: 1, resourceId: '', networkIds: [], windowsDriveLetters: ['X', 'Y', 'Z'], @@ -62,11 +65,12 @@ const defaultTestConfig: SingleAppConfig = { export async function setupExpectationManager( debugLogging: boolean, workerCount: number = 1, - callbacks: ExpectationManagerCallbacks + callbacks: ExpectationManagerCallbacks, + options?: ExpectationManagerOptions ) { const logger = new Winston.Logger({}) as LoggerInstance logger.add(Winston.transports.Console, { - level: debugLogging ? 'verbose' : 'warn', + level: debugLogging ? 'debug' : 'warn', }) const expectationManager = new ExpectationManager( @@ -75,7 +79,8 @@ export async function setupExpectationManager( { type: 'internal' }, undefined, { type: 'internal' }, - callbacks + callbacks, + options ) // Initializing HTTP proxy Server: @@ -119,70 +124,94 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise { - if (!expectationStatuses[expectationId]) { - expectationStatuses[expectationId] = { - actualVersionHash: null, - statusInfo: {}, + const WAIT_JOB_TIME = 500 // ms + const WAIT_SCAN_TIME = 1000 // ms + + const em = await setupExpectationManager( + debugLogging, + 1, + { + reportExpectationStatus: ( + expectationId: string, + _expectaction: Expectation.Any | null, + actualVersionHash: string | null, + statusInfo: { + status?: string + progress?: number + statusReason?: Reason } - } - const o = expectationStatuses[expectationId] - if (actualVersionHash) o.actualVersionHash = actualVersionHash - if (statusInfo.status) o.statusInfo.status = statusInfo.status - if (statusInfo.progress) o.statusInfo.progress = statusInfo.progress - if (statusInfo.statusReason) o.statusInfo.statusReason = statusInfo.statusReason - }, - reportPackageContainerPackageStatus: ( - containerId: string, - packageId: string, - packageStatus: Omit | null - ) => { - if (!containerStatuses[containerId]) { - containerStatuses[containerId] = { - packages: {}, + ) => { + if (debugLogging) console.log('reportExpectationStatus', expectationId, actualVersionHash, statusInfo) + + if (!expectationStatuses[expectationId]) { + expectationStatuses[expectationId] = { + actualVersionHash: null, + statusInfo: {}, + } } - } - const container = containerStatuses[containerId] - container.packages[packageId] = { - packageStatus: packageStatus, - } - }, - reportPackageContainerExpectationStatus: () => { - // todo - }, - messageFromWorker: async (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { - switch (message.type) { - case 'fetchPackageInfoMetadata': - return coreApi.fetchPackageInfoMetadata(...message.arguments) - case 'updatePackageInfo': - return coreApi.updatePackageInfo(...message.arguments) - case 'removePackageInfo': - return coreApi.removePackageInfo(...message.arguments) - case 'reportFromMonitorPackages': - return coreApi.reportFromMonitorPackages(...message.arguments) - default: - // @ts-expect-error message.type is never - throw new Error(`Unsupported message type "${message.type}"`) - } + const o = expectationStatuses[expectationId] + if (actualVersionHash) o.actualVersionHash = actualVersionHash + if (statusInfo.status) o.statusInfo.status = statusInfo.status + if (statusInfo.progress) o.statusInfo.progress = statusInfo.progress + if (statusInfo.statusReason) o.statusInfo.statusReason = statusInfo.statusReason + }, + reportPackageContainerPackageStatus: ( + containerId: string, + packageId: string, + packageStatus: Omit | null + ) => { + if (debugLogging) + console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) + if (!containerStatuses[containerId]) { + containerStatuses[containerId] = { + packages: {}, + } + } + const container = containerStatuses[containerId] + container.packages[packageId] = { + packageStatus: packageStatus, + } + }, + reportPackageContainerExpectationStatus: () => { + // todo + // if (debugLogging) console.log('reportPackageContainerExpectationStatus', containerId, packageId, packageStatus) + }, + messageFromWorker: async (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { + switch (message.type) { + case 'fetchPackageInfoMetadata': + return coreApi.fetchPackageInfoMetadata(...message.arguments) + case 'updatePackageInfo': + return coreApi.updatePackageInfo(...message.arguments) + case 'removePackageInfo': + return coreApi.removePackageInfo(...message.arguments) + case 'reportFromMonitorPackages': + return coreApi.reportFromMonitorPackages(...message.arguments) + default: + // @ts-expect-error message.type is never + throw new Error(`Unsupported message type "${message.type}"`) + } + }, }, - }) + { + constants: { + EVALUATE_INTERVAL: WAIT_SCAN_TIME - WAIT_JOB_TIME - 300, + FULLFILLED_MONITOR_TIME: WAIT_SCAN_TIME - WAIT_JOB_TIME - 300, + }, + } + ) return { + WAIT_JOB_TIME, + WAIT_SCAN_TIME, expectationManager: em.expectationManager, coreApi, expectationStatuses, containerStatuses, reset: () => { + if (debugLogging) { + console.log('RESET ENVIRONMENT') + } + em.expectationManager.resetWork() Object.keys(expectationStatuses).forEach((id) => delete expectationStatuses[id]) Object.keys(containerStatuses).forEach((id) => delete containerStatuses[id]) coreApi.reset() @@ -195,6 +224,8 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise Date: Thu, 26 Aug 2021 12:49:28 +0200 Subject: [PATCH 53/67] feat: make expectationManager able to detect when a job has stalled --- .../src/expectationManager.ts | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 958c23b4..e05ac21f 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -83,9 +83,10 @@ export class ExpectationManager { properties: ExpectationManagerWorkerAgent.WorkInProgressProperties trackedExp: TrackedExpectation worker: WorkerAgentAPI + lastUpdated: number } } = {} - private terminating = false + private terminated = false private status: ExpectationManagerStatus @@ -100,11 +101,14 @@ export class ExpectationManager { options?: ExpectationManagerOptions ) { this.constants = { + // Default values: EVALUATE_INTERVAL: 10 * 1000, FULLFILLED_MONITOR_TIME: 10 * 1000, + WORK_TIMEOUT_TIME: 30 * 1000, ALLOW_SKIPPING_QUEUE_TIME: 30 * 1000, SCALE_UP_TIME: 5 * 1000, SCALE_UP_COUNT: 1, + ...options?.constants, } this.workforceAPI = new WorkforceAPI(this.logger) @@ -159,9 +163,7 @@ export class ExpectationManager { serverAccessUrl += `:${this.websocketServer?.port}` } } - if (!serverAccessUrl) throw new Error(`ExpectationManager.serverAccessUrl not set!`) - await this.workforceAPI.registerExpectationManager(this.managerId, serverAccessUrl) this._triggerEvaluateExpectations(true) @@ -357,6 +359,7 @@ export class ExpectationManager { ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { + wip.lastUpdated = Date.now() if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash this.updateTrackedExpStatus( @@ -389,6 +392,7 @@ export class ExpectationManager { wipEventError: async (wipId: number, reason: Reason): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { + wip.lastUpdated = Date.now() if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.errorCount++ this.updateTrackedExpStatus( @@ -428,6 +432,8 @@ export class ExpectationManager { // Iterate through the PackageContainerExpectations: await this._evaluateAllTrackedPackageContainers() + this.monitorWorksInProgress() + // Iterate through all Expectations: const runAgainASAP = await this._evaluateAllExpectations() @@ -811,6 +817,7 @@ export class ExpectationManager { properties: wipInfo.properties, trackedExp: trackedExp, worker: assignedWorker.worker, + lastUpdated: Date.now(), } this.updateTrackedExpStatus( @@ -951,7 +958,7 @@ export class ExpectationManager { assertNever(trackedExp.state) } } catch (err) { - this.logger.error('Error thrown in evaluateExpectationState') + this.logger.error(`Error thrown in evaluateExpectationState for expectation "${trackedExp.id}"`) this.logger.error(err) this.updateTrackedExpStatus(trackedExp, undefined, { user: 'Internal error in Package Manager', @@ -1445,6 +1452,30 @@ export class ExpectationManager { await this.workforceAPI.requestResources(exp.exp) } } + /** */ + private monitorWorksInProgress() { + for (const [wipId, wip] of Object.entries(this.worksInProgress)) { + if ( + wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING && + Date.now() - wip.lastUpdated > this.constants.WORK_TIMEOUT_TIME + ) { + // It seems that the work has stalled.. + // Restart the job: + const reason: Reason = { + tech: 'WorkInProgress timeout', + user: 'The job timed out', + } + + wip.trackedExp.errorCount++ + this.updateTrackedExpStatus(wip.trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, reason) + this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { + status: wip.trackedExp.state, + statusReason: wip.trackedExp.reason, + }) + delete this.worksInProgress[wipId] + } + } + } } export interface ExpectationManagerOptions { constants: Partial @@ -1460,6 +1491,9 @@ export interface ExpectationManagerConstants { */ ALLOW_SKIPPING_QUEUE_TIME: number + /** If there has been no updated on a work-in-progress, time it out after this time */ + WORK_TIMEOUT_TIME: number + /** How long to wait before requesting more resources (workers) [ms] */ SCALE_UP_TIME: number /** How many resources to request at a time */ From e7d40a138daed6916911d5efc44907360d083a79 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 26 Aug 2021 12:50:57 +0200 Subject: [PATCH 54/67] chore: tests and piping for testing when a worker crashes --- shared/packages/api/src/adapterClient.ts | 6 ++- shared/packages/api/src/adapterServer.ts | 9 +++- shared/packages/api/src/lib.ts | 13 +++++ .../src/expectationManager.ts | 41 ++++++++++---- shared/packages/worker/src/workerAgent.ts | 37 ++++++++++--- tests/internal-tests/src/__mocks__/fs.ts | 20 +++++++ .../src/__tests__/issues.spec.ts | 54 +++++++++++++++++-- .../src/__tests__/lib/setupEnv.ts | 38 ++++++++++++- 8 files changed, 192 insertions(+), 26 deletions(-) diff --git a/shared/packages/api/src/adapterClient.ts b/shared/packages/api/src/adapterClient.ts index 3fa6fe3b..bdd4f86c 100644 --- a/shared/packages/api/src/adapterClient.ts +++ b/shared/packages/api/src/adapterClient.ts @@ -18,6 +18,7 @@ export abstract class AdapterClient { constructor(protected logger: LoggerInstance, private clientType: MessageIdentifyClient['clientType']) {} private conn?: WebsocketClient + private terminated = false async init(id: string, connectionOptions: ClientConnectionOptions, clientMethods: ME): Promise { if (connectionOptions.type === 'websocket') { @@ -48,12 +49,13 @@ export abstract class AdapterClient { await conn.connect() } else { - // TODO if (!this.serverHook) throw new Error(`AdapterClient: can't init() an internal connection, call hook() first!`) const serverHook: OTHER = this.serverHook(id, clientMethods) this._sendMessage = (type: keyof OTHER, ...args: any[]) => { + if (this.terminated) throw new Error(`Can't send message due to being terminated`) + const fcn = serverHook[type] as any if (fcn) { return fcn(...args) @@ -68,7 +70,9 @@ export abstract class AdapterClient { this.serverHook = serverHook } terminate(): void { + this.terminated = true this.conn?.close() + delete this.serverHook } } /** Options for an AdepterClient */ diff --git a/shared/packages/api/src/adapterServer.ts b/shared/packages/api/src/adapterServer.ts index 53824c5d..c62b5d64 100644 --- a/shared/packages/api/src/adapterServer.ts +++ b/shared/packages/api/src/adapterServer.ts @@ -1,4 +1,5 @@ -import { MessageBase } from './websocketConnection' +import { promiseTimeout } from './lib' +import { MessageBase, MESSAGE_TIMEOUT } from './websocketConnection' import { ClientConnection } from './websocketServer' /** @@ -9,7 +10,11 @@ import { ClientConnection } from './websocketServer' export abstract class AdapterServer { protected _sendMessage: (type: keyof OTHER, ...args: any[]) => Promise + public readonly type: string + constructor(serverMethods: ME, options: AdapterServerOptions) { + this.type = options.type + if (options.type === 'websocket') { this._sendMessage = ((type: string, ...args: any[]) => options.clientConnection.send(type, ...args)) as any @@ -27,7 +32,7 @@ export abstract class AdapterServer { this._sendMessage = (type: keyof OTHER, ...args: any[]) => { const fcn = (clientHook[type] as unknown) as (...args: any[]) => any if (fcn) { - return fcn(...args) + return promiseTimeout(fcn(...args), MESSAGE_TIMEOUT) } else { throw new Error(`Unknown method "${type}"`) } diff --git a/shared/packages/api/src/lib.ts b/shared/packages/api/src/lib.ts index ec5ff31b..5b2d4029 100644 --- a/shared/packages/api/src/lib.ts +++ b/shared/packages/api/src/lib.ts @@ -49,3 +49,16 @@ export function waitTime(duration: number): Promise { setTimeout(resolve, duration) }) } +export function promiseTimeout(p: Promise, timeoutTime: number): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject('Timeout') + }, timeoutTime) + + p.then(resolve) + .catch(reject) + .finally(() => { + clearTimeout(timeout) + }) + }) +} diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index e05ac21f..98f5b2c8 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -175,10 +175,14 @@ export class ExpectationManager { this.workforceAPI.hook(hook) } terminate(): void { - this.terminating = true + this.terminated = true if (this.websocketServer) { this.websocketServer.terminate() } + if (this._evaluateExpectationsTimeout) { + clearTimeout(this._evaluateExpectationsTimeout) + this._evaluateExpectationsTimeout = undefined + } } /** USED IN TESTS ONLY. Quickly reset the tracked work of the expectationManager. */ resetWork(): void { @@ -216,6 +220,15 @@ export class ExpectationManager { return workerAgentMethods } } + removeWorkerAgentHook(clientId: string): void { + const workerAgent = this.workerAgents[clientId] + if (!workerAgent) throw new Error(`WorkerAgent "${clientId}" not found!`) + + if (workerAgent.api.type !== 'internal') + throw new Error(`Cannot remove WorkerAgent "${clientId}", due to the type being "${workerAgent.api.type}"`) + + delete this.workerAgents[clientId] + } /** Called when there is an updated set of PackageContainerExpectations. */ updatePackageContainerExpectations(packageContainers: { [id: string]: PackageContainerExpectation }): void { // We store the incoming expectations here, so that we don't modify anything in the middle of the _evaluateExpectations() iteration loop. @@ -281,6 +294,8 @@ export class ExpectationManager { * @param asap If true, will re-schedule evaluateExpectations() to run as soon as possible */ private _triggerEvaluateExpectations(asap?: boolean): void { + if (this.terminated) return + if (asap) this._evaluateExpectationsRunAsap = true if (this._evaluateExpectationsIsBusy) return @@ -289,11 +304,9 @@ export class ExpectationManager { this._evaluateExpectationsTimeout = undefined } - if (this.terminating) return - this._evaluateExpectationsTimeout = setTimeout( () => { - if (this.terminating) return + if (this.terminated) return this._evaluateExpectationsRunAsap = false this._evaluateExpectationsIsBusy = true @@ -328,6 +341,7 @@ export class ExpectationManager { ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { + wip.lastUpdated = Date.now() if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash wip.trackedExp.status.workProgress = progress @@ -713,12 +727,21 @@ export class ExpectationManager { } await Promise.all( Object.entries(this.workerAgents).map(async ([id, workerAgent]) => { - const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) + try { + const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) - if (support.support) { - trackedExp.availableWorkers.push(id) - } else { - notSupportReason = support.reason + if (support.support) { + trackedExp.availableWorkers.push(id) + } else { + notSupportReason = support.reason + } + } catch (err) { + if ((err + '').match(/timeout/i)) { + notSupportReason = { + user: 'Worker timed out', + tech: `Worker "${id} timeout"`, + } + } else throw err } }) ) diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 3682ad88..0a8babc5 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -37,7 +37,7 @@ export class WorkerAgent { private wipI = 0 private worksInProgress: { [wipId: string]: IWorkInProgress } = {} - private id: string + public readonly id: string private workForceConnectionOptions: ClientConnectionOptions private appContainerConnectionOptions: ClientConnectionOptions | null @@ -53,6 +53,7 @@ export class WorkerAgent { ExpectationManagerWorkerAgent.WorkerAgent > } = {} + private terminated = false constructor(private logger: LoggerInstance, private config: WorkerConfig) { this.workforceAPI = new WorkforceAPI(this.logger) @@ -113,6 +114,7 @@ export class WorkerAgent { await this._worker.init() } terminate(): void { + this.terminated = true this.workforceAPI.terminate() Object.values(this.expectationManagers).forEach((expectationManager) => expectationManager.api.terminate()) // this._worker.terminate() @@ -222,8 +224,10 @@ export class WorkerAgent { workInProgress.on('progress', (actualVersionHash, progress: number) => { currentjob.progress = progress expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch((err) => { - this.logger.error('Error in wipEventProgress') - this.logger.error(err) + if (!this.terminated) { + this.logger.error('Error in wipEventProgress') + this.logger.error(err) + } }) }) workInProgress.on('error', (error: string) => { @@ -238,8 +242,10 @@ export class WorkerAgent { tech: error, }) .catch((err) => { - this.logger.error('Error in wipEventError') - this.logger.error(err) + if (!this.terminated) { + this.logger.error('Error in wipEventError') + this.logger.error(err) + } }) delete this.worksInProgress[`${wipId}`] }) @@ -250,8 +256,10 @@ export class WorkerAgent { ) expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch((err) => { - this.logger.error('Error in wipEventDone') - this.logger.error(err) + if (!this.terminated) { + this.logger.error('Error in wipEventDone') + this.logger.error(err) + } }) delete this.worksInProgress[`${wipId}`] }) @@ -302,8 +310,21 @@ export class WorkerAgent { return this._worker.disposePackageContainerMonitors(packageContainer) }, }) + // Wrap the methods, so that we can cut off communication upon termination: (this is used in tests) + for (const key of Object.keys(methods) as Array) { + const fcn = methods[key] as any + methods[key] = ((...args: any[]) => { + if (this.terminated) + return new Promise((_resolve, reject) => { + // Simulate a timed out message: + setTimeout(() => { + reject('Timeout') + }, 200) + }) + return fcn(...args) + }) as any + } // Connect to the ExpectationManager: - if (url === '__internal') { // This is used for an internal connection: const managerHookHook = this.expectationManagerHooks[id] diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index ae56df7b..9b7e266c 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -1,5 +1,6 @@ // eslint-disable-next-line node/no-unpublished-import import fsMockType from 'windows-network-drive' // Note: this is a mocked module +import { EventEmitter } from 'events' // Note: this is a mocked module // import * as Path from 'path' /* eslint-disable no-console */ @@ -36,6 +37,7 @@ const mockRoot: MockDirectory = { } const openFileDescriptors: { [fd: string]: MockAny } = {} let fdId = 0 +const fsMockEmitter = new EventEmitter() function getMock(path: string, orgPath?: string, dir?: MockDirectory): MockAny { dir = dir || mockRoot @@ -221,6 +223,7 @@ function fixPath(path: string) { export function __mockReset(): void { Object.keys(mockRoot.content).forEach((filePath) => delete mockRoot.content[filePath]) + fsMockEmitter.removeAllListeners() } fs.__mockReset = __mockReset @@ -258,9 +261,15 @@ export function __mockSetDirectory(path: string, accessOptions?: FileAccess): vo } fs.__mockSetDirectory = __mockSetDirectory +export function __emitter(): EventEmitter { + return fsMockEmitter +} +fs.__emitter = __emitter + export function stat(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.stat', path) + fsMockEmitter.emit('stat', path) try { const mockFile = getMock(path) if (mockFile.isDirectory) { @@ -281,6 +290,7 @@ fs.stat = stat export function access(path: string, mode: number | undefined, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.access', path, mode) + fsMockEmitter.emit('access', path, mode) try { const mockFile = getMock(path) if (mode === fsConstants.R_OK && !mockFile.accessRead) { @@ -299,6 +309,7 @@ fs.access = access export function unlink(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.unlink', path) + fsMockEmitter.emit('unlink', path) try { deleteMock(path) return callback(undefined, null) @@ -311,6 +322,7 @@ fs.unlink = unlink export function mkdir(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.mkdir', path) + fsMockEmitter.emit('mkdir', path) try { setMock( path, @@ -333,6 +345,7 @@ fs.mkdir = mkdir export function readdir(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.readdir', path) + fsMockEmitter.emit('readdir', path) try { const mockFile = getMock(path) if (!mockFile.isDirectory) { @@ -349,6 +362,7 @@ fs.readdir = readdir export function lstat(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.lstat', path) + fsMockEmitter.emit('lstat', path) try { const mockFile = getMock(path) return callback(undefined, { @@ -364,6 +378,7 @@ fs.lstat = lstat export function writeFile(path: string, data: Buffer | string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.writeFile', path) + fsMockEmitter.emit('writeFile', path, data) try { setMock( path, @@ -385,6 +400,7 @@ fs.writeFile = writeFile function readFile(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.readFile', path) + fsMockEmitter.emit('readFile', path) try { const file = getMock(path) return callback(undefined, file.content) @@ -397,6 +413,7 @@ fs.readFile = readFile export function open(path: string, _options: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.open', path) + fsMockEmitter.emit('open', path) try { const file = getMock(path) fdId++ @@ -410,6 +427,7 @@ export function open(path: string, _options: string, callback: (error: any, resu fs.open = open export function close(fd: number, callback: (error: any, result?: any) => void): void { if (DEBUG_LOG) console.log('fs.close') + fsMockEmitter.emit('close', fd) if (openFileDescriptors[fd + '']) { delete openFileDescriptors[fd + ''] return callback(undefined, null) @@ -422,6 +440,7 @@ export function copyFile(source: string, destination: string, callback: (error: source = fixPath(source) destination = fixPath(destination) if (DEBUG_LOG) console.log('fs.copyFile', source, destination) + fsMockEmitter.emit('copyFile', source, destination) try { const mockFile = getMock(source) if (DEBUG_LOG) console.log('source', source) @@ -438,6 +457,7 @@ export function rename(source: string, destination: string, callback: (error: an source = fixPath(source) destination = fixPath(destination) if (DEBUG_LOG) console.log('fs.rename', source, destination) + fsMockEmitter.emit('rename', source, destination) try { const mockFile = getMock(source) setMock(destination, mockFile, false) diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts index 4bb6d960..b30c8688 100644 --- a/tests/internal-tests/src/__tests__/issues.spec.ts +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -6,6 +6,7 @@ import type * as fsMockType from '../__mocks__/fs' import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' import { waitTime } from './lib/lib' import { getLocalSource, getLocalTarget } from './lib/containers' +import { WorkerAgent } from '@shared/worker' jest.mock('fs') jest.mock('child_process') jest.mock('windows-network-drive') @@ -94,9 +95,6 @@ describe('Handle unhappy paths', () => { expect(1).toEqual(1) }) test('Wait for read access on source', async () => { - // To be written - expect(1).toEqual(1) - fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234, { accessRead: false, accessWrite: false, @@ -188,10 +186,58 @@ describe('Handle unhappy paths', () => { // To be written expect(1).toEqual(1) }) - test.skip('A worker crashes', async () => { + test('A worker crashes', async () => { // A worker crashes while expectation waiting for a file // A worker crashes while expectation work-in-progress + expect(env.workerAgents).toHaveLength(1) + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target0') + let killedWorker: WorkerAgent | undefined + const listenToCopyFile = jest.fn(() => { + // While the copy is underway, kill off the worker: + // This simulates that the worker crashes, without telling anyone. + killedWorker = env.workerAgents[0] + killedWorker.terminate() + }) + + fs.__emitter().once('copyFile', listenToCopyFile) + + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('working') + expect(listenToCopyFile).toHaveBeenCalledTimes(1) + + await waitTime(env.WORK_TIMEOUT_TIME) + await waitTime(env.WAIT_JOB_TIME) + // By now, the work should have been aborted, and restarted: + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('new') + + // Add another worker: + env.addWorker() + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the copy to have completed by now: + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + // Clean up: + if (killedWorker) env.removeWorker(killedWorker.id) + }) + test.skip('One of the workers reply very slowly', async () => { + // The expectation should be picked up by one of the faster workers + // To be written expect(1).toEqual(1) }) diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index da593683..97d352ed 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -97,12 +97,14 @@ export async function setupExpectationManager( // Initialize workers: const workerAgents: Worker.WorkerAgent[] = [] - for (let i = 0; i < workerCount; i++) { + let workerI = 0 + const addWorker = async () => { + const workerId = defaultTestConfig.worker.workerId + '_' + workerI++ const workerAgent = new Worker.WorkerAgent(logger, { ...defaultTestConfig, worker: { ...defaultTestConfig.worker, - workerId: defaultTestConfig.worker.workerId + '_' + i, + workerId: workerId, }, }) workerAgents.push(workerAgent) @@ -110,12 +112,32 @@ export async function setupExpectationManager( workerAgent.hookToWorkforce(workforce.getWorkerAgentHook()) workerAgent.hookToExpectationManager(expectationManager.managerId, expectationManager.getWorkerAgentHook()) await workerAgent.init() + + return workerId + } + const removeWorker = async (workerId: string) => { + const index = workerAgents.findIndex((wa) => wa.id === workerId) + if (index !== -1) { + const workerAgent = workerAgents[index] + + expectationManager.removeWorkerAgentHook(workerAgent.id) + + workerAgent.terminate() + // Remove from array: + workerAgents.splice(index, 1) + } + } + + for (let i = 0; i < workerCount; i++) { + await addWorker() } return { workforce, workerAgents, expectationManager, + addWorker, + removeWorker, } } @@ -126,6 +148,7 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise workerAgent.terminate()) }, + addWorker: em.addWorker, + removeWorker: em.removeWorker, } } export interface TestEnviromnent { WAIT_JOB_TIME: number WAIT_SCAN_TIME: number + WORK_TIMEOUT_TIME: number expectationManager: ExpectationManager + workerAgents: Worker.WorkerAgent[] + workforce: Workforce.Workforce coreApi: CoreMockAPI expectationStatuses: ExpectationStatuses containerStatuses: ContainerStatuses reset: () => void terminate: () => void + addWorker: () => Promise + removeWorker: (id: string) => Promise } export interface ExpectationStatuses { From 7e6b7142a11f457e718906e2b1b95992aa2dcdad Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 10:22:36 +0200 Subject: [PATCH 55/67] chore: clean up in logging --- .../packages/generic/src/packageManager.ts | 10 +- scripts/gather-all-built.js | 31 ++-- shared/packages/api/src/logger.ts | 2 +- .../packages/workforce/src/workerHandler.ts | 143 ++---------------- 4 files changed, 36 insertions(+), 150 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index cf145c6d..9d457fc3 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -249,8 +249,8 @@ export class PackageManagerHandler { } } - this.logger.info(`Has ${expectedPackages.length} expectedPackages`) - // this.logger.info(JSON.stringify(expectedPackages, null, 2)) + this.logger.debug(`Has ${expectedPackages.length} expectedPackages`) + // this.logger.debug(JSON.stringify(expectedPackages, null, 2)) this.dataSnapshot.expectedPackages = expectedPackages this.dataSnapshot.packageContainers = this.packageContainersCache @@ -265,8 +265,8 @@ export class PackageManagerHandler { expectedPackages, this.settings ) - this.logger.info(`Has ${Object.keys(expectations).length} expectations`) - // this.logger.info(JSON.stringify(expectations, null, 2)) + this.logger.debug(`Has ${Object.keys(expectations).length} expectations`) + // this.logger.debug(JSON.stringify(expectations, null, 2)) this.dataSnapshot.expectations = expectations const packageContainerExpectations = generatePackageContainerExpectations( @@ -274,7 +274,7 @@ export class PackageManagerHandler { this.packageContainersCache, activePlaylist ) - this.logger.info(`Has ${Object.keys(packageContainerExpectations).length} packageContainerExpectations`) + this.logger.debug(`Has ${Object.keys(packageContainerExpectations).length} packageContainerExpectations`) ;(this.dataSnapshot.packageContainerExpectations = packageContainerExpectations), (this.dataSnapshot.updated = Date.now()) diff --git a/scripts/gather-all-built.js b/scripts/gather-all-built.js index 1cb6383c..1763ecc2 100644 --- a/scripts/gather-all-built.js +++ b/scripts/gather-all-built.js @@ -2,9 +2,12 @@ const promisify = require("util").promisify const glob = promisify(require("glob")) const fse = require('fs-extra'); const mkdirp = require('mkdirp'); -const rimraf = promisify(require('rimraf')) +const path = require('path'); +// const rimraf = promisify(require('rimraf')) const fseCopy = promisify(fse.copy) +const fseReaddir = promisify(fse.readdir) +const fseUnlink = promisify(fse.unlink) /* This script gathers all files in the deploy/ folders of the various apps @@ -15,20 +18,26 @@ const targetFolder = 'deploy/' ;(async function () { - await rimraf(targetFolder) - await mkdirp(targetFolder) + // await rimraf(targetFolder) + await mkdirp(targetFolder) + // clear folder: + const files = await fseReaddir(targetFolder) + for (const file of files) { + if (!file.match(/ffmpeg|ffprobe/)) { + await fseUnlink(path.join(targetFolder, file)) + } + } - const deployfolders = await glob(`apps/*/app/deploy`) + const deployfolders = await glob(`apps/*/app/deploy`) + for (const deployfolder of deployfolders) { - for (const deployfolder of deployfolders) { + if (deployfolder.match(/boilerplate/)) continue - if (deployfolder.match(/boilerplate/)) continue + console.log(`Copying: ${deployfolder}`) + await fseCopy(deployfolder, targetFolder) + } - console.log(`Copying: ${deployfolder}`) - await fseCopy(deployfolder, targetFolder) - } - - console.log(`All files have been copied to: ${targetFolder}`) + console.log(`All files have been copied to: ${targetFolder}`) })().catch(console.error) diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts index 482c87a3..51995eb5 100644 --- a/shared/packages/api/src/logger.ts +++ b/shared/packages/api/src/logger.ts @@ -43,7 +43,7 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance json: true, stringify: (obj) => { obj.localTimestamp = getCurrentTime() - obj.randomId = Math.round(Math.random() * 10000) + // obj.randomId = Math.round(Math.random() * 10000) return JSON.stringify(obj) // make single line }, }) diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts index 1b30687a..d0755eed 100644 --- a/shared/packages/workforce/src/workerHandler.ts +++ b/shared/packages/workforce/src/workerHandler.ts @@ -1,30 +1,21 @@ -import { Expectation } from '@shared/api' +import { Expectation, LoggerInstance } from '@shared/api' import { Workforce } from './workforce' -// const UPDATE_INTERVAL = 10 * 1000 - /** The WorkerHandler is in charge of spinning up/down Workers */ export class WorkerHandler { - // private updateTimeout: NodeJS.Timer | null = null - // private updateAgain = false - // private updateInterval: NodeJS.Timeout - // private terminated = false - - // private plannedWorkers: PlannedWorker[] = [] - // private workerCount = 0 + private logger: LoggerInstance constructor(private workForce: Workforce) { - // this.updateInterval = setInterval(() => { - // this.triggerUpdate() - // }, UPDATE_INTERVAL) + this.logger = workForce.logger } public terminate(): void { - // clearInterval(this.updateInterval) - // this.terminated = true + // nothing? } public async requestResources(exp: Expectation.Any): Promise { let best: { appContainerId: string; appType: string; cost: number } | null = null + this.logger.debug(`Workforce: Got request for resources for exp "${exp.id}"`) for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + this.logger.debug(`Workforce: Asking appContainer "${appContainerId}"`) const proposal = await appContainer.api.requestAppTypeForExpectation(exp) if (proposal) { if (!best || proposal.cost < best.cost) { @@ -37,132 +28,18 @@ export class WorkerHandler { } } if (best) { + this.logger.debug(`Workforce: Selecting appContainer "${best.appContainerId}"`) + const appContainer = this.workForce.appContainers[best.appContainerId] if (!appContainer) throw new Error(`WorkerHandler: AppContainer "${best.appContainerId}" not found`) - this.workForce.logger.info( - `Workforce: Spinning up another worker (${best.appType}) on "${best.appContainerId}"` - ) - - // const newPlannedWorker: PlannedWorker = { - // appContainerId: best.appContainerId, - // appType: best.appType, - // } - // this.plannedWorkers.push(newPlannedWorker) + this.logger.debug(`Workforce: Spinning up another worker (${best.appType}) on "${best.appContainerId}"`) await appContainer.api.spinUp(best.appType) - // newPlannedWorker.appId = appId return true } else { + this.logger.debug(`Workforce: No resources available`) return false } } - // public triggerUpdate(): void { - // if (this.terminated) return - - // if (!this.updateTimeout) { - // this.updateAgain = false - // this.updateTimeout = setTimeout(() => { - // this.update() - // .catch((error) => { - // this.workForce.logger.error(error) - // }) - // .finally(() => { - // this.updateTimeout = null - // if (this.updateAgain) this.triggerUpdate() - // }) - // }, 500) - // } else { - // this.updateAgain = true - // } - // } - // private async update(): Promise { - // // Update this.plannedWorkers - // for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { - // for (const runningApp of appContainer.runningApps) { - // const plannedWorker = this.plannedWorkers.find((pw) => pw.appId === runningApp.appId) - // if (!plannedWorker) { - // this.plannedWorkers.push({ - // appContainerId: appContainerId, - // appType: runningApp.appType, - // appId: runningApp.appId, - // isInUse: false, - // }) - // } - // } - // } - - // // This is a temporary stupid implementation, - // // to be reworked later.. - // const needs: AppTarget[] = [] - // for (let i = 0; i < this.workerCount; i++) { - // needs.push({ - // appType: 'worker', - // }) - // } - // // Reset plannedWorkers: - // for (const plannedWorker of this.plannedWorkers) { - // plannedWorker.isInUse = false - // } - - // // Initial check to see which needs are already fulfilled: - // for (const need of needs) { - // // Do we have anything that fullfills the need? - // for (const plannedWorker of this.plannedWorkers) { - // if (plannedWorker.isInUse) continue - - // if (plannedWorker.appType === need.appType) { - // // ^ Later, we'll add more checks here ^ - // need.fulfilled = true - // plannedWorker.isInUse = true - // break - // } - // } - // } - // for (const need of needs) { - // if (need.fulfilled) continue - - // // See which AppContainers can fullfill our need: - // let found = false - // for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { - // if (found) break - // if (!appContainer.initialized) continue - - // for (const availableApp of appContainer.availableApps) { - // if (availableApp.appType === need.appType) { - // // Spin up that worker: - - // this.workForce.logger.info( - // `Workforce: Spinning up another worker (${availableApp.appType}) on "${appContainerId}"` - // ) - - // const newPlannedWorker: PlannedWorker = { - // appContainerId: appContainerId, - // appType: availableApp.appType, - // isInUse: true, - // } - // this.plannedWorkers.push(newPlannedWorker) - - // const appId = await appContainer.api.spinUp(availableApp.appType) - - // newPlannedWorker.appId = appId - - // found = true - // break - // } - // } - // } - // } - // } } -// interface PlannedWorker { -// appType: string -// appContainerId: string -// appId?: string - -// isInUse: boolean -// } -// interface AppTarget { -// appType: string -// fulfilled?: boolean -// } From f2746f953182599928c4af421d0d2477bc312111 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 10:24:20 +0200 Subject: [PATCH 56/67] fix: better handling of checking for ffmpeg-processes --- shared/packages/worker/src/worker/worker.ts | 3 ++ .../worker/workers/linuxWorker/linuxWorker.ts | 3 ++ .../expectationHandlers/lib/ffmpeg.ts | 26 ++++++++++------- .../expectationHandlers/mediaFilePreview.ts | 6 ++-- .../expectationHandlers/mediaFileThumbnail.ts | 6 ++-- .../expectationHandlers/packageDeepScan.ts | 6 ++-- .../expectationHandlers/packageScan.ts | 6 ++-- .../expectationHandlers/quantelClipPreview.ts | 6 ++-- .../quantelClipThumbnail.ts | 6 ++-- .../workers/windowsWorker/windowsWorker.ts | 29 +++++++++++++++---- shared/packages/worker/src/workerAgent.ts | 4 ++- 11 files changed, 66 insertions(+), 35 deletions(-) diff --git a/shared/packages/worker/src/worker/worker.ts b/shared/packages/worker/src/worker/worker.ts index 4ee73017..249848a5 100644 --- a/shared/packages/worker/src/worker/worker.ts +++ b/shared/packages/worker/src/worker/worker.ts @@ -38,6 +38,9 @@ export abstract class GenericWorker { } /** Called upon startup */ abstract init(): Promise + + /** Called upon termination */ + abstract terminate(): void /** * Does the worker support this expectation? * This includes things like: diff --git a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts index 25071ed6..d98d44c7 100644 --- a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts +++ b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts @@ -38,6 +38,9 @@ export class LinuxWorker extends GenericWorker { async init(): Promise { throw new Error(`Not implemented yet`) } + terminate(): void { + throw new Error(`Not implemented yet`) + } getCostFortExpectation(_exp: Expectation.Any): Promise { throw new Error(`Not implemented yet`) } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts index f1579bc2..7c4ca56a 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts @@ -13,16 +13,16 @@ import { WorkInProgress } from '../../../../lib/workInProgress' export interface FFMpegProcess { cancel: () => void } -/** Check if FFMpeg is available */ -export function hasFFMpeg(): Promise { - return hasFFExecutable(process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg') +/** Check if FFMpeg is available, returns null if no error found */ +export function testFFMpeg(): Promise { + return testFFExecutable(process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg') } /** Check if FFProbe is available */ -export function hasFFProbe(): Promise { - return hasFFExecutable(process.platform === 'win32' ? 'ffprobe.exe' : 'ffprobe') +export function testFFProbe(): Promise { + return testFFExecutable(process.platform === 'win32' ? 'ffprobe.exe' : 'ffprobe') } -export function hasFFExecutable(ffExecutable: string): Promise { - return new Promise((resolve, reject) => { +export function testFFExecutable(ffExecutable: string): Promise { + return new Promise((resolve) => { const ffMpegProcess: ChildProcess = spawn(ffExecutable, ['-version'], { shell: true, }) @@ -36,15 +36,19 @@ export function hasFFExecutable(ffExecutable: string): Promise { output += str }) ffMpegProcess.on('error', (err) => { - reject(err) + resolve(`Process ${ffExecutable} emitted error: ${err}`) }) ffMpegProcess.on('close', (code) => { const m = output.match(/version ([\w-]+)/) // version N-102494-g2899fb61d2 - if (code === 0 && m) { - resolve(m[1]) + if (code === 0) { + if (m) { + resolve(null) + } else { + resolve(`Process ${ffExecutable} bad version: ${output}`) + } } else { - reject(null) + resolve(`Process ${ffExecutable} exited with code ${code}`) } }) }) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index dcbc4270..a72da83c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -30,12 +30,12 @@ export const MediaFilePreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index 15e41341..2f94204d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -36,12 +36,12 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts index 6211e6bc..fbfa456d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts @@ -35,12 +35,12 @@ export const PackageDeepScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts index 0e9e264c..f1f89e5f 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts @@ -33,12 +33,12 @@ export const PackageScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index c3404e9e..92a1ad8f 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -29,12 +29,12 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index 260fef2d..b81c22a7 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -35,12 +35,12 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) + if (windowsWorker.testFFMpeg) return { support: false, reason: { - user: 'There is an issue with the Worker: FFMpeg not found', - tech: 'Cannot access FFMpeg executable', + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, }, } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts index 914dddb8..8245dc1c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts @@ -27,15 +27,19 @@ import { QuantelClipCopy } from './expectationHandlers/quantelClipCopy' import * as PackageContainerExpHandler from './packageContainerExpectationHandler' import { QuantelClipPreview } from './expectationHandlers/quantelClipPreview' import { QuantelThumbnail } from './expectationHandlers/quantelClipThumbnail' -import { hasFFMpeg, hasFFProbe } from './expectationHandlers/lib/ffmpeg' +import { testFFMpeg, testFFProbe } from './expectationHandlers/lib/ffmpeg' import { JsonDataCopy } from './expectationHandlers/jsonDataCopy' /** This is a type of worker that runs on a windows machine */ export class WindowsWorker extends GenericWorker { static readonly type = 'windowsWorker' - public hasFFMpeg = false - public hasFFProbe = false + /** null = all is well */ + public testFFMpeg: null | string = 'Not initialized' + /** null = all is well */ + public testFFProbe: null | string = 'Not initialized' + + private monitor: NodeJS.Timer | undefined constructor( logger: LoggerInstance, @@ -49,8 +53,23 @@ export class WindowsWorker extends GenericWorker { return this.getExpectationHandler(exp).doYouSupportExpectation(exp, this, this) } async init(): Promise { - this.hasFFMpeg = !!(await hasFFMpeg()) - this.hasFFProbe = !!(await hasFFProbe()) + await this.checkExecutables() + this.monitor = setInterval(() => { + this.checkExecutables().catch((err) => { + this.logger.error('Error in checkExecutables') + this.logger.error(err) + }) + }, 10 * 1000) + } + terminate(): void { + if (this.monitor) { + clearInterval(this.monitor) + delete this.monitor + } + } + private async checkExecutables() { + this.testFFMpeg = await testFFMpeg() + this.testFFProbe = await testFFProbe() } getCostFortExpectation(exp: Expectation.Any): Promise { return this.getExpectationHandler(exp).getCostForExpectation(exp, this, this) diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 0a8babc5..e426fe9c 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -94,6 +94,8 @@ export class WorkerAgent { ) } async init(): Promise { + await this._worker.init() + // Connect to WorkForce: if (this.workForceConnectionOptions.type === 'websocket') { this.logger.info(`Worker: Connecting to Workforce at "${this.workForceConnectionOptions.url}"`) @@ -111,13 +113,13 @@ export class WorkerAgent { const list = await this.workforceAPI.getExpectationManagerList() await this.updateListOfExpectationManagers(list) - await this._worker.init() } terminate(): void { this.terminated = true this.workforceAPI.terminate() Object.values(this.expectationManagers).forEach((expectationManager) => expectationManager.api.terminate()) // this._worker.terminate() + this._worker.terminate() } /** Called when running in the same-process-mode, it */ hookToWorkforce(hook: Hook): void { From c6195b068f57fffb78cac770cf0592276cd4eee8 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 10:27:26 +0200 Subject: [PATCH 57/67] fix: ExpectationManager: improvements on expectation-handling in regards to workers are being added and removed dynamically --- apps/single-app/app/src/index.ts | 2 +- apps/single-app/app/src/singleApp.ts | 1 + .../src/expectationManager.ts | 232 ++++++++++++------ 3 files changed, 161 insertions(+), 74 deletions(-) diff --git a/apps/single-app/app/src/index.ts b/apps/single-app/app/src/index.ts index e0c64269..244930d9 100644 --- a/apps/single-app/app/src/index.ts +++ b/apps/single-app/app/src/index.ts @@ -2,5 +2,5 @@ import { startSingleApp } from './singleApp' // eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup - +// eslint-disable-next-line no-console startSingleApp().catch(console.log) diff --git a/apps/single-app/app/src/singleApp.ts b/apps/single-app/app/src/singleApp.ts index 6e990a60..3582a42a 100644 --- a/apps/single-app/app/src/singleApp.ts +++ b/apps/single-app/app/src/singleApp.ts @@ -23,6 +23,7 @@ export async function startSingleApp(): Promise { logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) logger.info('------------------------------------------------------------------') + // eslint-disable-next-line no-console console.log(JSON.stringify(config, undefined, 2)) logger.info('------------------------------------------------------------------') diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 98f5b2c8..7456e31a 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -102,12 +102,13 @@ export class ExpectationManager { ) { this.constants = { // Default values: - EVALUATE_INTERVAL: 10 * 1000, + EVALUATE_INTERVAL: 5 * 1000, FULLFILLED_MONITOR_TIME: 10 * 1000, - WORK_TIMEOUT_TIME: 30 * 1000, + WORK_TIMEOUT_TIME: 10 * 1000, ALLOW_SKIPPING_QUEUE_TIME: 30 * 1000, SCALE_UP_TIME: 5 * 1000, SCALE_UP_COUNT: 1, + WORKER_SUPPORT_TIME: 60 * 1000, ...options?.constants, } @@ -131,6 +132,8 @@ export class ExpectationManager { client.on('close', () => { delete this.workerAgents[client.clientId] }) + + this._triggerEvaluateExpectations(true) break } case 'N/A': @@ -411,7 +414,7 @@ export class ExpectationManager { wip.trackedExp.errorCount++ this.updateTrackedExpStatus( wip.trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.NEW, reason ) this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { @@ -489,7 +492,12 @@ export class ExpectationManager { id: id, exp: exp, state: ExpectedPackageStatusAPI.WorkStatusState.NEW, - availableWorkers: [], + queriedWorkers: {}, + availableWorkers: {}, + noAvailableWorkersReason: { + user: 'Unknown reason', + tech: 'N/A (init)', + }, lastEvaluationTime: 0, waitingForWorkerTime: null, errorCount: 0, @@ -718,43 +726,69 @@ export class ExpectationManager { // Check which workers might want to handle it: // Reset properties: - trackedExp.availableWorkers = [] + // trackedExp.availableWorkers = [] trackedExp.status = {} - let notSupportReason: Reason = { - user: 'No workers registered (this is likely a configuration issue)', - tech: 'No workers registered', - } + let hasQueriedAnyone = false await Promise.all( - Object.entries(this.workerAgents).map(async ([id, workerAgent]) => { - try { - const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) - - if (support.support) { - trackedExp.availableWorkers.push(id) - } else { - notSupportReason = support.reason - } - } catch (err) { - if ((err + '').match(/timeout/i)) { - notSupportReason = { - user: 'Worker timed out', - tech: `Worker "${id} timeout"`, + Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { + // Only ask each worker once: + if ( + !trackedExp.queriedWorkers[workerId] || + Date.now() - trackedExp.queriedWorkers[workerId] > this.constants.WORKER_SUPPORT_TIME + ) { + trackedExp.queriedWorkers[workerId] = Date.now() + hasQueriedAnyone = true + try { + const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) + + if (support.support) { + trackedExp.availableWorkers[workerId] = true + } else { + delete trackedExp.availableWorkers[workerId] + trackedExp.noAvailableWorkersReason = support.reason } - } else throw err + } catch (err) { + delete trackedExp.availableWorkers[workerId] + + if ((err + '').match(/timeout/i)) { + trackedExp.noAvailableWorkersReason = { + user: 'Worker timed out', + tech: `Worker "${workerId} timeout"`, + } + } else throw err + } } }) ) - if (trackedExp.availableWorkers.length) { - this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.WAITING, { - user: `${trackedExp.availableWorkers.length} workers available, about to start...`, - tech: `Found ${trackedExp.availableWorkers.length} workers who supports this Expectation`, - }) - trackedExp.session.triggerExpectationAgain = true + const availableWorkersCount = Object.keys(trackedExp.availableWorkers).length + if (availableWorkersCount) { + if (hasQueriedAnyone) { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.WAITING, { + user: `${availableWorkersCount} workers available, about to start...`, + tech: `Found ${availableWorkersCount} workers who supports this Expectation`, + }) + trackedExp.session.triggerExpectationAgain = true + } else { + // If we didn't query anyone, just skip ahead to next state without being too verbose: + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + undefined + ) + } } else { + if (!Object.keys(trackedExp.queriedWorkers).length) { + trackedExp.noAvailableWorkersReason = { + user: 'No workers registered (this is likely a configuration issue)', + tech: 'No workers registered', + } + } this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { - user: `Found no workers who supports this Expectation, due to: ${notSupportReason.user}`, - tech: `Found no workers who supports this Expectation: "${notSupportReason.tech}"`, + user: `Found no workers who supports this Expectation, due to: ${trackedExp.noAvailableWorkersReason.user}`, + tech: `Found no workers who supports this Expectation: "${ + trackedExp.noAvailableWorkersReason.tech + }", have asked ${Object.keys(trackedExp.queriedWorkers).join(',')}`, }) } } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { @@ -803,7 +837,7 @@ export class ExpectationManager { // Not ready to start this.updateTrackedExpStatus( trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.NEW, readyToStart.reason, newStatus ) @@ -814,7 +848,7 @@ export class ExpectationManager { // Do nothing, hopefully some will be available at a later iteration this.updateTrackedExpStatus( trackedExp, - undefined, + ExpectedPackageStatusAPI.WorkStatusState.NEW, this.getNoAssignedWorkerReason(trackedExp.session) ) } @@ -882,7 +916,7 @@ export class ExpectationManager { trackedExp.status.workProgress = undefined this.updateTrackedExpStatus( trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.NEW, fulfilled.reason ) trackedExp.session.triggerExpectationAgain = true @@ -1093,34 +1127,33 @@ export class ExpectationManager { /** How many answers we want to have to be content */ const minWorkerCount = batchSize / 2 - if (!trackedExp.availableWorkers.length) { + if (!Object.keys(trackedExp.availableWorkers).length) { session.noAssignedWorkerReason = { user: `No workers available`, tech: `No workers available` } } const workerCosts: WorkerAgentAssignment[] = [] - for (let i = 0; i < trackedExp.availableWorkers.length; i += batchSize) { - const batchOfWorkers = trackedExp.availableWorkers.slice(i, i + batchSize) - - await Promise.all( - batchOfWorkers.map(async (workerId) => { - const workerAgent = this.workerAgents[workerId] - if (workerAgent) { - const cost = await workerAgent.api.getCostForExpectation(trackedExp.exp) - - if (cost.cost < Number.POSITIVE_INFINITY) { - workerCosts.push({ - worker: workerAgent.api, - id: workerId, - cost, - randomCost: Math.random(), // To randomize if there are several with the same best cost - }) - } + // Send a number of requests simultaneously: + await runInBatches( + Object.keys(trackedExp.availableWorkers), + async (workerId: string, abort: () => void) => { + const workerAgent = this.workerAgents[workerId] + if (workerAgent) { + const cost = await workerAgent.api.getCostForExpectation(trackedExp.exp) + + if (cost.cost < Number.POSITIVE_INFINITY) { + workerCosts.push({ + worker: workerAgent.api, + id: workerId, + cost, + randomCost: Math.random(), // To randomize if there are several with the same best cost + }) } - }) - ) - if (workerCosts.length >= minWorkerCount) break - } + } + if (workerCosts.length >= minWorkerCount) abort() + }, + batchSize + ) workerCosts.sort((a, b) => { // Lowest cost first: @@ -1143,8 +1176,10 @@ export class ExpectationManager { session.assignedWorker = bestWorker } else { session.noAssignedWorkerReason = { - user: `Waiting for a free worker (${trackedExp.availableWorkers.length} workers are currently busy)`, - tech: `Waiting for a free worker (${trackedExp.availableWorkers.length} busy)`, + user: `Waiting for a free worker (${ + Object.keys(trackedExp.availableWorkers).length + } workers are currently busy)`, + tech: `Waiting for a free worker (${Object.keys(trackedExp.availableWorkers).length} busy)`, } } } @@ -1173,6 +1208,22 @@ export class ExpectationManager { trackedExp: TrackedExpectation ): Promise { // First check if the Expectation depends on the fullfilled-status of another Expectation: + const waitingFor = this.isExpectationWaitingForOther(trackedExp) + + if (waitingFor) { + return { + ready: false, + reason: { + user: `Waiting for "${waitingFor.exp.statusReport.label}"`, + tech: `Waiting for "${waitingFor.exp.statusReport.label}"`, + }, + } + } + + return workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) + } + /** Checks if the expectation is waiting for another expectation, and returns the awaited Expectation, otherwise null */ + private isExpectationWaitingForOther(trackedExp: TrackedExpectation): TrackedExpectation | null { if (trackedExp.exp.dependsOnFullfilled?.length) { // Check if those are fullfilled: let waitingFor: TrackedExpectation | undefined = undefined @@ -1183,17 +1234,10 @@ export class ExpectationManager { } } if (waitingFor) { - return { - ready: false, - reason: { - user: `Waiting for "${waitingFor.exp.statusReport.label}"`, - tech: `Waiting for "${waitingFor.exp.statusReport.label}"`, - }, - } + return waitingFor } } - - return workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) + return null } private async _updateReceivedPackageContainerExpectations() { this.receivedUpdates.packageContainersHasBeenUpdated = false @@ -1452,9 +1496,11 @@ export class ExpectationManager { for (const exp of Object.values(this.trackedExpectations)) { if ( (exp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW || - exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) && - !exp.availableWorkers.length && // No workers supports it - !exp.session?.assignedWorker // No worker has time to work on it + exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING || + exp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) && + (!exp.availableWorkers.length || // No workers supports it + !exp.session?.assignedWorker) && // No worker has time to work on it + !this.isExpectationWaitingForOther(exp) // Filter out expectations that aren't ready to begin working on anyway ) { if (!exp.waitingForWorkerTime) { exp.waitingForWorkerTime = Date.now() @@ -1471,7 +1517,7 @@ export class ExpectationManager { } for (const exp of waitingExpectations) { - this.logger.info(`Requesting more resources to handle expectation "${exp.id}"`) + this.logger.debug(`Requesting more resources to handle expectation "${exp.id}"`) await this.workforceAPI.requestResources(exp.exp) } } @@ -1521,6 +1567,9 @@ export interface ExpectationManagerConstants { SCALE_UP_TIME: number /** How many resources to request at a time */ SCALE_UP_COUNT: number + + /** How often to re-query a worker if it supports an expectation [ms] */ + WORKER_SUPPORT_TIME: number } export type ExpectationManagerServerOptions = | { @@ -1543,8 +1592,11 @@ interface TrackedExpectation { /** Reason for the current state. */ reason: Reason + /** List of worker ids that have gotten the question wether they support this expectation */ + queriedWorkers: { [workerId: string]: number } /** List of worker ids that supports this Expectation */ - availableWorkers: string[] + availableWorkers: { [workerId: string]: true } + noAvailableWorkersReason: Reason /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number /** Timestamp to be track how long the expectation has been awiting for a worker (can't start working) */ @@ -1644,3 +1696,37 @@ interface TrackedPackageContainerExpectation { } } } +/** Execute callback in batches */ +async function runInBatches( + values: T[], + cb: (value: T, abortAndReturn: (returnValue?: ReturnValue) => void) => Promise, + batchSize: number +): Promise { + const batches: T[][] = [] + let batch: T[] = [] + for (const value of values) { + batch.push(value) + if (batch.length >= batchSize) { + batches.push(batch) + batch = [] + } + } + batches.push(batch) + + let aborted = false + let returnValue: ReturnValue | undefined = undefined + const abortAndReturn = (value?: ReturnValue) => { + aborted = true + returnValue = value + } + + for (const batch of batches) { + if (aborted) break + await Promise.all( + batch.map(async (value) => { + return cb(value, abortAndReturn) + }) + ) + } + return returnValue +} From 8aadc981de1bc43d86d9cf0a38c20b1832d51757 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 10:28:42 +0200 Subject: [PATCH 58/67] feat: Workers can now detect when idle for too long and request to be shut down. --- .../packages/generic/src/appContainer.ts | 98 +++++++++++++++---- .../packages/generic/src/workerAgentApi.ts | 3 + shared/packages/api/src/appContainer.ts | 5 + shared/packages/api/src/config.ts | 12 +++ shared/packages/api/src/methods.ts | 2 + shared/packages/worker/src/appContainerApi.ts | 3 + shared/packages/worker/src/workerAgent.ts | 95 +++++++++++++++--- .../src/__tests__/lib/setupEnv.ts | 1 + 8 files changed, 191 insertions(+), 28 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 4df02ba9..e83a3d46 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -12,6 +12,7 @@ import { assertNever, Expectation, waitTime, + APPCONTAINER_PING_TIME, } from '@shared/api' import { WorkforceAPI } from './workforceApi' import { WorkerAgentAPI } from './workerAgentApi' @@ -32,7 +33,10 @@ export class AppContainer { toBeKilled: boolean restarts: number lastRestart: number + spinDownTime: number workerAgentApi?: WorkerAgentAPI + monitorPing: boolean + lastPing: number } } = {} private availableApps: { @@ -40,27 +44,34 @@ export class AppContainer { } = {} private websocketServer?: WebsocketServer + private monitorAppsTimer: NodeJS.Timer | undefined + constructor(private logger: LoggerInstance, private config: AppContainerProcessConfig) { if (config.appContainer.port !== null) { this.websocketServer = new WebsocketServer(config.appContainer.port, (client: ClientConnection) => { // A new client has connected - this.logger.info(`AppContainer: New client "${client.clientType}" connected, id "${client.clientId}"`) + this.logger.debug(`AppContainer: New client "${client.clientType}" connected, id "${client.clientId}"`) switch (client.clientType) { case 'workerAgent': { - const workForceMethods = this.getWorkerAgentAPI() + const workForceMethods = this.getWorkerAgentAPI(client.clientId) const api = new WorkerAgentAPI(workForceMethods, { type: 'websocket', clientConnection: client, }) - if (!this.apps[client.clientId]) { + const app = this.apps[client.clientId] + if (!app) { throw new Error(`Unknown app "${client.clientId}" just connected to the appContainer`) } - this.apps[client.clientId].workerAgentApi = api + app.workerAgentApi = api client.on('close', () => { - delete this.apps[client.clientId].workerAgentApi + delete app.workerAgentApi }) + // Set upp the app for pinging and automatic spin-down: + app.monitorPing = true + app.lastPing = Date.now() + api.setSpinDownTime(app.spinDownTime) break } case 'expectationManager': @@ -103,22 +114,37 @@ export class AppContainer { } }) ) - - // Todo later: - // Sent the workforce a list of: - // * the types of workers we can spin up - // * what the running cost is for them - // * how many can be spun up - // * etc... + this.monitorAppsTimer = setInterval(() => { + this.monitorApps() + }, APPCONTAINER_PING_TIME) + this.monitorApps() // Also run right away } /** Return the API-methods that the AppContainer exposes to the WorkerAgent */ - private getWorkerAgentAPI(): AppContainerWorkerAgent.AppContainer { + private getWorkerAgentAPI(clientId: string): AppContainerWorkerAgent.AppContainer { return { ping: async (): Promise => { - // todo: Set last seen + this.apps[clientId].lastPing = Date.now() + }, + requestSpinDown: async (): Promise => { + const app = this.apps[clientId] + if (app) { + if (this.getAppCount(app.appType) > this.config.appContainer.minRunningApps) { + this.spinDown(clientId).catch((error) => { + this.logger.error(`AppContainer: Error when spinning down app "${clientId}"`) + this.logger.error(error) + }) + } + } }, } } + private getAppCount(appType: string): number { + let count = 0 + for (const app of Object.values(this.apps)) { + if (app.appType === appType) count++ + } + return count + } private async setupAvailableApps() { const getWorkerArgs = (appId: string): string[] => { return [ @@ -167,6 +193,11 @@ export class AppContainer { this.workforceAPI.terminate() this.websocketServer?.terminate() + if (this.monitorAppsTimer) { + clearInterval(this.monitorAppsTimer) + delete this.monitorAppsTimer + } + // kill child processes } async setLogLevel(logLevel: LogLevel): Promise { @@ -181,7 +212,9 @@ export class AppContainer { } async requestAppTypeForExpectation(exp: Expectation.Any): Promise<{ appType: string; cost: number } | null> { + this.logger.debug(`AppContainer: Got request for resources, for exp "${exp.id}"`) if (Object.keys(this.apps).length >= this.config.appContainer.maxRunningApps) { + this.logger.debug(`AppContainer: Is already at our limit, no more resources available`) // If we're at our limit, we can't possibly run anything else return null } @@ -193,7 +226,7 @@ export class AppContainer { }) if (!runningApp) { - const newAppId = await this.spinUp(appType) // todo: make it not die too soon + const newAppId = await this.spinUp(appType, true) // todo: make it not die too soon // wait for the app to connect to us: tryAfewTimes(async () => { @@ -214,11 +247,13 @@ export class AppContainer { cost: availableApp.cost, } } + } else { + this.logger.debug(`AppContainer: appType "${appType}" not available`) } } return null } - async spinUp(appType: string): Promise { + async spinUp(appType: string, longSpinDownTime = false): Promise { const availableApp = this.availableApps[appType] if (!availableApp) throw new Error(`Unknown appType "${appType}"`) @@ -231,6 +266,9 @@ export class AppContainer { toBeKilled: false, restarts: 0, lastRestart: 0, + monitorPing: false, + lastPing: Date.now(), + spinDownTime: this.config.appContainer.spinDownTime * (longSpinDownTime ? 10 : 1), } return appId } @@ -238,6 +276,8 @@ export class AppContainer { const app = this.apps[appId] if (!app) throw new Error(`App "${appId}" not found`) + this.logger.debug(`AppContainer: Spinning down app "${appId}"`) + app.toBeKilled = true const success = app.process.kill() if (!success) throw new Error(`Internal error: Killing of process "${app.process.pid}" failed`) @@ -260,7 +300,7 @@ export class AppContainer { appId: string, availableApp: AvailableAppInfo ): ChildProcess.ChildProcess { - this.logger.info(`Starting process "${appId}" (${appType}): "${availableApp.file}"`) + this.logger.debug(`Starting process "${appId}" (${appType}): "${availableApp.file}"`) const cwd = process.execPath.match(/node.exe$/) ? undefined // Process runs as a node process, we're probably in development mode. : path.dirname(process.execPath) // Process runs as a node process, we're probably in development mode. @@ -300,6 +340,30 @@ export class AppContainer { return child } + private monitorApps() { + for (const [appId, app] of Object.entries(this.apps)) { + if (app.monitorPing) { + if (Date.now() - app.lastPing > APPCONTAINER_PING_TIME * 2.5) { + // The app seems to have crashed. + this.spinDown(appId).catch((error) => { + this.logger.error(`AppContainer: Error when spinning down app "${appId}"`) + this.logger.error(error) + }) + } + } + } + this.spinUpMinimumApps().catch((error) => { + this.logger.error(`AppContainer: Error in spinUpMinimumApps`) + this.logger.error(error) + }) + } + private async spinUpMinimumApps(): Promise { + for (const appType of Object.keys(this.availableApps)) { + while (this.getAppCount(appType) < this.config.appContainer.minRunningApps) { + await this.spinUp(appType) + } + } + } } interface AvailableAppInfo { file: string diff --git a/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts index 1232288c..03b04632 100644 --- a/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts +++ b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts @@ -31,4 +31,7 @@ export class WorkerAgentAPI async doYouSupportExpectation(exp: Expectation.Any): Promise { return this._sendMessage('doYouSupportExpectation', exp) } + async setSpinDownTime(spinDownTime: number): Promise { + return this._sendMessage('setSpinDownTime', spinDownTime) + } } diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts index 7ac0523e..ad335ae1 100644 --- a/shared/packages/api/src/appContainer.ts +++ b/shared/packages/api/src/appContainer.ts @@ -2,11 +2,16 @@ import { WorkerAgentConfig } from './worker' /** The AppContainer is a host application responsible for spawning other applications */ +/** How often the appContainer expect to be pinged by its child apps */ +export const APPCONTAINER_PING_TIME = 5000 // ms + export interface AppContainerConfig { workforceURL: string | null port: number | null appContainerId: string + minRunningApps: number maxRunningApps: number + spinDownTime: number resourceId: string networkIds: string[] diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 2ec834ee..2a2cfca0 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -152,6 +152,16 @@ const appContainerArguments = defineArguments({ default: parseInt(process.env.APP_CONTAINER_MAX_RUNNING_APPS || '', 10) || 3, describe: 'How many apps the appContainer can run at the same time', }, + minRunningApps: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_MIN_RUNNING_APPS || '', 10) || 0, + describe: 'Minimum amount of apps (of a certain appType) to be running', + }, + spinDownTime: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_SPIN_DOWN_TIME || '', 10) || 60 * 1000, + describe: 'How long a Worker should stay idle before attempting to be spun down', + }, resourceId: { type: 'string', @@ -343,6 +353,8 @@ export function getAppContainerConfig(): AppContainerProcessConfig { port: argv.port, appContainerId: argv.appContainerId, maxRunningApps: argv.maxRunningApps, + minRunningApps: argv.minRunningApps, + spinDownTime: argv.spinDownTime, resourceId: argv.resourceId, networkIds: argv.networkIds ? argv.networkIds.split(';') : [], diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 52afd610..7f90484f 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -192,9 +192,11 @@ export namespace AppContainerWorkerAgent { _debugKill: () => Promise doYouSupportExpectation: (exp: Expectation.Any) => Promise + setSpinDownTime: (spinDownTime: number) => Promise } /** Methods on AppContainer, called by WorkerAgent */ export interface AppContainer { ping: () => Promise + requestSpinDown: () => Promise } } diff --git a/shared/packages/worker/src/appContainerApi.ts b/shared/packages/worker/src/appContainerApi.ts index b6f159cc..ad18abd4 100644 --- a/shared/packages/worker/src/appContainerApi.ts +++ b/shared/packages/worker/src/appContainerApi.ts @@ -14,4 +14,7 @@ export class AppContainerAPI async ping(): Promise { return this._sendMessage('ping') } + async requestSpinDown(): Promise { + return this._sendMessage('requestSpinDown') + } } diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index e426fe9c..e4575508 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -17,6 +17,7 @@ import { ReturnTypeSetupPackageContainerMonitors, ReturnTypeDisposePackageContainerMonitors, LogLevel, + APPCONTAINER_PING_TIME, } from '@shared/api' import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' @@ -54,6 +55,10 @@ export class WorkerAgent { > } = {} private terminated = false + private spinDownTime = 0 + private intervalCheckTimer: NodeJS.Timer | null = null + private lastWorkTime = 0 + private activeMonitors: { [monitorId: string]: true } = {} constructor(private logger: LoggerInstance, private config: WorkerConfig) { this.workforceAPI = new WorkforceAPI(this.logger) @@ -113,12 +118,22 @@ export class WorkerAgent { const list = await this.workforceAPI.getExpectationManagerList() await this.updateListOfExpectationManagers(list) + this.IDidSomeWork() } terminate(): void { this.terminated = true this.workforceAPI.terminate() - Object.values(this.expectationManagers).forEach((expectationManager) => expectationManager.api.terminate()) - // this._worker.terminate() + + for (const expectationManager of Object.values(this.expectationManagers)) { + expectationManager.api.terminate() + } + for (const wipId of Object.keys(this.worksInProgress)) { + this.cancelWorkInProgress(wipId).catch((error) => { + this.logger.error('WorkerAgent.terminate: Error in cancelWorkInProgress') + this.logger.error(error) + }) + } + if (this.intervalCheckTimer) clearInterval(this.intervalCheckTimer) this._worker.terminate() } /** Called when running in the same-process-mode, it */ @@ -148,6 +163,7 @@ export class WorkerAgent { // return this._busyMethodCount === 0 // } async doYouSupportExpectation(exp: Expectation.Any): Promise { + this.IDidSomeWork() return this._worker.doYouSupportExpectation(exp) } async expectationManagerAvailable(id: string, url: string): Promise { @@ -165,12 +181,19 @@ export class WorkerAgent { this.logger.level = logLevel } async _debugKill(): Promise { + this.terminate() // This is for testing purposes only setTimeout(() => { // eslint-disable-next-line no-process-exit process.exit(42) }, 1) } + public async setSpinDownTime(spinDownTime: number): Promise { + this.spinDownTime = spinDownTime + this.IDidSomeWork() + + this.setupIntervalCheck() + } private async connectToExpectationManager(id: string, url: string): Promise { this.logger.info(`Worker: Connecting to Expectation Manager "${id}" at url "${url}"`) @@ -201,12 +224,14 @@ export class WorkerAgent { exp: Expectation.Any, wasFullfilled: boolean ): Promise => { + this.IDidSomeWork() return this._worker.isExpectationFullfilled(exp, wasFullfilled) }, workOnExpectation: async ( exp: Expectation.Any, cost: ExpectationManagerWorkerAgent.ExpectationCost ): Promise => { + this.IDidSomeWork() const currentjob = { cost: cost, progress: 0, @@ -224,6 +249,7 @@ export class WorkerAgent { this.worksInProgress[`${wipId}`] = workInProgress workInProgress.on('progress', (actualVersionHash, progress: number) => { + this.IDidSomeWork() currentjob.progress = progress expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch((err) => { if (!this.terminated) { @@ -233,6 +259,7 @@ export class WorkerAgent { }) }) workInProgress.on('error', (error: string) => { + this.IDidSomeWork() this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to error. (${this.currentJobs.length})` @@ -252,6 +279,7 @@ export class WorkerAgent { delete this.worksInProgress[`${wipId}`] }) workInProgress.on('done', (actualVersionHash, reason, result) => { + this.IDidSomeWork() this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) this.logger.debug( `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), done. (${this.currentJobs.length})` @@ -282,34 +310,46 @@ export class WorkerAgent { } }, removeExpectation: async (exp: Expectation.Any): Promise => { + this.IDidSomeWork() return this._worker.removeExpectation(exp) }, cancelWorkInProgress: async (wipId: number): Promise => { - const wip = this.worksInProgress[`${wipId}`] - if (wip) { - await wip.cancel() - } - delete this.worksInProgress[`${wipId}`] + this.IDidSomeWork() + return this.cancelWorkInProgress(wipId) }, doYouSupportPackageContainer: ( packageContainer: PackageContainerExpectation ): Promise => { + this.IDidSomeWork() return this._worker.doYouSupportPackageContainer(packageContainer) }, runPackageContainerCronJob: ( packageContainer: PackageContainerExpectation ): Promise => { + this.IDidSomeWork() return this._worker.runPackageContainerCronJob(packageContainer) }, - setupPackageContainerMonitors: ( + setupPackageContainerMonitors: async ( packageContainer: PackageContainerExpectation ): Promise => { - return this._worker.setupPackageContainerMonitors(packageContainer) + this.IDidSomeWork() + const monitors = await this._worker.setupPackageContainerMonitors(packageContainer) + if (monitors.success) { + for (const monitorId of Object.keys(monitors.monitors)) { + this.activeMonitors[monitorId] = true + } + } + return monitors }, - disposePackageContainerMonitors: ( + disposePackageContainerMonitors: async ( packageContainer: PackageContainerExpectation ): Promise => { - return this._worker.disposePackageContainerMonitors(packageContainer) + this.IDidSomeWork() + const success = await this._worker.disposePackageContainerMonitors(packageContainer) + if (success.success) { + this.activeMonitors = {} + } + return success }, }) // Wrap the methods, so that we can cut off communication upon termination: (this is used in tests) @@ -363,4 +403,37 @@ export class WorkerAgent { } } } + private async cancelWorkInProgress(wipId: string | number): Promise { + const wip = this.worksInProgress[`${wipId}`] + if (wip) { + await wip.cancel() + } + delete this.worksInProgress[`${wipId}`] + } + private setupIntervalCheck() { + if (!this.intervalCheckTimer) { + this.intervalCheckTimer = setInterval(() => { + this.intervalCheck() + }, APPCONTAINER_PING_TIME) + } + } + private intervalCheck() { + // Check the SpinDownTime: + if (this.spinDownTime) { + if (Date.now() - this.lastWorkTime > this.spinDownTime) { + this.IDidSomeWork() // so that we won't ask again until later + + // Don's spin down if a monitor is active + if (!Object.keys(this.activeMonitors).length) { + this.logger.info(`Worker: is idle, requesting spinning down`) + this.appContainerAPI.requestSpinDown() + } + } + } + // Also ping the AppContainer + this.appContainerAPI.ping() + } + private IDidSomeWork() { + this.lastWorkTime = Date.now() + } } diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index 97d352ed..d08c2094 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -56,6 +56,7 @@ const defaultTestConfig: SingleAppConfig = { workforceURL: null, port: 0, maxRunningApps: 1, + minRunningApps: 1, resourceId: '', networkIds: [], windowsDriveLetters: ['X', 'Y', 'Z'], From 04b24f9d4fad253fe752d8d0adcc527f17f270fa Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 11:08:41 +0200 Subject: [PATCH 59/67] chore: expextationManager: Improve error handling --- .../src/expectationManager.ts | 208 ++++++++++-------- 1 file changed, 122 insertions(+), 86 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 7456e31a..39e0ec9b 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -797,51 +797,60 @@ export class ExpectationManager { await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { - // First, check if it is already fulfilled: - const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( - trackedExp.exp, - false - ) - if (fulfilled.fulfilled) { - // The expectation is already fulfilled: - this.updateTrackedExpStatus( - trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, - undefined - ) - if (this.handleTriggerByFullfilledIds(trackedExp)) { - // Something was triggered, run again ASAP: - trackedExp.session.triggerOtherExpectationsAgain = true - } - } else { - const readyToStart = await this.isExpectationReadyToStartWorkingOn( - trackedExp.session.assignedWorker.worker, - trackedExp + try { + // First, check if it is already fulfilled: + const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( + trackedExp.exp, + false ) - - const newStatus: Partial = {} - if (readyToStart.sourceExists !== undefined) newStatus.sourceExists = readyToStart.sourceExists - - if (readyToStart.ready) { + if (fulfilled.fulfilled) { + // The expectation is already fulfilled: this.updateTrackedExpStatus( trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.READY, - { - user: 'About to start working..', - tech: 'About to start working..', - }, - newStatus + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, + undefined ) - trackedExp.session.triggerExpectationAgain = true + if (this.handleTriggerByFullfilledIds(trackedExp)) { + // Something was triggered, run again ASAP: + trackedExp.session.triggerOtherExpectationsAgain = true + } } else { - // Not ready to start - this.updateTrackedExpStatus( - trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.NEW, - readyToStart.reason, - newStatus + const readyToStart = await this.isExpectationReadyToStartWorkingOn( + trackedExp.session.assignedWorker.worker, + trackedExp ) + + const newStatus: Partial = {} + if (readyToStart.sourceExists !== undefined) + newStatus.sourceExists = readyToStart.sourceExists + + if (readyToStart.ready) { + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.READY, + { + user: 'About to start working..', + tech: 'About to start working..', + }, + newStatus + ) + trackedExp.session.triggerExpectationAgain = true + } else { + // Not ready to start + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + readyToStart.reason, + newStatus + ) + } } + } catch (error) { + // There was an error, clearly it's not ready to start + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: 'Restarting due to error', + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) } } else { // No worker is available at the moment. @@ -859,30 +868,41 @@ export class ExpectationManager { if (trackedExp.session.assignedWorker) { const assignedWorker = trackedExp.session.assignedWorker - this.logger.debug(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) + try { + this.logger.debug(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) - // Start working on the Expectation: - const wipInfo = await assignedWorker.worker.workOnExpectation(trackedExp.exp, assignedWorker.cost) + // Start working on the Expectation: + const wipInfo = await assignedWorker.worker.workOnExpectation( + trackedExp.exp, + assignedWorker.cost + ) - trackedExp.status.workInProgressCancel = async () => { - await assignedWorker.worker.cancelWorkInProgress(wipInfo.wipId) - delete trackedExp.status.workInProgressCancel - } + trackedExp.status.workInProgressCancel = async () => { + await assignedWorker.worker.cancelWorkInProgress(wipInfo.wipId) + delete trackedExp.status.workInProgressCancel + } - // trackedExp.status.workInProgress = new WorkInProgressReceiver(wipInfo.properties) - this.worksInProgress[`${assignedWorker.id}_${wipInfo.wipId}`] = { - properties: wipInfo.properties, - trackedExp: trackedExp, - worker: assignedWorker.worker, - lastUpdated: Date.now(), - } + // trackedExp.status.workInProgress = new WorkInProgressReceiver(wipInfo.properties) + this.worksInProgress[`${assignedWorker.id}_${wipInfo.wipId}`] = { + properties: wipInfo.properties, + trackedExp: trackedExp, + worker: assignedWorker.worker, + lastUpdated: Date.now(), + } - this.updateTrackedExpStatus( - trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.WORKING, - undefined, - wipInfo.properties - ) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WORKING, + undefined, + wipInfo.properties + ) + } catch (error) { + // There was an error + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: 'Restarting due to an error', + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) + } } else { // No worker is available at the moment. // Do nothing, hopefully some will be available at a later iteration @@ -901,25 +921,34 @@ export class ExpectationManager { if (timeSinceLastEvaluation > this.getFullfilledWaitTime()) { await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { - // Check if it is still fulfilled: - const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( - trackedExp.exp, - true - ) - if (fulfilled.fulfilled) { - // Yes it is still fullfiled - // No need to update the tracked state, since it's already fullfilled: - // this.updateTrackedExp(trackedExp, WorkStatusState.FULFILLED, fulfilled.reason) - } else { - // It appears like it's not fullfilled anymore - trackedExp.status.actualVersionHash = undefined - trackedExp.status.workProgress = undefined - this.updateTrackedExpStatus( - trackedExp, - ExpectedPackageStatusAPI.WorkStatusState.NEW, - fulfilled.reason + try { + // Check if it is still fulfilled: + const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( + trackedExp.exp, + true ) - trackedExp.session.triggerExpectationAgain = true + if (fulfilled.fulfilled) { + // Yes it is still fullfiled + // No need to update the tracked state, since it's already fullfilled: + // this.updateTrackedExp(trackedExp, WorkStatusState.FULFILLED, fulfilled.reason) + } else { + // It appears like it's not fullfilled anymore + trackedExp.status.actualVersionHash = undefined + trackedExp.status.workProgress = undefined + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + fulfilled.reason + ) + trackedExp.session.triggerExpectationAgain = true + } + } catch (error) { + // Do nothing, hopefully some will be available at a later iteration + // todo: Is this the right thing to do? + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Can't check if fulfilled, due to an error`, + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) } } else { // No worker is available at the moment. @@ -1132,6 +1161,7 @@ export class ExpectationManager { } const workerCosts: WorkerAgentAssignment[] = [] + let noCostReason: string | undefined = undefined // Send a number of requests simultaneously: await runInBatches( @@ -1139,15 +1169,19 @@ export class ExpectationManager { async (workerId: string, abort: () => void) => { const workerAgent = this.workerAgents[workerId] if (workerAgent) { - const cost = await workerAgent.api.getCostForExpectation(trackedExp.exp) - - if (cost.cost < Number.POSITIVE_INFINITY) { - workerCosts.push({ - worker: workerAgent.api, - id: workerId, - cost, - randomCost: Math.random(), // To randomize if there are several with the same best cost - }) + try { + const cost = await workerAgent.api.getCostForExpectation(trackedExp.exp) + + if (cost.cost < Number.POSITIVE_INFINITY) { + workerCosts.push({ + worker: workerAgent.api, + id: workerId, + cost, + randomCost: Math.random(), // To randomize if there are several with the same best cost + }) + } + } catch (error) { + noCostReason = error.toString() } } if (workerCosts.length >= minWorkerCount) abort() @@ -1179,7 +1213,9 @@ export class ExpectationManager { user: `Waiting for a free worker (${ Object.keys(trackedExp.availableWorkers).length } workers are currently busy)`, - tech: `Waiting for a free worker (${Object.keys(trackedExp.availableWorkers).length} busy)`, + tech: `Waiting for a free worker (${ + Object.keys(trackedExp.availableWorkers).length + } busy) ${noCostReason}`, } } } From d2e5c52d4ead9f9c17f2142cf080f14db3716d8b Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 27 Aug 2021 11:23:36 +0200 Subject: [PATCH 60/67] chore: fix tests --- shared/packages/api/src/adapterServer.ts | 8 ++++++-- tests/internal-tests/package.json | 4 ++-- tests/internal-tests/src/__mocks__/child_process.ts | 7 +++++++ tests/internal-tests/src/__mocks__/fs.ts | 11 ++++++++++- tests/internal-tests/src/__tests__/issues.spec.ts | 7 ++++--- tests/internal-tests/src/__tests__/lib/setupEnv.ts | 1 + 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/shared/packages/api/src/adapterServer.ts b/shared/packages/api/src/adapterServer.ts index c62b5d64..f2bc2de6 100644 --- a/shared/packages/api/src/adapterServer.ts +++ b/shared/packages/api/src/adapterServer.ts @@ -29,10 +29,14 @@ export abstract class AdapterServer { }) } else { const clientHook: OTHER = options.hookMethods - this._sendMessage = (type: keyof OTHER, ...args: any[]) => { + this._sendMessage = async (type: keyof OTHER, ...args: any[]) => { const fcn = (clientHook[type] as unknown) as (...args: any[]) => any if (fcn) { - return promiseTimeout(fcn(...args), MESSAGE_TIMEOUT) + try { + return await promiseTimeout(fcn(...args), MESSAGE_TIMEOUT) + } catch (err) { + throw new Error(`Error in message "${type}": ${err.toString()}`) + } } else { throw new Error(`Unknown method "${type}"`) } diff --git a/tests/internal-tests/package.json b/tests/internal-tests/package.json index 99a24e21..619ac3cb 100644 --- a/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "test": "jest", + "test": "jest --runInBand", "precommit": "lint-staged" }, "devDependencies": { @@ -40,4 +40,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/tests/internal-tests/src/__mocks__/child_process.ts b/tests/internal-tests/src/__mocks__/child_process.ts index 200dc7cc..53c8e72d 100644 --- a/tests/internal-tests/src/__mocks__/child_process.ts +++ b/tests/internal-tests/src/__mocks__/child_process.ts @@ -42,6 +42,8 @@ function spawn(command: string, args: string[] = []) { spawned.emit('close', 9999) }) }) + } else if (command === 'taskkill') { + // mock killing a task? } else { throw new Error(`Mock child_process.spawn: command not implemented: "${command}"`) } @@ -52,6 +54,11 @@ child_process.spawn = spawn class SpawnedProcess extends EventEmitter { public stdout = new EventEmitter() public stderr = new EventEmitter() + public pid: number + constructor() { + super() + this.pid = Date.now() + } } async function robocopy(spawned: SpawnedProcess, args: string[]) { let sourceFolder diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index 9b7e266c..ecd76d15 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -397,8 +397,17 @@ export function writeFile(path: string, data: Buffer | string, callback: (error: } } fs.writeFile = writeFile -function readFile(path: string, callback: (error: any, result?: any) => void): void { +function readFile(path: string, ...args: any[]): void { path = fixPath(path) + + let callback: (error: any, result?: any) => void + if (args.length === 1) { + callback = args[0] + } else if (args.length === 2) { + // const options = args[0] + callback = args[1] + } else throw new Error(`Mock poorly implemented: ` + args) + if (DEBUG_LOG) console.log('fs.readFile', path) fsMockEmitter.emit('readFile', path) try { diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts index b30c8688..5e183933 100644 --- a/tests/internal-tests/src/__tests__/issues.spec.ts +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -132,7 +132,7 @@ describe('Handle unhappy paths', () => { expect(env.expectationStatuses['copy0']).toMatchObject({ actualVersionHash: null, statusInfo: { - status: 'waiting', + status: 'new', statusReason: { tech: /not able to access target/i, }, @@ -211,14 +211,15 @@ describe('Handle unhappy paths', () => { ) await waitTime(env.WAIT_JOB_TIME) - // Expect the Expectation to be waiting: + // Expect the Expectation to be still working + // (the worker has crashed, byt expectationManger hasn't noticed yet) expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('working') expect(listenToCopyFile).toHaveBeenCalledTimes(1) await waitTime(env.WORK_TIMEOUT_TIME) await waitTime(env.WAIT_JOB_TIME) // By now, the work should have been aborted, and restarted: - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('new') + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual(expect.stringMatching(/new|waiting/)) // Add another worker: env.addWorker() diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index d08c2094..ff3a72e1 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -57,6 +57,7 @@ const defaultTestConfig: SingleAppConfig = { port: 0, maxRunningApps: 1, minRunningApps: 1, + spinDownTime: 0, resourceId: '', networkIds: [], windowsDriveLetters: ['X', 'Y', 'Z'], From 8c4503669ab6e5caa25f609ba2ca77d661b17529 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 30 Aug 2021 08:42:22 +0200 Subject: [PATCH 61/67] chore: improve build-win32 scripts --- apps/_boilerplate/app/package.json | 4 +- apps/_boilerplate/app/scripts/copy-natives.js | 33 ----- apps/appcontainer-node/app/package.json | 4 +- .../packages/generic/src/appContainer.ts | 4 + apps/http-server/app/package.json | 4 +- apps/http-server/app/scripts/copy-natives.js | 33 ----- apps/package-manager/app/package.json | 4 +- .../app/scripts/copy-natives.js | 33 ----- .../app/package.json | 4 +- .../app/scripts/copy-natives.js | 33 ----- apps/single-app/app/package.json | 4 +- apps/single-app/app/scripts/build-win32.js | 131 ------------------ apps/single-app/app/scripts/copy-natives.js | 33 ----- apps/worker/app/package.json | 5 +- apps/worker/app/scripts/copy-natives.js | 33 ----- apps/workforce/app/package.json | 4 +- apps/workforce/app/scripts/copy-natives.js | 33 ----- package.json | 3 +- scripts/build-win32.js | 103 ++++++++++++++ .../app/scripts => scripts}/copy-natives.js | 39 ++++-- scripts/reset.js | 63 +++++++++ scripts/run-sequencially.js | 74 ++++++++++ .../src/worker/accessorHandlers/quantel.ts | 8 +- shared/packages/worker/src/worker/lib/lib.ts | 2 +- .../quantelClipThumbnail.ts | 2 +- 25 files changed, 297 insertions(+), 396 deletions(-) delete mode 100644 apps/_boilerplate/app/scripts/copy-natives.js delete mode 100644 apps/http-server/app/scripts/copy-natives.js delete mode 100644 apps/package-manager/app/scripts/copy-natives.js delete mode 100644 apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js delete mode 100644 apps/single-app/app/scripts/build-win32.js delete mode 100644 apps/single-app/app/scripts/copy-natives.js delete mode 100644 apps/worker/app/scripts/copy-natives.js delete mode 100644 apps/workforce/app/scripts/copy-natives.js create mode 100644 scripts/build-win32.js rename {apps/appcontainer-node/app/scripts => scripts}/copy-natives.js (56%) create mode 100644 scripts/reset.js create mode 100644 scripts/run-sequencially.js diff --git a/apps/_boilerplate/app/package.json b/apps/_boilerplate/app/package.json index 0093206e..9a5ed7ce 100644 --- a/apps/_boilerplate/app/package.json +++ b/apps/_boilerplate/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/boilerplate.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/boilerplate.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js boilerplate.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -33,4 +33,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/_boilerplate/app/scripts/copy-natives.js b/apps/_boilerplate/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/_boilerplate/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/appcontainer-node/app/package.json b/apps/appcontainer-node/app/package.json index 774734f4..652e517d 100644 --- a/apps/appcontainer-node/app/package.json +++ b/apps/appcontainer-node/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js appContainer-node.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -33,4 +33,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index e83a3d46..0eb5a08e 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -188,6 +188,10 @@ export class AppContainer { } }) } + this.logger.info(`AppContainer: Available apps`) + for (const [appType, availableApp] of Object.entries(this.availableApps)) { + this.logger.info(`${appType}: ${availableApp.file}`) + } } terminate(): void { this.workforceAPI.terminate() diff --git a/apps/http-server/app/package.json b/apps/http-server/app/package.json index bde53183..eac1c1b4 100644 --- a/apps/http-server/app/package.json +++ b/apps/http-server/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/http-server.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/http-server.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js http-server.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -33,4 +33,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/http-server/app/scripts/copy-natives.js b/apps/http-server/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/http-server/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/package-manager/app/package.json b/apps/package-manager/app/package.json index 96489ff8..02bdad8e 100644 --- a/apps/package-manager/app/package.json +++ b/apps/package-manager/app/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/package-manager.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/package-manager.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js package-manager.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -32,4 +32,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/package-manager/app/scripts/copy-natives.js b/apps/package-manager/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/package-manager/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/quantel-http-transformer-proxy/app/package.json b/apps/quantel-http-transformer-proxy/app/package.json index daba3435..fc45a080 100644 --- a/apps/quantel-http-transformer-proxy/app/package.json +++ b/apps/quantel-http-transformer-proxy/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/quantel-http-transformer-proxy.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/quantel-http-transformer-proxy.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js quantel-http-transformer-proxy.exe && node ../../../scripts/copy-natives.js win32-x64", "start": "node dist/index.js", "precommit": "lint-staged" }, @@ -32,4 +32,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js b/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index 7cd802ae..d849bb9f 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & node scripts/build-win32.js && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js package-manager-single-app.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -40,4 +40,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/single-app/app/scripts/build-win32.js b/apps/single-app/app/scripts/build-win32.js deleted file mode 100644 index 91c427dc..00000000 --- a/apps/single-app/app/scripts/build-win32.js +++ /dev/null @@ -1,131 +0,0 @@ -const promisify = require("util").promisify -const cp = require('child_process') -const path = require('path') -const nexe = require('nexe') -const exec = promisify(cp.exec) -const glob = promisify(require("glob")) - -const fse = require('fs-extra'); -const mkdirp = require('mkdirp'); -const rimraf = promisify(require('rimraf')) - -const fseCopy = promisify(fse.copy) - -const packageJson = require('../package.json') - -/* - Due to nexe not taking into account the packages in the mono-repo, we're doing a hack, - copying the packages into node_modules, so that nexe will include them. -*/ - -const outputPath = './deploy/' - - -;(async () => { - - const basePath = './' - - // List all Lerna packages: - const list = await exec( 'yarn lerna list -a --json') - const str = list.stdout - .replace(/^\$.*$/gm, '') - .replace(/^Done in.*$/gm, '') - const packages = JSON.parse(str) - - await mkdirp(basePath + 'node_modules') - - // Copy the packages into node_modules: - const copiedFolders = [] - for (const package of packages) { - if (package.name.match(/boilerplate/)) continue - if (package.name.match(packageJson.name)) continue - - console.log(`Copying: ${package.name}`) - const target = basePath + `node_modules/${package.name}` - await fseCopy(package.location, target) - copiedFolders.push(target) - } - - // Remove things that arent used, to reduce file size: - const copiedFiles = [ - ...await glob(`${basePath}node_modules/@*/app/*`), - ...await glob(`${basePath}node_modules/@*/generic/*`), - ] - for (const file of copiedFiles) { - if ( - // Only keep these: - !file.match(/package.json$/) && - !file.match(/node_modules$/) && - !file.match(/dist$/) - ) { - await rimraf(file) - } - } - - await nexe.compile({ - input: './dist/index.js', - output: outputPath + 'package-manager-single-app.exe', - // build: true, //required to use patches - targets: [ - 'windows-x64-12.18.1' - ], - }) - - // Clean after ourselves: - for (const copiedFolder of copiedFolders) { - await rimraf(copiedFolder) - } - - // const basePath = 'C:\\Users/johan/Desktop/New folder/' - // await nexe.compile({ - // input: './dist/index.js', - // output: basePath + 'package-manager-single-app.exe', - // // build: true, //required to use patches - // targets: [ - // 'windows-x64-12.18.1' - // ], - // }) - - // // Copy node_modules: - // const list = await exec( 'yarn lerna list -a --json') - // const str = list.stdout - // // .replace(/.*(\[.*)\nDone in.*/gs, '$1') - // .replace(/^\$.*$/gm, '') - // .replace(/^Done in.*$/gm, '') - // // console.log('str', str) - // const packages = JSON.parse(str) - - // await mkdirp(basePath + 'node_modules') - - - // for (const package of packages) { - // if (package.name.match(/boilerplate/)) continue - - // console.log(`Copying: ${package.name}`) - // await fseCopy(package.location, basePath + `node_modules/${package.name}`) - // } - - // // remove things that arent used: - // const copiedFiles = [ - // ...await glob(`${basePath}node_modules/@*/app/*`), - // ...await glob(`${basePath}node_modules/@*/generic/*`), - // ] - // console.log(copiedFiles) - // for (const file of copiedFiles) { - // if ( - // file.match(/package.json$/) || - // file.match(/node_modules$/) || - // file.match(/dist$/) - // ) { - // // keep it - // } else { - // await rimraf(file) - // } - // } - - - - - -})().catch(console.error) - diff --git a/apps/single-app/app/scripts/copy-natives.js b/apps/single-app/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/single-app/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/worker/app/package.json b/apps/worker/app/package.json index ac035cf2..07467154 100644 --- a/apps/worker/app/package.json +++ b/apps/worker/app/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "oldbuild-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js worker.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -33,4 +34,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/worker/app/scripts/copy-natives.js b/apps/worker/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/worker/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/workforce/app/package.json b/apps/workforce/app/package.json index 5a219a7a..2f65bc71 100644 --- a/apps/workforce/app/package.json +++ b/apps/workforce/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/workforce.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/workforce.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js workforce.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -33,4 +33,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/workforce/app/scripts/copy-natives.js b/apps/workforce/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/workforce/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/package.json b/package.json index c108e91e..81798675 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "scripts": { "ci": "yarn install && yarn build && yarn lint && yarn test", "setup": "lerna bootstrap", + "reset": "node scripts/reset.js", "build": "lerna run build --stream", "build:changed": "lerna run build --since head --exclude-dependents --stream", "lint": "lerna exec -- eslint . --ext .ts,.tsx", @@ -22,7 +23,7 @@ "test:update:changed": "lerna run --since origin/master --include-dependents test -- -u", "typecheck": "lerna exec -- tsc --noEmit", "typecheck:changed": "lerna exec --since origin/master --include-dependents -- tsc --noEmit", - "build-win32": "lerna run build-win32", + "build-win32": "node scripts/run-sequencially.js lerna run build-win32 --stream", "gather-built": "node scripts/gather-all-built.js", "start:http-server": "lerna run start --stream --scope @http-server/app", "start:workforce": "lerna run start --stream --scope @workforce/app", diff --git a/scripts/build-win32.js b/scripts/build-win32.js new file mode 100644 index 00000000..a6178274 --- /dev/null +++ b/scripts/build-win32.js @@ -0,0 +1,103 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const cp = require('child_process') +const path = require('path') +const nexe = require('nexe') +const exec = promisify(cp.exec) +const glob = promisify(require('glob')) + +const fse = require('fs-extra'); +const mkdirp = require('mkdirp'); +const rimraf = promisify(require('rimraf')) + +const fseCopy = promisify(fse.copy) + +/* + Due to nexe not taking into account the packages in the mono-repo, we're doing a hack, + copying the packages into node_modules, so that nexe will include them. +*/ +const basePath = process.cwd() +const packageJson = require(path.join(basePath, '/package.json')) +const outputDirectory = path.join(basePath, './deploy/') +const executableName = process.argv[2] +if (!executableName) { + throw new Error(`Argument for the output executable file name not provided`) +} + +;(async () => { + + log(`Collecting dependencies for ${packageJson.name}...`) + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') + + const packages = JSON.parse(str) + + await mkdirp(basePath + 'node_modules') + + // Copy the packages into node_modules: + const copiedFolders = [] + let ps = [] + for (const package0 of packages) { + if (package0.name.match(/boilerplate/)) continue + if (package0.name.match(packageJson.name)) continue + + log(` Copying: ${package0.name}`) + const target = path.resolve(path.join(basePath, 'node_modules', package0.name)) + + // log(` ${package0.location} -> ${target}`) + ps.push(fseCopy(package0.location, target)) + + copiedFolders.push(target) + } + + await Promise.all(ps) + ps = [] + + // Remove things that arent used, to reduce file size: + log(`Remove unused files...`) + const copiedFiles = [ + ...(await glob(`${basePath}node_modules/@*/app/*`)), + ...(await glob(`${basePath}node_modules/@*/generic/*`)), + ] + + for (const file of copiedFiles) { + if ( + // Only keep these: + !file.match(/package0.json$/) && + !file.match(/node_modules$/) && + !file.match(/dist$/) + ) { + ps.push(rimraf(file)) + } + } + await Promise.all(ps) + ps = [] + + log(`Compiling using nexe...`) + + const nexeOutputPath = path.join(outputDirectory, executableName) + + console.log('nexeOutputPath', nexeOutputPath) + + await nexe.compile({ + input: path.join(basePath, './dist/index.js'), + output: nexeOutputPath, + // build: true, //required to use patches + targets: ['windows-x64-12.18.1'], + }) + + log(`Cleaning up...`) + // Clean up after ourselves: + for (const copiedFolder of copiedFolders) { + await rimraf(copiedFolder) + } + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/apps/appcontainer-node/app/scripts/copy-natives.js b/scripts/copy-natives.js similarity index 56% rename from apps/appcontainer-node/app/scripts/copy-natives.js rename to scripts/copy-natives.js index 763c96fb..3ddad6d8 100644 --- a/apps/appcontainer-node/app/scripts/copy-natives.js +++ b/scripts/copy-natives.js @@ -1,4 +1,6 @@ -const find = require('find'); +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const find = require('find') const os = require('os') const path = require('path') const fs = require('fs-extra') @@ -7,27 +9,36 @@ const arch = os.arch() const platform = os.platform() const prebuildType = process.argv[2] || `${platform}-${arch}` -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} +log('Copying native dependencies...') -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) +const basePath = process.cwd() -console.log(process.argv[2]) +const dirName = path.join(basePath, '../../..') +log(`Running in directory ${dirName} for patform "${prebuildType}"`) + +// log(process.argv[2]) find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { + files.forEach((fullPath) => { if (fullPath.indexOf(dirName) === 0) { const file = fullPath.substr(dirName.length + 1) if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) + log('Copy prebuild binary:', file) fs.copySync(file, path.join('deploy', file)) } } - }); + }) }) + +function isFileForPlatform(filename) { + if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { + return true + } else { + return false + } +} + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/scripts/reset.js b/scripts/reset.js new file mode 100644 index 00000000..e2565a85 --- /dev/null +++ b/scripts/reset.js @@ -0,0 +1,63 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const glob = promisify(require('glob')) +const rimraf = promisify(require('rimraf')) + +/* + Removing all /node_modules and /dist folders +*/ + +const basePath = process.cwd() + +;(async () => { + + log('Gathering files to remove...') + + // Remove things that arent used, to reduce file size: + const searchForFolder = async (name) => { + return [ + ...(await glob(`${basePath}/${name}`)), + ...(await glob(`${basePath}/*/${name}`)), + ...(await glob(`${basePath}/*/*/${name}`)), + ...(await glob(`${basePath}/*/*/*/${name}`)), + ...(await glob(`${basePath}/*/*/*/*/${name}`)), + ] + } + const allFolders = [ + ...(await searchForFolder('node_modules')), + ...(await searchForFolder('dist')), + ...(await searchForFolder('deploy')), + ] + const resultingFolders = [] + for (const folder of allFolders) { + let found = false + for (const resultingFolder of resultingFolders) { + if (folder.match(resultingFolder)) { + found = true + break + } + } + if (!found) { + resultingFolders.push(folder) + } + } + + const rootNodeModules = await glob(`${basePath}/node_modules`) + if (rootNodeModules.length !== 1) throw new Error(`Wrong length of root node_modules (${rootNodeModule.length})`) + const rootNodeModule = rootNodeModules[0] + + log(`Removing ${resultingFolders.length} files...`) + for (const folder of resultingFolders) { + if (folder === rootNodeModule) continue + log(`Removing ${folder}`) + await rimraf(folder) + } + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/scripts/run-sequencially.js b/scripts/run-sequencially.js new file mode 100644 index 00000000..d65024eb --- /dev/null +++ b/scripts/run-sequencially.js @@ -0,0 +1,74 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const cp = require('child_process') +const path = require('path') +// const nexe = require('nexe') +const exec = promisify(cp.exec) +// const spawn = promisify(cp.spawn) +const glob = promisify(require('glob')) + +const fse = require('fs-extra'); +const mkdirp = require('mkdirp'); +const rimraf = promisify(require('rimraf')) + +const fseCopy = promisify(fse.copy) + +/* + Runs a command in all lerna packages, one at a time +*/ +const commands = process.argv.slice(2) + + + +;(async () => { + + log(`Running command "${commands.join(' ')}" in all packages...`) + + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') + + const packages = JSON.parse(str) + + for (const package of packages) { + const cmd = `${commands.join(' ')} --scope=${package.name}` + log(cmd) + + + await new Promise((resolve, reject) => { + const process = cp.exec(cmd, {}) + // const process = cp.spawn(commands[0], [commands.slice(1), `--scope=${package.name}`] ) + process.stdout.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.stderr.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.on('error', (error) => { + reject(error) + }) + process.on('close', (code) => { + if (code === 0) { + resolve() + } else { + reject('Process exited with code '+code) + } + }) + + }) + + + + + } + // log(packages) + + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index 99f16a77..498797a1 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -338,7 +338,7 @@ export class QuantelAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle Date: Tue, 31 Aug 2021 12:40:57 +0200 Subject: [PATCH 62/67] fix: move ProcessHandler to shared/api so that it can be used by all of the applications --- .../packages/generic/src/index.ts | 5 ++++- apps/http-server/packages/generic/src/index.ts | 5 ++++- .../packages/generic/src/connector.ts | 15 ++++++--------- .../packages/generic/src/coreHandler.ts | 4 +--- .../package-manager/packages/generic/src/index.ts | 8 ++++++-- .../packages/generic/src/index.ts | 5 ++++- apps/single-app/app/src/singleApp.ts | 7 +++++-- apps/worker/packages/generic/src/index.ts | 5 ++++- shared/packages/api/src/index.ts | 1 + .../packages/api}/src/process.ts | 6 ++++-- 10 files changed, 39 insertions(+), 22 deletions(-) rename {apps/package-manager/packages/generic => shared/packages/api}/src/process.ts (85%) diff --git a/apps/appcontainer-node/packages/generic/src/index.ts b/apps/appcontainer-node/packages/generic/src/index.ts index bd87a44b..dc4c39fe 100644 --- a/apps/appcontainer-node/packages/generic/src/index.ts +++ b/apps/appcontainer-node/packages/generic/src/index.ts @@ -1,4 +1,4 @@ -import { getAppContainerConfig, setupLogging } from '@shared/api' +import { getAppContainerConfig, ProcessHandler, setupLogging } from '@shared/api' import { AppContainer } from './appContainer' export { AppContainer } from './appContainer' @@ -12,6 +12,9 @@ export async function startProcess(): Promise { logger.info('Starting AppContainer') logger.info('------------------------------------------------------------------') + const process = new ProcessHandler(logger) + process.init(config.process) + const appContainer = new AppContainer(logger, config) appContainer.init().catch(logger.error) diff --git a/apps/http-server/packages/generic/src/index.ts b/apps/http-server/packages/generic/src/index.ts index c9a015eb..a6cf4c55 100644 --- a/apps/http-server/packages/generic/src/index.ts +++ b/apps/http-server/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { PackageProxyServer } from './server' -import { getHTTPServerConfig, setupLogging } from '@shared/api' +import { getHTTPServerConfig, ProcessHandler, setupLogging } from '@shared/api' export { PackageProxyServer } export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('------------------------------------------------------------------') logger.info('Starting HTTP Server') + const process = new ProcessHandler(logger) + process.init(config.process) + const app = new PackageProxyServer(logger, config) app.init().catch((e) => { logger.error(e) diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts index 5b22a75c..5449d6c4 100644 --- a/apps/package-manager/packages/generic/src/connector.ts +++ b/apps/package-manager/packages/generic/src/connector.ts @@ -1,8 +1,7 @@ -import { ClientConnectionOptions, LoggerInstance, PackageManagerConfig } from '@shared/api' +import { ClientConnectionOptions, LoggerInstance, PackageManagerConfig, ProcessHandler } from '@shared/api' import { ExpectationManager, ExpectationManagerServerOptions } from '@shared/expectation-manager' import { CoreHandler, CoreConfig } from './coreHandler' import { PackageManagerHandler } from './packageManager' -import { ProcessHandler } from './process' import chokidar from 'chokidar' import fs from 'fs' import { promisify } from 'util' @@ -30,10 +29,12 @@ export interface DeviceConfig { export class Connector { private packageManagerHandler: PackageManagerHandler private coreHandler: CoreHandler - private _process: ProcessHandler - constructor(private _logger: LoggerInstance, private config: PackageManagerConfig) { - this._process = new ProcessHandler(this._logger) + constructor( + private _logger: LoggerInstance, + private config: PackageManagerConfig, + private _process: ProcessHandler + ) { this.coreHandler = new CoreHandler(this._logger, this.config.packageManager) const packageManagerServerOptions: ExpectationManagerServerOptions = @@ -62,10 +63,6 @@ export class Connector { public async init(): Promise { try { - this._logger.info('Initializing Process...') - this._process.init(this.config.process) - this._logger.info('Process initialized') - this._logger.info('Initializing Core...') await this.coreHandler.init(this.config, this._process) this._logger.info('Core initialized') diff --git a/apps/package-manager/packages/generic/src/coreHandler.ts b/apps/package-manager/packages/generic/src/coreHandler.ts index 2c985d89..6043f55a 100644 --- a/apps/package-manager/packages/generic/src/coreHandler.ts +++ b/apps/package-manager/packages/generic/src/coreHandler.ts @@ -9,9 +9,7 @@ import { import { DeviceConfig } from './connector' import fs from 'fs' -import { LoggerInstance, PackageManagerConfig } from '@shared/api' - -import { ProcessHandler } from './process' +import { LoggerInstance, PackageManagerConfig, ProcessHandler } from '@shared/api' import { PACKAGE_MANAGER_DEVICE_CONFIG } from './configManifest' import { PackageManagerHandler } from './packageManager' diff --git a/apps/package-manager/packages/generic/src/index.ts b/apps/package-manager/packages/generic/src/index.ts index 5d13f25d..6df38b98 100644 --- a/apps/package-manager/packages/generic/src/index.ts +++ b/apps/package-manager/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { Connector, Config } from './connector' -import { getPackageManagerConfig, LoggerInstance, setupLogging } from '@shared/api' +import { getPackageManagerConfig, LoggerInstance, ProcessHandler, setupLogging } from '@shared/api' export { Connector, Config } export function startProcess(startInInternalMode?: boolean): { logger: LoggerInstance; connector: Connector } { @@ -15,7 +15,11 @@ export function startProcess(startInInternalMode?: boolean): { logger: LoggerIns config.packageManager.accessUrl = null config.packageManager.workforceURL = null } - const connector = new Connector(logger, config) + + const process = new ProcessHandler(logger) + process.init(config.process) + + const connector = new Connector(logger, config, process) logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) logger.info('------------------------------------------------------------------') diff --git a/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts b/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts index 3b22e849..467ddbf8 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts +++ b/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { QuantelHTTPTransformerProxy } from './server' -import { getQuantelHTTPTransformerProxyConfig, setupLogging } from '@shared/api' +import { getQuantelHTTPTransformerProxyConfig, ProcessHandler, setupLogging } from '@shared/api' export { QuantelHTTPTransformerProxy } export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('------------------------------------------------------------------') logger.info('Starting Quantel HTTP Transformer Proxy Server') + const process = new ProcessHandler(logger) + process.init(config.process) + const app = new QuantelHTTPTransformerProxy(logger, config) app.init().catch((e) => { logger.error(e) diff --git a/apps/single-app/app/src/singleApp.ts b/apps/single-app/app/src/singleApp.ts index 3582a42a..a5abfcf2 100644 --- a/apps/single-app/app/src/singleApp.ts +++ b/apps/single-app/app/src/singleApp.ts @@ -6,7 +6,7 @@ import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/ge import * as PackageManager from '@package-manager/generic' import * as Workforce from '@shared/workforce' import * as AppConatainerNode from '@appcontainer-node/generic' -import { getSingleAppConfig, setupLogging } from '@shared/api' +import { getSingleAppConfig, ProcessHandler, setupLogging } from '@shared/api' // import { MessageToAppContainerSpinUp, MessageToAppContainerType } from './__api' export async function startSingleApp(): Promise { @@ -18,6 +18,9 @@ export async function startSingleApp(): Promise { config.packageManager.workforceURL = null // Filled in later config.workforce.port = 0 // 0 = Set the workforce port to whatever is available + const process = new ProcessHandler(logger) + process.init(config.process) + logger.info('------------------------------------------------------------------') logger.info('Starting Package Manager - Single App') @@ -41,7 +44,7 @@ export async function startSingleApp(): Promise { await appContainer.init() logger.info('Initializing Package Manager Connector') - const connector = new PackageManager.Connector(logger, config) + const connector = new PackageManager.Connector(logger, config, process) const expectationManager = connector.getExpectationManager() logger.info('Initializing HTTP proxy Server') diff --git a/apps/worker/packages/generic/src/index.ts b/apps/worker/packages/generic/src/index.ts index 55d50ad7..58444c44 100644 --- a/apps/worker/packages/generic/src/index.ts +++ b/apps/worker/packages/generic/src/index.ts @@ -1,4 +1,4 @@ -import { getWorkerConfig, setupLogging } from '@shared/api' +import { getWorkerConfig, ProcessHandler, setupLogging } from '@shared/api' import { WorkerAgent } from '@shared/worker' export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('Starting Worker') logger.info('------------------------------------------------------------------') + const process = new ProcessHandler(logger) + process.init(config.process) + const workforce = new WorkerAgent(logger, config) workforce.init().catch(logger.error) diff --git a/shared/packages/api/src/index.ts b/shared/packages/api/src/index.ts index 77bda134..549b136d 100644 --- a/shared/packages/api/src/index.ts +++ b/shared/packages/api/src/index.ts @@ -7,6 +7,7 @@ export * from './lib' export * from './logger' export * from './methods' export * from './packageContainerApi' +export * from './process' export * from './status' export * from './websocketClient' export { MessageBase, MessageReply, MessageIdentifyClient, Hook } from './websocketConnection' diff --git a/apps/package-manager/packages/generic/src/process.ts b/shared/packages/api/src/process.ts similarity index 85% rename from apps/package-manager/packages/generic/src/process.ts rename to shared/packages/api/src/process.ts index c9dc53b4..9df051b2 100644 --- a/apps/package-manager/packages/generic/src/process.ts +++ b/shared/packages/api/src/process.ts @@ -1,6 +1,8 @@ -import { LoggerInstance } from '@shared/api' +import { ProcessConfig } from './config' import fs from 'fs' -import { ProcessConfig } from './connector' +import { LoggerInstance } from './logger' + +// export function setupProcess(config: ProcessConfig): void {} export class ProcessHandler { logger: LoggerInstance From c0b985c385134676e59300d988a7258d4e0959f3 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 31 Aug 2021 12:42:38 +0200 Subject: [PATCH 63/67] fix: Don't try to set up monitors or cronjobs if no monitors are defined to be set up --- .../src/expectationManager.ts | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 39e0ec9b..e397a521 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -1389,40 +1389,44 @@ export class ExpectationManager { if (trackedPackageContainer.currentWorker) { const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - if (!trackedPackageContainer.monitorIsSetup) { - const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( - trackedPackageContainer.packageContainer - ) + if (Object.keys(trackedPackageContainer.packageContainer.monitors).length !== 0) { + if (!trackedPackageContainer.monitorIsSetup) { + const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( + trackedPackageContainer.packageContainer + ) - trackedPackageContainer.status.monitors = {} - if (monitorSetup.success) { - trackedPackageContainer.monitorIsSetup = true - for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { - trackedPackageContainer.status.monitors[monitorId] = { - label: monitor.label, - reason: { - user: 'Starting up', - tech: 'Starting up', - }, + trackedPackageContainer.status.monitors = {} + if (monitorSetup.success) { + trackedPackageContainer.monitorIsSetup = true + for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { + trackedPackageContainer.status.monitors[monitorId] = { + label: monitor.label, + reason: { + user: 'Starting up', + tech: 'Starting up', + }, + } } + } else { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, + tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + }) } - } else { + } + } + if (Object.keys(trackedPackageContainer.packageContainer.cronjobs).length !== 0) { + const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( + trackedPackageContainer.packageContainer + ) + if (!cronJobStatus.success) { this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, - tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + user: 'Cron job not completed: ' + cronJobStatus.reason.user, + tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, }) + continue } } - const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( - trackedPackageContainer.packageContainer - ) - if (!cronJobStatus.success) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: 'Cron job not completed: ' + cronJobStatus.reason.user, - tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, - }) - continue - } } } } From 2aaebc092f96a50cf6403bc1423ffdf881e1cd85 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Tue, 31 Aug 2021 12:43:20 +0200 Subject: [PATCH 64/67] chore: fix script so that it takes scope into account --- scripts/run-sequencially.js | 74 +++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/scripts/run-sequencially.js b/scripts/run-sequencially.js index d65024eb..8e9a5776 100644 --- a/scripts/run-sequencially.js +++ b/scripts/run-sequencially.js @@ -17,52 +17,62 @@ const fseCopy = promisify(fse.copy) /* Runs a command in all lerna packages, one at a time */ -const commands = process.argv.slice(2) +let commands = process.argv.slice(2) +const lastCommand = commands[commands.length-1] +;(async () => { + let scopes = [] + if (lastCommand.match(/scope/)) { -;(async () => { + log(`Running command "${commands.join(' ')}" for ${lastCommand}`) + + scopes = [lastCommand] + commands = commands.slice(0, -1) + } else { - log(`Running command "${commands.join(' ')}" in all packages...`) + log(`Running command "${commands.join(' ')}" in all packages...`) - // List all Lerna packages: - const list = await exec('yarn lerna list -a --json') - const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') - const packages = JSON.parse(str) + const packages = JSON.parse(str) - for (const package of packages) { - const cmd = `${commands.join(' ')} --scope=${package.name}` - log(cmd) + scopes = packages.map((p) => `--scope=${p.name}`) + } + for (const scope of scopes) { + const cmd = `${commands.join(' ')} ${scope}` + log(cmd) - await new Promise((resolve, reject) => { - const process = cp.exec(cmd, {}) - // const process = cp.spawn(commands[0], [commands.slice(1), `--scope=${package.name}`] ) - process.stdout.on('data', (data) => { - log((data+'').trimEnd() ) - }) - process.stderr.on('data', (data) => { - log((data+'').trimEnd() ) - }) - process.on('error', (error) => { - reject(error) - }) - process.on('close', (code) => { - if (code === 0) { - resolve() - } else { - reject('Process exited with code '+code) - } - }) + await new Promise((resolve, reject) => { + const process = cp.exec(cmd, {}) + // const process = cp.spawn(commands[0], [commands.slice(1), `--scope=${package.name}`] ) + process.stdout.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.stderr.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.on('error', (error) => { + reject(error) + }) + process.on('close', (code) => { + if (code === 0) { + resolve() + } else { + reject('Process exited with code '+code) + } + }) - }) + }) - } - // log(packages) + } + // log(packages) log(`...done!`) From 575b3ac277330f070bedb5c1fe63f38a1601dd73 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 2 Sep 2021 06:58:24 +0200 Subject: [PATCH 65/67] chore: adjust package label for quantel clips --- .../packages/generic/src/expectationGenerator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index 953e7f46..d616c131 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -344,6 +344,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E const expWrapQuantelClip = expWrap as ExpectedPackageWrapQuantel const content = expWrapQuantelClip.expectedPackage.content + const label = content.title && content.guid ? `${content.title} (${content.guid})` : content.title || content.guid const exp: Expectation.QuantelClipCopy = { id: '', // set later priority: expWrap.priority * 10 || 0, @@ -357,7 +358,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E ], statusReport: { - label: `Copy Quantel clip ${content.title || content.guid}`, + label: `Copy Quantel clip ${label}`, description: `Copy Quantel clip ${content.title || content.guid} to server for "${ expWrapQuantelClip.playoutDeviceId }", from ${expWrapQuantelClip.sources.map((source) => `"${source.label}"`).join(', ')}`, From f53e6a48500c5b7de798148100df35750ba0e22f Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 3 Sep 2021 08:53:02 +0200 Subject: [PATCH 66/67] fix: fix an issue where an error in a packageContainer.runCronjob could halt the whole evaulateExpectation loop. --- .../src/expectationManager.ts | 165 +++++++++--------- .../src/worker/accessorHandlers/fileShare.ts | 8 +- .../src/worker/accessorHandlers/http.ts | 11 +- .../src/worker/accessorHandlers/httpProxy.ts | 11 +- .../accessorHandlers/lib/FileHandler.ts | 5 +- .../worker/accessorHandlers/localFolder.ts | 8 +- 6 files changed, 113 insertions(+), 95 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index e397a521..9bd4aefe 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -1328,105 +1328,112 @@ export class ExpectationManager { } private async _evaluateAllTrackedPackageContainers(): Promise { for (const trackedPackageContainer of Object.values(this.trackedPackageContainers)) { - if (trackedPackageContainer.isUpdated) { - // If the packageContainer was newly updated, reset and set up again: + try { + if (trackedPackageContainer.isUpdated) { + // If the packageContainer was newly updated, reset and set up again: + if (trackedPackageContainer.currentWorker) { + const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] + const disposeMonitorResult = await workerAgent.api.disposePackageContainerMonitors( + trackedPackageContainer.packageContainer + ) + if (!disposeMonitorResult.success) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to remove monitor, due to ${disposeMonitorResult.reason.user}`, + tech: `Unable to dispose monitor: ${disposeMonitorResult.reason.tech}`, + }) + continue // Break further execution for this PackageContainer + } + trackedPackageContainer.currentWorker = null + } + trackedPackageContainer.isUpdated = false + } + if (trackedPackageContainer.currentWorker) { - const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - const disposeMonitorResult = await workerAgent.api.disposePackageContainerMonitors( - trackedPackageContainer.packageContainer + // Check that the worker still exists: + if (!this.workerAgents[trackedPackageContainer.currentWorker]) { + trackedPackageContainer.currentWorker = null + } + } + if (!trackedPackageContainer.currentWorker) { + // Find a worker that supports this PackageContainer + + let notSupportReason: Reason | null = null + await Promise.all( + Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { + const support = await workerAgent.api.doYouSupportPackageContainer( + trackedPackageContainer.packageContainer + ) + if (!trackedPackageContainer.currentWorker) { + if (support.support) { + trackedPackageContainer.currentWorker = workerId + } else { + notSupportReason = support.reason + } + } + }) ) - if (!disposeMonitorResult.success) { + if (!trackedPackageContainer.currentWorker) { + notSupportReason = { + user: 'Found no worker that supports this packageContainer', + tech: 'Found no worker that supports this packageContainer', + } + } + if (notSupportReason) { this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: `Unable to remove monitor, due to ${disposeMonitorResult.reason.user}`, - tech: `Unable to dispose monitor: ${disposeMonitorResult.reason.tech}`, + user: `Unable to handle PackageContainer, due to: ${notSupportReason.user}`, + tech: `Unable to handle PackageContainer, due to: ${notSupportReason.tech}`, }) continue // Break further execution for this PackageContainer } - trackedPackageContainer.currentWorker = null } - trackedPackageContainer.isUpdated = false - } - if (trackedPackageContainer.currentWorker) { - // Check that the worker still exists: - if (!this.workerAgents[trackedPackageContainer.currentWorker]) { - trackedPackageContainer.currentWorker = null - } - } - if (!trackedPackageContainer.currentWorker) { - // Find a worker that supports this PackageContainer + if (trackedPackageContainer.currentWorker) { + const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - let notSupportReason: Reason | null = null - await Promise.all( - Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { - const support = await workerAgent.api.doYouSupportPackageContainer( - trackedPackageContainer.packageContainer - ) - if (!trackedPackageContainer.currentWorker) { - if (support.support) { - trackedPackageContainer.currentWorker = workerId + if (Object.keys(trackedPackageContainer.packageContainer.monitors).length !== 0) { + if (!trackedPackageContainer.monitorIsSetup) { + const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( + trackedPackageContainer.packageContainer + ) + + trackedPackageContainer.status.monitors = {} + if (monitorSetup.success) { + trackedPackageContainer.monitorIsSetup = true + for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { + trackedPackageContainer.status.monitors[monitorId] = { + label: monitor.label, + reason: { + user: 'Starting up', + tech: 'Starting up', + }, + } + } } else { - notSupportReason = support.reason + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, + tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + }) } } - }) - ) - if (!trackedPackageContainer.currentWorker) { - notSupportReason = { - user: 'Found no worker that supports this packageContainer', - tech: 'Found no worker that supports this packageContainer', } - } - if (notSupportReason) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: `Unable to handle PackageContainer, due to: ${notSupportReason.user}`, - tech: `Unable to handle PackageContainer, due to: ${notSupportReason.tech}`, - }) - continue // Break further execution for this PackageContainer - } - } - - if (trackedPackageContainer.currentWorker) { - const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - - if (Object.keys(trackedPackageContainer.packageContainer.monitors).length !== 0) { - if (!trackedPackageContainer.monitorIsSetup) { - const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( + if (Object.keys(trackedPackageContainer.packageContainer.cronjobs).length !== 0) { + const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( trackedPackageContainer.packageContainer ) - - trackedPackageContainer.status.monitors = {} - if (monitorSetup.success) { - trackedPackageContainer.monitorIsSetup = true - for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { - trackedPackageContainer.status.monitors[monitorId] = { - label: monitor.label, - reason: { - user: 'Starting up', - tech: 'Starting up', - }, - } - } - } else { + if (!cronJobStatus.success) { this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, - tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + user: 'Cron job not completed: ' + cronJobStatus.reason.user, + tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, }) + continue } } } - if (Object.keys(trackedPackageContainer.packageContainer.cronjobs).length !== 0) { - const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( - trackedPackageContainer.packageContainer - ) - if (!cronJobStatus.success) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, { - user: 'Cron job not completed: ' + cronJobStatus.reason.user, - tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, - }) - continue - } - } + } catch (err) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: 'Internal Error', + tech: `Unhandled Error: ${err}`, + }) } } } diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index a5f8650a..b280713f 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -2,7 +2,7 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' -import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import { WindowsWorker } from '../workers/windowsWorker/windowsWorker' import networkDrive from 'windows-network-drive' @@ -287,18 +287,20 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] + let badReason: Reason | null = null for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return { success: true } + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } async setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index 50f61d35..5fd464b9 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -6,7 +6,7 @@ import { PutPackageHandler, AccessorHandlerResult, } from './genericHandle' -import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' import FormData from 'form-data' @@ -140,19 +140,21 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + let badReason: Reason | null = null const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return { success: true } + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } async setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation @@ -302,7 +304,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + async removeDuePackages(): Promise { let packagesToRemove = await this.getPackagesToRemove() const removedFilePaths: string[] = [] @@ -336,6 +338,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { const result = await fetch(url, { diff --git a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts index d312ba21..64dd82ab 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/httpProxy.ts @@ -6,7 +6,7 @@ import { PutPackageHandler, AccessorHandlerResult, } from './genericHandle' -import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' import FormData from 'form-data' @@ -168,19 +168,21 @@ export class HTTPProxyAccessorHandle extends GenericAccessorHandle { + let badReason: Reason | null = null const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return { success: true } + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } async setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation @@ -330,7 +332,7 @@ export class HTTPProxyAccessorHandle extends GenericAccessorHandle { + async removeDuePackages(): Promise { let packagesToRemove = await this.getPackagesToRemove() const removedFilePaths: string[] = [] @@ -364,6 +366,7 @@ export class HTTPProxyAccessorHandle extends GenericAccessorHandle { const result = await fetch(url, { diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index 7871ec9e..a184f9a9 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -1,7 +1,7 @@ import path from 'path' import { promisify } from 'util' import fs from 'fs' -import { Expectation, hashObj, literal, PackageContainerExpectation, assertNever } from '@shared/api' +import { Expectation, hashObj, literal, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import chokidar from 'chokidar' import { GenericWorker } from '../../worker' import { Accessor, AccessorOnPackage, ExpectedPackage } from '@sofie-automation/blueprints-integration' @@ -74,7 +74,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso } } /** Remove any packages that are due for removal */ - async removeDuePackages(): Promise { + async removeDuePackages(): Promise { let packagesToRemove = await this.getPackagesToRemove() const removedFilePaths: string[] = [] @@ -107,6 +107,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso if (changed) { await this.storePackagesToRemove(packagesToRemove) } + return null } /** Unlink (remove) a file, if it exists. */ async unlinkIfExists(filePath: string): Promise { diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index 242bc116..edec1184 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -3,7 +3,7 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' -import { Expectation, PackageContainerExpectation, assertNever } from '@shared/api' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import { GenericFileAccessorHandle, LocalFolderAccessorHandleType } from './lib/FileHandler' @@ -235,19 +235,21 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand await this.unlinkIfExists(this.metadataPath) } async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + let badReason: Reason | null = null const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return { success: true } + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } async setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation From 8f8473909d9200fab5476cda1c4f99c686531185 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 3 Sep 2021 08:53:43 +0200 Subject: [PATCH 67/67] fix: Improve status reasons --- .../src/expectationManager.ts | 27 ++++++++++++------- .../windowsWorker/expectationHandlers/lib.ts | 20 +++++++++++++- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 9bd4aefe..960e159b 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -724,7 +724,6 @@ export class ExpectationManager { try { if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW) { // Check which workers might want to handle it: - // Reset properties: // trackedExp.availableWorkers = [] trackedExp.status = {} @@ -779,17 +778,25 @@ export class ExpectationManager { } } else { if (!Object.keys(trackedExp.queriedWorkers).length) { - trackedExp.noAvailableWorkersReason = { - user: 'No workers registered (this is likely a configuration issue)', - tech: 'No workers registered', + if (!Object.keys(this.workerAgents).length) { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `No Workers available (this is likely a configuration issue)`, + tech: `No Workers available`, + }) + } else { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `No Workers available (this is likely a configuration issue)`, + tech: `No Workers queried, ${Object.keys(this.workerAgents).length} available`, + }) } + } else { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `Found no workers who supports this Expectation, due to: ${trackedExp.noAvailableWorkersReason.user}`, + tech: `Found no workers who supports this Expectation: "${ + trackedExp.noAvailableWorkersReason.tech + }", have asked workers: [${Object.keys(trackedExp.queriedWorkers).join(',')}]`, + }) } - this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { - user: `Found no workers who supports this Expectation, due to: ${trackedExp.noAvailableWorkersReason.user}`, - tech: `Found no workers who supports this Expectation: "${ - trackedExp.noAvailableWorkersReason.tech - }", have asked ${Object.keys(trackedExp.queriedWorkers).join(',')}`, - }) } } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { // Check if the expectation is ready to start: diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index c580c582..4e81c090 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -18,6 +18,15 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( let accessSourcePackageContainer: ReturnType // Check that we have access to the packageContainers if (checks.sources !== undefined) { + if (checks.sources.length === 0) { + return { + support: false, + reason: { + user: `No sources configured`, + tech: `No sources configured`, + }, + } + } accessSourcePackageContainer = findBestPackageContainerWithAccessToPackage(genericWorker, checks.sources) if (!accessSourcePackageContainer) { return { @@ -25,7 +34,7 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( reason: { user: `There is an issue with the configuration of the Worker, it doesn't have access to any of the source PackageContainers`, tech: `Worker doesn't have access to any of the source packageContainers (${checks.sources - .map((o) => o.containerId) + .map((o) => `${o.containerId} "${o.label}"`) .join(', ')})`, }, } @@ -34,6 +43,15 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( let accessTargetPackageContainer: ReturnType if (checks.targets !== undefined) { + if (checks.targets.length === 0) { + return { + support: false, + reason: { + user: `No targets configured`, + tech: `No targets configured`, + }, + } + } accessTargetPackageContainer = findBestPackageContainerWithAccessToPackage(genericWorker, checks.targets) if (!accessTargetPackageContainer) { return {