From c0c061703f7e8f0f10262295ff8fa822350a31dc Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 14 Mar 2024 16:14:25 +0100 Subject: [PATCH 01/30] feat: add a configurable criticalWorkerPoolSize --- .../packages/generic/src/connector.ts | 3 ++- .../packages/generic/src/packageManager.ts | 4 +++- shared/packages/api/src/config.ts | 7 +++++++ .../evaluateExpectationStates/new.ts | 3 ++- .../src/expectationManager.ts | 1 + .../src/internalManager/internalManager.ts | 2 ++ .../lib/trackedWorkerAgents.ts | 19 +++++++++++++++++-- 7 files changed, 34 insertions(+), 5 deletions(-) diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts index 8c382d4a..20139b1b 100644 --- a/apps/package-manager/packages/generic/src/connector.ts +++ b/apps/package-manager/packages/generic/src/connector.ts @@ -82,7 +82,8 @@ export class Connector { config.packageManager.accessUrl || undefined, workForceConnectionOptions, config.packageManager.concurrency, - config.packageManager.chaosMonkey + config.packageManager.chaosMonkey, + config.packageManager.criticalWorkerPoolSize ) } diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index ff55c2ee..43df13be 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -115,7 +115,8 @@ export class PackageManagerHandler { private serverAccessUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, concurrency: number | undefined, - chaosMonkey: boolean + chaosMonkey: boolean, + criticalWorkerPoolSize: number | undefined ) { this.logger = logger.category('PackageManager') this.callbacksHandler = new ExpectationManagerCallbacksHandler(this.logger, this) @@ -129,6 +130,7 @@ export class PackageManagerHandler { this.callbacksHandler, { chaosMonkey: chaosMonkey, + criticalWorkerPoolSize: criticalWorkerPoolSize, constants: { PARALLEL_CONCURRENCY: concurrency, }, diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 46941fce..b7c562cf 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -119,6 +119,11 @@ const packageManagerArguments = defineArguments({ default: process.env.CHAOS_MONKEY === '1', describe: 'If true, enables the "chaos monkey"-feature, which will randomly kill processes every few seconds', }, + criticalWorkerPoolSize: { + type: 'number', + default: 0, + describe: 'Percentage of Workers reserved for fulfilling playout-critical expectations', + }, concurrency: { type: 'number', default: parseInt(process.env.CONCURRENCY || '', 10) || undefined, @@ -373,6 +378,7 @@ export interface PackageManagerConfig { watchFiles: boolean noCore: boolean chaosMonkey: boolean + criticalWorkerPoolSize?: number concurrency?: number } } @@ -400,6 +406,7 @@ export async function getPackageManagerConfig(): Promise { watchFiles: argv.watchFiles, noCore: argv.noCore, chaosMonkey: argv.chaosMonkey, + criticalWorkerPoolSize: argv.criticalWorkerPoolSize, concurrency: argv.concurrency, }, } diff --git a/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts b/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts index 49bf9f6b..41642b5b 100644 --- a/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts +++ b/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts @@ -16,7 +16,8 @@ export async function evaluateExpectationStateNew({ manager, tracker, trackedExp trackedExp.status = {} const { hasQueriedAnyone, workerCount } = await manager.workerAgents.updateAvailableWorkersForExpectation( - trackedExp + trackedExp, + manager.criticalWorkerPoolSize ) const availableWorkersCount = trackedExp.availableWorkers.size diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index f4ff9f3a..59157ad8 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -136,6 +136,7 @@ export class ExpectationManager { export interface ExpectationManagerOptions { constants?: Partial chaosMonkey?: boolean + criticalWorkerPoolSize?: number } export type ExpectationManagerServerOptions = diff --git a/shared/packages/expectationManager/src/internalManager/internalManager.ts b/shared/packages/expectationManager/src/internalManager/internalManager.ts index 4270f49b..cc467b27 100644 --- a/shared/packages/expectationManager/src/internalManager/internalManager.ts +++ b/shared/packages/expectationManager/src/internalManager/internalManager.ts @@ -43,6 +43,7 @@ export class InternalManager { public statuses: ManagerStatusReporter private enableChaosMonkey = false + public criticalWorkerPoolSize = 0 private managerWatchdog: ManagerStatusWatchdog public statusReport: StatusReportCache @@ -83,6 +84,7 @@ export class InternalManager { this.statusReport = new StatusReportCache(this) this.enableChaosMonkey = options?.chaosMonkey ?? false + this.criticalWorkerPoolSize = options?.criticalWorkerPoolSize ?? 0 } /** Initialize the ExpectationManager. This method is should be called shortly after the class has been instantiated. */ diff --git a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts index 88ee2513..a6374b28 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts @@ -36,7 +36,10 @@ export class TrackedWorkerAgents { * Asks the Workers if they support a certain Expectation. * Updates trackedExp.availableWorkers to reflect the result. */ - public async updateAvailableWorkersForExpectation(trackedExp: TrackedExpectation): Promise<{ + public async updateAvailableWorkersForExpectation( + trackedExp: TrackedExpectation, + criticalWorkerPoolSize: number + ): Promise<{ hasQueriedAnyone: boolean workerCount: number }> { @@ -47,8 +50,16 @@ export class TrackedWorkerAgents { let hasQueriedAnyone = false await Promise.all( - workerAgents.map(async ({ workerId, workerAgent }) => { + workerAgents.map(async ({ workerId, workerAgent }, index, allWorkers) => { if (!workerAgent.connected) return + if ( + !trackedExp.exp.statusReport.requiredForPlayout && + // 1 - criticalWorkerPoolSize, so that the "criticalPool" consists of the "freshest" workers + index / (allWorkers.length - 1) > 1 - criticalWorkerPoolSize + ) { + trackedExp.availableWorkers.delete(workerId) + return + } // Only ask each worker once, or after a certain time has passed: const queriedWorker = trackedExp.queriedWorkers.get(workerId) @@ -104,6 +115,10 @@ export class TrackedWorkerAgents { const workerIds = Array.from(trackedExp.availableWorkers.keys()) + if (trackedExp.exp.statusReport.requiredForPlayout) { + // TODO: decimate the workerIds + } + let noCostReason: Reason = { user: `${workerIds.length} workers are currently busy`, tech: `${workerIds.length} busy, ${trackedExp.queriedWorkers.size} queried`, From 51c66077a4eaae947a42173396b217543ff5cb81 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Fri, 15 Mar 2024 13:19:02 +0100 Subject: [PATCH 02/30] chore: remove unnecessary mkdirp dependency --- .../http-server/packages/generic/package.json | 2 -- .../generic/src/storage/fileStorage.ts | 6 ++--- .../packages/generic/package.json | 2 -- package.json | 1 - scripts/prepare-for-build32.js | 3 +-- shared/packages/worker/package.json | 1 - .../src/worker/accessorHandlers/atem.ts | 2 +- .../src/worker/accessorHandlers/fileShare.ts | 4 ++-- .../worker/accessorHandlers/localFolder.ts | 4 ++-- .../expectationHandlers/lib/ffmpeg.ts | 6 ++--- tests/internal-tests/src/__mocks__/fs.ts | 16 ++++++++++--- tests/internal-tests/src/__mocks__/mkdirp.ts | 22 ----------------- .../src/__tests__/basic.spec.ts | 10 ++------ .../src/__tests__/issues.spec.ts | 5 +--- yarn.lock | 24 ------------------- 15 files changed, 28 insertions(+), 80 deletions(-) delete mode 100644 tests/internal-tests/src/__mocks__/mkdirp.ts diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index 19ba4200..c9c526c0 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -17,7 +17,6 @@ "koa-range": "^0.3.0", "koa-router": "^8.0.8", "mime-types": "^2.1.28", - "mkdirp": "^3.0.1", "pechkin": "^1.0.1", "pretty-bytes": "^5.5.0", "tslib": "^2.1.0", @@ -32,7 +31,6 @@ "@types/koa-router": "^7.4.0", "@types/koa__cors": "^4.0.0", "@types/mime-types": "^2.1.0", - "@types/mkdirp": "^1.0.1", "@types/node": "^14.14.31", "@types/underscore": "^1.10.24", "@types/yargs": "^17.0.24" diff --git a/apps/http-server/packages/generic/src/storage/fileStorage.ts b/apps/http-server/packages/generic/src/storage/fileStorage.ts index 1ff1e3a8..117b1b18 100644 --- a/apps/http-server/packages/generic/src/storage/fileStorage.ts +++ b/apps/http-server/packages/generic/src/storage/fileStorage.ts @@ -2,7 +2,6 @@ import fs from 'fs' import path from 'path' import { promisify } from 'util' import mime from 'mime-types' -import { mkdirp } from 'mkdirp' import prettyBytes from 'pretty-bytes' import { asyncPipe, CTX, CTXPost } from '../lib' import { HTTPServerConfig, LoggerInstance } from '@sofie-package-manager/api' @@ -17,6 +16,7 @@ const fsRmDir = promisify(fs.rmdir) const fsReaddir = promisify(fs.readdir) const fsLstat = promisify(fs.lstat) const fsWriteFile = promisify(fs.writeFile) +const fsMkDir = promisify(fs.mkdir) type FileInfo = { found: true @@ -43,7 +43,7 @@ export class FileStorage extends Storage { } async init(): Promise { - await mkdirp(this._basePath) + await fsMkDir(this._basePath, { recursive: true }) } async listPackages(ctx: CTX): Promise { @@ -154,7 +154,7 @@ export class FileStorage extends Storage { ): Promise { const fullPath = path.join(this._basePath, paramPath) - await mkdirp(path.dirname(fullPath)) + await fsMkDir(path.dirname(fullPath), { recursive: true }) const exists = await this.exists(fullPath) if (exists) await fsUnlink(fullPath) diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index 9f081c2b..57aa4c87 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -19,7 +19,6 @@ "koa-ratelimit": "^5.0.1", "koa-router": "^12.0.0", "mime-types": "^2.1.28", - "mkdirp": "^3.0.1", "pretty-bytes": "^5.5.0", "tslib": "^2.1.0", "underscore": "^1.12.0", @@ -35,7 +34,6 @@ "@types/koa-router": "^7.4.4", "@types/koa__cors": "^4.0.0", "@types/mime-types": "^2.1.0", - "@types/mkdirp": "^1.0.1", "@types/node": "^14.14.31", "@types/underscore": "^1.10.24", "@types/xml2js": "^0.4.7", diff --git a/package.json b/package.json index 1376e90e..1b9bb912 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "json-schema-to-typescript": "^10.1.5", "lerna": "^6.6.1", "lint-staged": "^15.2.2", - "mkdirp": "^3.0.1", "node-fetch": "^2.6.9", "pkg": "^5.8.0", "rimraf": "^5.0.5", diff --git a/scripts/prepare-for-build32.js b/scripts/prepare-for-build32.js index b63fd9c9..762dd83b 100644 --- a/scripts/prepare-for-build32.js +++ b/scripts/prepare-for-build32.js @@ -7,7 +7,6 @@ const os = require('os') const exec = promisify(cp.exec) const fse = require('fs-extra') -const { mkdirp } = require('mkdirp') const fseCopy = promisify(fse.copy) @@ -36,7 +35,7 @@ const packageJson = require(path.join(basePath, '/package.json')) const packages = JSON.parse(str) - await mkdirp(path.join(basePath, 'node_modules')) + await fse.mkdirp(path.join(basePath, 'node_modules')) // Copy the packages into node_modules: diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json index 967c5cf1..8104df38 100644 --- a/shared/packages/worker/package.json +++ b/shared/packages/worker/package.json @@ -29,7 +29,6 @@ "atem-connection": "^3.2.0", "deep-diff": "^1.0.2", "form-data": "^4.0.0", - "mkdirp": "^3.0.1", "node-fetch": "^2.6.1", "proper-lockfile": "^4.1.2", "tmp": "~0.2.1", diff --git a/shared/packages/worker/src/worker/accessorHandlers/atem.ts b/shared/packages/worker/src/worker/accessorHandlers/atem.ts index 03d91620..a7b05727 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/atem.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/atem.ts @@ -510,7 +510,7 @@ export async function createTGASequence(inputFile: string, opts?: { width: numbe if (opts) { args.push('-vf', `scale=${opts.width}:${opts.height}`) } - args.push(outputFile) + args.push(escapeFilePath(outputFile)) return ffmpeg(args) } diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 7586833c..436eb3e6 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -37,7 +37,6 @@ import { MonitorInProgress } from '../lib/monitorInProgress' import { MAX_EXEC_BUFFER } from '../lib/lib' import { defaultCheckHandleRead, defaultCheckHandleWrite } from './lib/lib' import * as path from 'path' -import { mkdirp } from 'mkdirp' const fsStat = promisify(fs.stat) const fsAccess = promisify(fs.access) @@ -46,6 +45,7 @@ const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) const fsRename = promisify(fs.rename) +const fsMkDir = promisify(fs.mkdir) const pExec = promisify(exec) const PREPARE_FILE_ACCESS_TIMEOUT = INNER_ACTION_TIMEOUT * 0.5 @@ -269,7 +269,7 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath - await mkdirp(path.dirname(fullPath)) // Create folder if it doesn't exist + await fsMkDir(path.dirname(fullPath), { recursive: true }) // Create folder if it doesn't exist // Remove the file if it already exists: if (await this.unlinkIfExists(fullPath)) diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index 97a5aecc..38fc3e0c 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -31,7 +31,6 @@ import { GenericFileAccessorHandle, LocalFolderAccessorHandleType } from './lib/ import { MonitorInProgress } from '../lib/monitorInProgress' import { compareResourceIds } from '../workers/windowsWorker/lib/lib' import { defaultCheckHandleRead, defaultCheckHandleWrite } from './lib/lib' -import { mkdirp } from 'mkdirp' const fsStat = promisify(fs.stat) const fsAccess = promisify(fs.access) @@ -40,6 +39,7 @@ const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) const fsRename = promisify(fs.rename) +const fsMkDir = promisify(fs.mkdir) /** Accessor handle for accessing files in a local folder */ export class LocalFolderAccessorHandle extends GenericFileAccessorHandle { @@ -216,7 +216,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath - await mkdirp(path.dirname(fullPath)) // Create folder if it doesn't exist + await fsMkDir(path.dirname(fullPath), { recursive: true }) // Create folder if it doesn't exist // Remove the file if it exists: if (await this.unlinkIfExists(fullPath)) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts index 81358dcf..c8b11533 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts @@ -1,6 +1,6 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process' import path from 'path' -import { mkdirp } from 'mkdirp' +import { mkdir as fsMkDir } from 'fs/promises' import { isFileShareAccessorHandle, isHTTPProxyAccessorHandle, @@ -103,11 +103,11 @@ export async function spawnFFMpeg( let pipeStdOut = false if (isLocalFolderAccessorHandle(targetHandle)) { - await mkdirp(path.dirname(targetHandle.fullPath)) // Create folder if it doesn't exist + await fsMkDir(path.dirname(targetHandle.fullPath), { recursive: true }) // Create folder if it doesn't exist args.push(escapeFilePath(targetHandle.fullPath)) } else if (isFileShareAccessorHandle(targetHandle)) { await targetHandle.prepareFileAccess() - await mkdirp(path.dirname(targetHandle.fullPath)) // Create folder if it doesn't exist + await fsMkDir(path.dirname(targetHandle.fullPath), { recursive: true }) // Create folder if it doesn't exist args.push(escapeFilePath(targetHandle.fullPath)) } else if (isHTTPProxyAccessorHandle(targetHandle)) { pipeStdOut = true diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index b36fb0e4..8f27c356 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -361,7 +361,17 @@ export function unlink(path: string, callback: (error: any, result?: any) => voi } fs.unlink = unlink -export function mkdir(path: string, callback: (error: any, result?: any) => void): void { +export function mkdir(path: string, callback: (error: any, result?: any) => void): void +export function mkdir(path: string, opts: { recursive?: boolean }, callback: (error: any, result?: any) => void): void +export function mkdir( + path: string, + optsOrCallback: { recursive?: boolean } | ((error: any, result?: any) => void), + callback?: (error: any, result?: any) => void +): void { + if (typeof optsOrCallback === 'function') { + callback = optsOrCallback + } + path = fixPath(path) if (DEBUG_LOG) console.log('fs.mkdir', path) fsMockEmitter.emit('mkdir', path) @@ -377,9 +387,9 @@ export function mkdir(path: string, callback: (error: any, result?: any) => void false ) - return callback(undefined, null) + return callback?.(undefined, null) } catch (err) { - callback(err) + callback?.(err) } } fs.mkdir = mkdir diff --git a/tests/internal-tests/src/__mocks__/mkdirp.ts b/tests/internal-tests/src/__mocks__/mkdirp.ts deleted file mode 100644 index 9cc6c39b..00000000 --- a/tests/internal-tests/src/__mocks__/mkdirp.ts +++ /dev/null @@ -1,22 +0,0 @@ -const mod: any = jest.createMockFromModule('mkdirp') - -import fsOrg from 'fs' -import { promisify } from 'util' -import type * as fsMockType from '../__mocks__/fs' -const fs = fsOrg as any as typeof fsMockType - -const fsStat = promisify(fs.stat) - -export async function mkdirp(path: string): Promise { - try { - // check if the folder already exists before creating a new one: - await fsStat(path) - } catch (err: any) { - if (err.code !== 'ENOENT') throw err - - fs.__mockSetDirectory(path) - } -} -mod.mkdirp = mkdirp - -module.exports = mod diff --git a/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts index 9e5edbc9..f1294fba 100644 --- a/tests/internal-tests/src/__tests__/basic.spec.ts +++ b/tests/internal-tests/src/__tests__/basic.spec.ts @@ -27,7 +27,6 @@ import { getQuantelTarget, } from './lib/containers' jest.mock('fs') -jest.mock('mkdirp') jest.mock('child_process') jest.mock('windows-network-drive') jest.mock('tv-automation-quantel-gateway-client') @@ -264,17 +263,12 @@ describeForAllPlatforms( }, startRequirement: { sources: [ - getLocalSource( - SOURCE0, - 'myData0.json' - ) as Expectation.SpecificPackageContainerOnPackage.JSONDataSource, + getLocalSource(SOURCE0, 'myData0.json') as Expectation.SpecificPackageContainerOnPackage.JSONDataSource, ], }, endRequirement: { targets: [ - getCorePackageInfoTarget( - TARGET1 - ) as Expectation.SpecificPackageContainerOnPackage.JSONDataTarget, + getCorePackageInfoTarget(TARGET1) as Expectation.SpecificPackageContainerOnPackage.JSONDataTarget, ], content: {}, version: { type: Expectation.Version.Type.JSON_DATA }, diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts index 22293319..5f4ad641 100644 --- a/tests/internal-tests/src/__tests__/issues.spec.ts +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -18,7 +18,6 @@ import { waitUntil, waitTime, describeForAllPlatforms } from './lib/lib' import { getLocalSource, getLocalTarget } from './lib/containers' import { WorkerAgent } from '@sofie-package-manager/worker' jest.mock('fs') -jest.mock('mkdirp') jest.mock('child_process') jest.mock('windows-network-drive') jest.mock('tv-automation-quantel-gateway-client') @@ -311,9 +310,7 @@ describeForAllPlatforms( // Wait until the work have been aborted, and restarted: await waitUntil(() => { - expect(env.expectationStatuses[EXP_copy0].statusInfo.status).toEqual( - expect.stringMatching(/new|waiting/) - ) + expect(env.expectationStatuses[EXP_copy0].statusInfo.status).toEqual(expect.stringMatching(/new|waiting/)) }, env.WORK_TIMEOUT_TIME + env.WAIT_JOB_TIME_SAFE) // Add another worker: diff --git a/yarn.lock b/yarn.lock index 057e4f22..2fc36aec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -601,7 +601,6 @@ __metadata: "@types/koa-router": ^7.4.0 "@types/koa__cors": ^4.0.0 "@types/mime-types": ^2.1.0 - "@types/mkdirp": ^1.0.1 "@types/node": ^14.14.31 "@types/underscore": ^1.10.24 "@types/yargs": ^17.0.24 @@ -610,7 +609,6 @@ __metadata: koa-range: ^0.3.0 koa-router: ^8.0.8 mime-types: ^2.1.28 - mkdirp: ^3.0.1 pechkin: ^1.0.1 pretty-bytes: ^5.5.0 tslib: ^2.1.0 @@ -1824,7 +1822,6 @@ __metadata: "@types/koa-router": ^7.4.4 "@types/koa__cors": ^4.0.0 "@types/mime-types": ^2.1.0 - "@types/mkdirp": ^1.0.1 "@types/node": ^14.14.31 "@types/underscore": ^1.10.24 "@types/xml2js": ^0.4.7 @@ -1836,7 +1833,6 @@ __metadata: koa-ratelimit: ^5.0.1 koa-router: ^12.0.0 mime-types: ^2.1.28 - mkdirp: ^3.0.1 pretty-bytes: ^5.5.0 tslib: ^2.1.0 underscore: ^1.12.0 @@ -2007,7 +2003,6 @@ __metadata: deep-diff: ^1.0.2 form-data: ^4.0.0 jest-mock-extended: ^3.0.5 - mkdirp: ^3.0.1 node-fetch: ^2.6.1 proper-lockfile: ^4.1.2 tmp: ~0.2.1 @@ -2449,15 +2444,6 @@ __metadata: languageName: node linkType: hard -"@types/mkdirp@npm:^1.0.1": - version: 1.0.2 - resolution: "@types/mkdirp@npm:1.0.2" - dependencies: - "@types/node": "*" - checksum: 72dedfb2d250f0e44ec964ac68ff6a4a03d7d9d6e83ae1d5ba6d212791af69cf051a5fac59846826242d099de56a18c8e0581861792fae8b845b3c9ad67ecdeb - languageName: node - linkType: hard - "@types/node-fetch@npm:^2.5.8": version: 2.6.2 resolution: "@types/node-fetch@npm:2.6.2" @@ -8653,15 +8639,6 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:^3.0.1": - version: 3.0.1 - resolution: "mkdirp@npm:3.0.1" - bin: - mkdirp: dist/cjs/src/bin.js - checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d - languageName: node - linkType: hard - "modify-values@npm:^1.0.0": version: 1.0.1 resolution: "modify-values@npm:1.0.1" @@ -9614,7 +9591,6 @@ __metadata: json-schema-to-typescript: ^10.1.5 lerna: ^6.6.1 lint-staged: ^15.2.2 - mkdirp: ^3.0.1 node-fetch: ^2.6.9 pkg: ^5.8.0 rimraf: ^5.0.5 From 2b7e24c36e5fffc482c6932adcbbf1f0a24990b1 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Mon, 18 Mar 2024 17:20:36 +0100 Subject: [PATCH 03/30] fix: make the lock for criticalWorkerPool work --- .../internalManager/lib/trackedWorkerAgents.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts index a6374b28..a972667a 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts @@ -38,7 +38,7 @@ export class TrackedWorkerAgents { */ public async updateAvailableWorkersForExpectation( trackedExp: TrackedExpectation, - criticalWorkerPoolSize: number + criticalWorkerPoolRatio: number ): Promise<{ hasQueriedAnyone: boolean workerCount: number @@ -49,15 +49,17 @@ export class TrackedWorkerAgents { // workers, but instead just ask until we have got an enough number of available workers. let hasQueriedAnyone = false + const criticalWorkerPool = (1 - criticalWorkerPoolRatio) * workerAgents.length + const criticalWorkerPoolFirstIndex = Math.floor(criticalWorkerPool) await Promise.all( - workerAgents.map(async ({ workerId, workerAgent }, index, allWorkers) => { + workerAgents.map(async ({ workerId, workerAgent }, workerIndex) => { if (!workerAgent.connected) return - if ( - !trackedExp.exp.statusReport.requiredForPlayout && - // 1 - criticalWorkerPoolSize, so that the "criticalPool" consists of the "freshest" workers - index / (allWorkers.length - 1) > 1 - criticalWorkerPoolSize - ) { + if (!trackedExp.exp.statusReport.requiredForPlayout && workerIndex >= criticalWorkerPoolFirstIndex) { trackedExp.availableWorkers.delete(workerId) + trackedExp.noAvailableWorkersReason = { + user: 'Worker reserved for requiredForPlayout expectations', + tech: `Worker "${workerId}" is within the pool of critical expectation workers, currently that size is ${criticalWorkerPool} out of ${workerAgents.length} total`, + } return } From e78cf1d07e88d345ceb191cc7857c1bd7848db7f Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 21 Mar 2024 06:29:40 +0100 Subject: [PATCH 04/30] chore: fix: ensure the cache is cleared on error --- .../src/worker/accessorHandlers/lib/CachedQuantelGateway.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts index aa64c73b..7e2a0606 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts @@ -61,6 +61,11 @@ export class CachedQuantelGateway extends QuantelGateway { } else { const promise: Promise = getValueFcn() + // Clear the cache on error: + promise.catch(() => { + this._cache.delete(cacheKey) + }) + this._cache.set(cacheKey, { timestamp: Date.now(), promise: promise, From d856063bdd4d5c85f590b114ad4817dc02a1ab17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:22:29 +0000 Subject: [PATCH 05/30] chore(deps): bump ws and @types/ws Bumps [ws](https://github.com/websockets/ws) and [@types/ws](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/ws). These dependencies needed to be updated together. Updates `ws` from 8.12.0 to 8.16.0 - [Release notes](https://github.com/websockets/ws/releases) - [Commits](https://github.com/websockets/ws/compare/8.12.0...8.16.0) Updates `@types/ws` from 8.5.4 to 8.5.10 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/ws) --- updated-dependencies: - dependency-name: ws dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: "@types/ws" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yarn.lock b/yarn.lock index 13466aff..8d8a20be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2601,11 +2601,11 @@ __metadata: linkType: hard "@types/ws@npm:^8.5.4": - version: 8.5.4 - resolution: "@types/ws@npm:8.5.4" + version: 8.5.10 + resolution: "@types/ws@npm:8.5.10" dependencies: "@types/node": "npm:*" - checksum: 10/8ad37f8ec1f7a1e2b8c0d53353ac30d182277c0bce4d877a497a0b7bcfbeee1838270eb6247a6978da66cc2891106d3c77511ebc827c58967ede8e756446422f + checksum: 10/9b414dc5e0b6c6f1ea4b1635b3568c58707357f68076df9e7cd33194747b7d1716d5189c0dbdd68c8d2521b148e88184cf881bac7429eb0e5c989b001539ed31 languageName: node linkType: hard @@ -12490,8 +12490,8 @@ __metadata: linkType: hard "ws@npm:^8.12.0": - version: 8.12.0 - resolution: "ws@npm:8.12.0" + version: 8.16.0 + resolution: "ws@npm:8.16.0" peerDependencies: bufferutil: ^4.0.1 utf-8-validate: ">=5.0.2" @@ -12500,7 +12500,7 @@ __metadata: optional: true utf-8-validate: optional: true - checksum: 10/325fbcf6bbed07350b82d7a5bdb43e8a4e81512973241c656c2119a37883a74fe49e7cac09646f9bfc28c517cd63f4111c78f5898bcdd25a3ec2cc4e59375331 + checksum: 10/7c511c59e979bd37b63c3aea4a8e4d4163204f00bd5633c053b05ed67835481995f61a523b0ad2b603566f9a89b34cb4965cb9fab9649fbfebd8f740cea57f17 languageName: node linkType: hard From 73b70a19b50d09b245e0674118e6971e98a00c6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:23:05 +0000 Subject: [PATCH 06/30] chore(deps): bump follow-redirects from 1.15.5 to 1.15.6 Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.5 to 1.15.6. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.5...v1.15.6) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 13466aff..a0fa2fb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5545,12 +5545,12 @@ __metadata: linkType: hard "follow-redirects@npm:^1.15.4": - version: 1.15.5 - resolution: "follow-redirects@npm:1.15.5" + version: 1.15.6 + resolution: "follow-redirects@npm:1.15.6" peerDependenciesMeta: debug: optional: true - checksum: 10/d467f13c1c6aa734599b8b369cd7a625b20081af358f6204ff515f6f4116eb440de9c4e0c49f10798eeb0df26c95dd05d5e0d9ddc5786ab1a8a8abefe92929b4 + checksum: 10/70c7612c4cab18e546e36b991bbf8009a1a41cf85354afe04b113d1117569abf760269409cb3eb842d9f7b03d62826687086b081c566ea7b1e6613cf29030bf7 languageName: node linkType: hard From 017a75301d9347c2c633729e5df56a66f9a893f4 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Mon, 25 Mar 2024 14:56:30 +0100 Subject: [PATCH 07/30] fix: ensure we don't call QuantelGateway.connect" more than once at a time --- .../lib/CachedQuantelGateway.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts index 7e2a0606..96307a4c 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/CachedQuantelGateway.ts @@ -1,6 +1,11 @@ import { ProtectedString, protectString } from '@sofie-package-manager/api' import { ClipSearchQuery, QuantelGateway } from 'tv-automation-quantel-gateway-client' -import { ClipDataSummary, ServerInfo, ZoneInfo } from 'tv-automation-quantel-gateway-client/dist/quantelTypes' +import { + ClipDataSummary, + ConnectionDetails, + ServerInfo, + ZoneInfo, +} from 'tv-automation-quantel-gateway-client/dist/quantelTypes' const DEFAULT_CACHE_EXPIRE = 3000 @@ -13,6 +18,8 @@ export class CachedQuantelGateway extends QuantelGateway { private cacheExpire: number private purgeExpiredCacheTimeout: NodeJS.Timeout | null = null + private _connectToISAPromise: Promise | null = null + constructor( config?: | { @@ -25,6 +32,24 @@ export class CachedQuantelGateway extends QuantelGateway { this.cacheExpire = config?.timeout ?? DEFAULT_CACHE_EXPIRE } + async connectToISA(ISAUrls: string | string[]): Promise { + // Ensure that we only call super.connectToISA() once at a time: + if (this._connectToISAPromise) return this._connectToISAPromise + + this._connectToISAPromise = super + .connectToISA(ISAUrls) + .then((connectionDetails) => { + this._connectToISAPromise = null + return connectionDetails + }) + .catch((e) => { + this._connectToISAPromise = null + throw e + }) + + return this._connectToISAPromise + } + async purgeCacheSearchClip(searchQuery: ClipSearchQuery): Promise { return this.clearCache('searchClip', [searchQuery]) ?? [] } From 2fd7143d292dafe139a93fc0b8915f38f9b7d9da Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Tue, 26 Mar 2024 15:49:55 +0100 Subject: [PATCH 08/30] fix: move the critical worker functionality into appContainer/workerAgent Co-authored-by: Johan Nyman --- .../packages/generic/src/appContainer.ts | 119 ++++-- .../packages/generic/src/connector.ts | 3 +- .../nrk/expectations-lib.ts | 24 +- .../packages/generic/src/packageManager.ts | 5 +- shared/packages/api/src/appContainer.ts | 1 + shared/packages/api/src/config.ts | 16 +- shared/packages/api/src/expectationApi.ts | 4 +- .../evaluateExpectationStates/new.ts | 3 +- .../src/expectationManager.ts | 1 - .../src/internalManager/internalManager.ts | 2 - .../lib/trackedWorkerAgents.ts | 21 +- shared/packages/worker/src/workerAgent.ts | 365 ++++++++++-------- 12 files changed, 324 insertions(+), 240 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 4c196166..9eead331 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -55,10 +55,15 @@ export class AppContainer { private busyPorts = new Set() private apps: Map< + // <- TODO: Needs to understand which apps are special and which aren't AppId, { process: cp.ChildProcess appType: AppType + /** Set to true if app should be considered for scaling down */ + isAutoScaling: boolean + /** Set to true if the app is only handling playout-critical expectations */ + isOnlyForCriticalExpectations: boolean /** Set to true when the process is about to be killed */ toBeKilled: boolean restarts: number @@ -71,7 +76,7 @@ export class AppContainer { start: number } > = new Map() - private availableApps: Map = new Map() + private availableApps: Map = new Map() // <- needs to be smarter private websocketServer?: WebsocketServer private monitorAppsTimer: NodeJS.Timer | undefined @@ -191,8 +196,8 @@ export class AppContainer { }) } async init(): Promise { - await this.setupAvailableApps() - // Note: if we later change this.setupAvailableApps to run on an interval + await this.discoverAvailableApps() + // Note: if we later change this.discoverAvailableApps to run on an interval // don't throw here: if (this.availableApps.size === 0) { throw new Error(`AppContainer found no apps upon init. (Check if there are any Worker executables?)`) @@ -239,12 +244,11 @@ export class AppContainer { }, requestSpinDown: async (): Promise => { const app = this.apps.get(clientId) - if (app) { - if (this.getAppCount(app.appType) > this.config.appContainer.minRunningApps) { - this.spinDown(clientId, `Requested by app`).catch((error) => { - this.logger.error(`Error when spinning down app "${clientId}": ${stringifyError(error)}`) - }) - } + if (!app || !app.isAutoScaling) return + if (this.getScalingAppCount(app.appType) > this.config.appContainer.minRunningApps) { + this.spinDown(clientId, `Requested by app`).catch((error) => { + this.logger.error(`Error when spinning down app "${clientId}": ${stringifyError(error)}`) + }) } }, workerStorageWriteLock: async ( @@ -265,17 +269,26 @@ export class AppContainer { } } - private getAppCount(appType: AppType): number { + private getScalingAppCount(appType: AppType): number { + let count = 0 + for (const app of this.apps.values()) { + if (app.appType === appType && app.isAutoScaling) count++ + } + return count + } + private getCriticalExpectationAppCount(appType: AppType): number { let count = 0 for (const app of this.apps.values()) { - if (app.appType === appType) count++ + if (app.appType === appType && app.isOnlyForCriticalExpectations) count++ } return count } - private async setupAvailableApps() { - const getWorkerArgs = (appId: AppId): string[] => { + + private async discoverAvailableApps() { + const getWorkerArgs = (appId: AppId, pickUpCriticalExpectationsOnly: boolean): string[] => { return [ `--workerId=${appId}`, + pickUpCriticalExpectationsOnly ? `--pickUpCriticalExpectationsOnly=1` : '', `--workforceURL=${this.config.appContainer.workforceURL}`, `--appContainerURL=${'ws://127.0.0.1:' + this.websocketServer?.port}`, @@ -291,7 +304,7 @@ export class AppContainer { ? `--costMultiplier=${this.config.appContainer.worker.costMultiplier}` : '', this.config.appContainer.worker.considerCPULoad - ? `--costMultiplier=${this.config.appContainer.worker.considerCPULoad}` + ? `--considerCPULoad=${this.config.appContainer.worker.considerCPULoad}` : '', this.config.appContainer.worker.resourceId ? `--resourceId=${this.config.appContainer.worker.resourceId}` @@ -309,9 +322,10 @@ export class AppContainer { const appType = protectString('worker') this.availableApps.set(appType, { file: process.execPath, - args: (appId: AppId) => { - return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId)] + getExecArgs: (appId: AppId) => { + return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId, false)] }, + canRunInCriticalExpectationsOnlyMode: true, cost: 0, }) } else { @@ -319,20 +333,20 @@ export class AppContainer { // Look for the worker executable(s) in the same folder: const dirPath = path.dirname(process.execPath) - // Note: nexe causes issues with its virtual file system: https://github.com/nexe/nexe/issues/613#issuecomment-579107593 ;(await fs.promises.readdir(dirPath)).forEach((fileName) => { - if (fileName.match(/worker/i)) { - // We use the filename to identify the appType: - const appType: AppType = protectString(fileName) - this.availableApps.set(appType, { - file: path.join(dirPath, fileName), - args: (appId: AppId) => { - return [...getWorkerArgs(appId)] - }, - cost: 0, - }) - } + if (!fileName.match(/worker/i)) return + + // We use the filename to identify the appType: + const appType: AppType = protectString(fileName) + this.availableApps.set(appType, { + file: path.join(dirPath, fileName), + getExecArgs: (appId: AppId) => { + return [...getWorkerArgs(appId, false)] + }, + canRunInCriticalExpectationsOnlyMode: true, + cost: 0, + }) }) } @@ -522,7 +536,11 @@ export class AppContainer { async spinUp(appType: AppType, longSpinDownTime = false): Promise { return this._spinUp(appType, longSpinDownTime) } - private async _spinUp(appType: AppType, longSpinDownTime = false): Promise { + private async _spinUp( + appType: AppType, + longSpinDownTime = false, + isOnlyForCriticalExpectations = false + ): Promise { const availableApp = this.availableApps.get(appType) if (!availableApp) throw new Error(`Unknown appType "${appType}"`) @@ -530,7 +548,13 @@ export class AppContainer { this.logger.debug(`Spinning up app "${appId}" of type "${appType}"`) - const child = this.setupChildProcess(appType, appId, availableApp) + const child = this.setupChildProcess(appType, appId, availableApp, isOnlyForCriticalExpectations) + + let isAutoScaling = true + if (isOnlyForCriticalExpectations) { + isAutoScaling = false + } + this.apps.set(appId, { process: child, appType: appType, @@ -538,6 +562,8 @@ export class AppContainer { restarts: 0, lastRestart: 0, monitorPing: false, + isAutoScaling: isAutoScaling, + isOnlyForCriticalExpectations: isOnlyForCriticalExpectations, lastPing: Date.now(), spinDownTime: this.config.appContainer.spinDownTime * (longSpinDownTime ? 10 : 1), workerAgentApi: null, @@ -582,7 +608,12 @@ export class AppContainer { } }) } - private setupChildProcess(appType: AppType, appId: AppId, availableApp: AvailableAppInfo): cp.ChildProcess { + private setupChildProcess( + appType: AppType, + appId: AppId, + availableApp: AvailableAppInfo, + useCriticalOnlyMode: boolean + ): cp.ChildProcess { const cwd = process.execPath.match(/node.exe$/) ? undefined // Process runs as a node process, we're probably in development mode. : path.dirname(process.execPath) // Process runs as a node process, we're probably in development mode. @@ -602,7 +633,7 @@ export class AppContainer { this.usedInspectPorts.add(inspectPort) } - const child = cp.spawn(availableApp.file, availableApp.args(appId), { + const child = cp.spawn(availableApp.file, availableApp.getExecArgs(appId, useCriticalOnlyMode), { cwd: cwd, env: { ...process.env, @@ -613,10 +644,10 @@ export class AppContainer { this.logger.info(`Starting process "${appId}" (${appType}), pid=${child.pid}: "${availableApp.file}"`) child.stdout.on('data', (message) => { - this.logFromApp(appId, appType, message, this.logger.debug) + this.onOutputFromApp(appId, appType, message, this.logger.debug) }) child.stderr.on('data', (message) => { - this.logFromApp(appId, appType, message, this.logger.error) + this.onOutputFromApp(appId, appType, message, this.logger.error) // this.logger.debug(`${appId} stderr: ${message}`) }) child.on('error', (err) => { @@ -648,7 +679,7 @@ export class AppContainer { app.process.removeAllListeners() - const newChild = this.setupChildProcess(appType, appId, availableApp) + const newChild = this.setupChildProcess(appType, appId, availableApp, useCriticalOnlyMode) app.process = newChild }, timeUntilRestart) @@ -683,18 +714,29 @@ export class AppContainer { }) } } + this.spinUpMinimumApps().catch((error) => { this.logger.error(`Error in spinUpMinimumApps: ${stringifyError(error)}`) }) } + private async spinUpMinimumApps(): Promise { + if (this.config.appContainer.minCriticalWorkerApps !== null) { + for (const [appType, appInfo] of this.availableApps.entries()) { + if (!appInfo.canRunInCriticalExpectationsOnlyMode) continue + while (this.getCriticalExpectationAppCount(appType) < this.config.appContainer.minCriticalWorkerApps) { + await this._spinUp(appType, false, true) + } + } + } + for (const appType of this.availableApps.keys()) { - while (this.getAppCount(appType) < this.config.appContainer.minRunningApps) { + while (this.getScalingAppCount(appType) < this.config.appContainer.minRunningApps) { await this._spinUp(appType) } } } - private logFromApp(appId: AppId, appType: AppType, data: any, defaultLog: LeveledLogMethod): void { + private onOutputFromApp(appId: AppId, appType: AppType, data: any, defaultLog: LeveledLogMethod): void { const messages = `${data}`.split('\n') for (const message of messages) { @@ -774,8 +816,9 @@ export class AppContainer { } interface AvailableAppInfo { file: string - args: (appId: AppId) => string[] + getExecArgs: (appId: AppId, useCriticalOnlyMode: boolean) => string[] /** Some kind of value, how much it costs to run it, per minute */ + canRunInCriticalExpectationsOnlyMode: boolean cost: number } diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts index 20139b1b..8c382d4a 100644 --- a/apps/package-manager/packages/generic/src/connector.ts +++ b/apps/package-manager/packages/generic/src/connector.ts @@ -82,8 +82,7 @@ export class Connector { config.packageManager.accessUrl || undefined, workForceConnectionOptions, config.packageManager.concurrency, - config.packageManager.chaosMonkey, - config.packageManager.criticalWorkerPoolSize + config.packageManager.chaosMonkey ) } diff --git a/apps/package-manager/packages/generic/src/generateExpectations/nrk/expectations-lib.ts b/apps/package-manager/packages/generic/src/generateExpectations/nrk/expectations-lib.ts index 5344cbfe..61738106 100644 --- a/apps/package-manager/packages/generic/src/generateExpectations/nrk/expectations-lib.ts +++ b/apps/package-manager/packages/generic/src/generateExpectations/nrk/expectations-lib.ts @@ -60,7 +60,6 @@ export function generateMediaFileCopy( description: `Copy media file "${expWrapMediaFile.expectedPackage.content.filePath}" to the device "${ expWrapMediaFile.playoutDeviceId }", from ${expWrapMediaFile.sources.map((source) => `"${source.label}"`).join(', ')}`, - requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, }, @@ -74,6 +73,7 @@ export function generateMediaFileCopy( removeDelay: settings.delayRemoval, allowWaitForCPU: false, useTemporaryFilePath: settings.useTemporaryFilePath, + requiredForPlayout: true, }, } @@ -108,7 +108,6 @@ export function generateMediaFileVerify( statusReport: { label: `Check media "${expWrapMediaFile.expectedPackage.content.filePath}"`, description: `Check that file "${expWrapMediaFile.expectedPackage.content.filePath}" exists for the device "${expWrapMediaFile.playoutDeviceId}"`, - requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, }, @@ -120,6 +119,7 @@ export function generateMediaFileVerify( endRequirement, workOptions: { allowWaitForCPU: false, + requiredForPlayout: true, }, } @@ -171,7 +171,6 @@ export function generateQuantelCopy( description: `Copy Quantel clip ${title || guid} to server for "${ expWrapQuantelClip.playoutDeviceId }", from ${expWrapQuantelClip.sources.map((source) => `"${source.label}"`).join(', ')}`, - requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, }, @@ -183,6 +182,7 @@ export function generateQuantelCopy( endRequirement, workOptions: { allowWaitForCPU: false, + requiredForPlayout: true, // removeDelay: 0 // Not used by Quantel }, } @@ -210,7 +210,6 @@ export function generatePackageScan( statusReport: { label: `Scanning`, description: `Scanning the media, to provide data to the Sofie GUI`, - requiredForPlayout: !!(expectation as any).__isSmartbull, // For smartbull, the scan result _is_ required for playout displayRank: 10, sendReport: expectation.statusReport.sendReport, }, @@ -237,6 +236,7 @@ export function generatePackageScan( }, workOptions: { ...expectation.workOptions, + requiredForPlayout: !!(expectation as any).__isSmartbull, // For smartbull, the scan result _is_ required for playout allowWaitForCPU: false, removeDelay: settings.delayRemovalPackageInfo, }, @@ -258,7 +258,6 @@ export function generatePackageDeepScan( statusReport: { label: `Deep Scanning`, description: `Detecting scenes, black frames, freeze frames etc.`, - requiredForPlayout: false, displayRank: 13, sendReport: expectation.statusReport.sendReport, }, @@ -290,6 +289,7 @@ export function generatePackageDeepScan( }, workOptions: { ...expectation.workOptions, + requiredForPlayout: false, allowWaitForCPU: true, usesCPUCount: 1, removeDelay: settings.delayRemovalPackageInfo, @@ -314,7 +314,6 @@ export function generatePackageLoudness( statusReport: { label: `Loudness Scan`, description: `Measure clip loudness, using channels ${packageSettings.channelSpec.join(', ')}`, - requiredForPlayout: false, displayRank: 14, sendReport: expectation.statusReport.sendReport, }, @@ -346,6 +345,7 @@ export function generatePackageLoudness( workOptions: { ...expectation.workOptions, allowWaitForCPU: true, + requiredForPlayout: false, usesCPUCount: 1, removeDelay: settings.delayRemovalPackageInfo, }, @@ -370,7 +370,6 @@ export function generateMediaFileThumbnail( statusReport: { label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, - requiredForPlayout: false, displayRank: 11, sendReport: expectation.statusReport.sendReport, }, @@ -400,6 +399,7 @@ export function generateMediaFileThumbnail( workOptions: { ...expectation.workOptions, allowWaitForCPU: true, + requiredForPlayout: false, usesCPUCount: 1, removeDelay: 0, // The removal of the thumnail shouldn't be delayed removePackageOnUnFulfill: true, @@ -424,7 +424,6 @@ export function generateMediaFilePreview( statusReport: { label: `Generating preview`, description: `Preview is used in Sofie GUI`, - requiredForPlayout: false, displayRank: 12, sendReport: expectation.statusReport.sendReport, }, @@ -453,6 +452,7 @@ export function generateMediaFilePreview( workOptions: { ...expectation.workOptions, allowWaitForCPU: true, + requiredForPlayout: false, usesCPUCount: 1, removeDelay: 0, // The removal of the preview shouldn't be delayed removePackageOnUnFulfill: true, @@ -478,7 +478,6 @@ export function generateQuantelClipThumbnail( statusReport: { label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, - requiredForPlayout: false, displayRank: 11, sendReport: expectation.statusReport.sendReport, }, @@ -507,6 +506,7 @@ export function generateQuantelClipThumbnail( workOptions: { ...expectation.workOptions, allowWaitForCPU: true, + requiredForPlayout: false, usesCPUCount: 1, removeDelay: 0, // The removal of the thumbnail shouldn't be delayed removePackageOnUnFulfill: true, @@ -531,7 +531,6 @@ export function generateQuantelClipPreview( statusReport: { label: `Generating preview`, description: `Preview is used in Sofie GUI`, - requiredForPlayout: false, displayRank: 12, sendReport: expectation.statusReport.sendReport, }, @@ -562,6 +561,7 @@ export function generateQuantelClipPreview( workOptions: { ...expectation.workOptions, allowWaitForCPU: true, + requiredForPlayout: false, usesCPUCount: 1, removeDelay: 0, // The removal of the preview shouldn't be delayed removePackageOnUnFulfill: true, @@ -602,7 +602,6 @@ export function generateJsonDataCopy( description: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}" from "${JSON.stringify( expWrapMediaFile.sources )}"`, - requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, }, @@ -613,6 +612,7 @@ export function generateJsonDataCopy( endRequirement, workOptions: { + requiredForPlayout: true, removeDelay: settings.delayRemoval, useTemporaryFilePath: settings.useTemporaryFilePath, allowWaitForCPU: false, @@ -657,7 +657,6 @@ export function generatePackageCopyFileProxy( statusReport: { label: `Copy proxy`, description: `Making a copy as a proxy, used in later steps to scan, generate thumbnail etc..`, - requiredForPlayout: !!(expectation as any).__isSmartbull, // For smartbull, this _is_ required for playout displayRank: 9, sendReport: expectation.statusReport.sendReport, }, @@ -683,6 +682,7 @@ export function generatePackageCopyFileProxy( }, workOptions: { ...expectation.workOptions, + requiredForPlayout: !!(expectation as any).__isSmartbull, // For smartbull, this _is_ required for playout allowWaitForCPU: false, removeDelay: settings.delayRemovalPackageInfo, }, diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index 43df13be..5f334f32 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -115,8 +115,7 @@ export class PackageManagerHandler { private serverAccessUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, concurrency: number | undefined, - chaosMonkey: boolean, - criticalWorkerPoolSize: number | undefined + chaosMonkey: boolean ) { this.logger = logger.category('PackageManager') this.callbacksHandler = new ExpectationManagerCallbacksHandler(this.logger, this) @@ -130,7 +129,6 @@ export class PackageManagerHandler { this.callbacksHandler, { chaosMonkey: chaosMonkey, - criticalWorkerPoolSize: criticalWorkerPoolSize, constants: { PARALLEL_CONCURRENCY: concurrency, }, @@ -553,6 +551,7 @@ class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks // Updated properties: ...expectaction.statusReport, + requiredForPlayout: expectaction.workOptions.requiredForPlayout ?? false, ...statusInfo, fromPackages: expectaction.fromPackages.map((fromPackage) => { diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts index 86287fda..e03686b5 100644 --- a/shared/packages/api/src/appContainer.ts +++ b/shared/packages/api/src/appContainer.ts @@ -15,6 +15,7 @@ export interface AppContainerConfig { maxRunningApps: number maxAppKeepalive: number spinDownTime: number + minCriticalWorkerApps: number | null worker: { resourceId: string diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index ffd9348b..d0c0676d 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -119,11 +119,6 @@ const packageManagerArguments = defineArguments({ default: process.env.CHAOS_MONKEY === '1', describe: 'If true, enables the "chaos monkey"-feature, which will randomly kill processes every few seconds', }, - criticalWorkerPoolSize: { - type: 'number', - default: 0, - describe: 'Percentage of Workers reserved for fulfilling playout-critical expectations', - }, concurrency: { type: 'number', default: parseInt(process.env.CONCURRENCY || '', 10) || undefined, @@ -235,6 +230,11 @@ const appContainerArguments = defineArguments({ describe: 'If set, the worker will consider the CPU load of the system it runs on before it accepts jobs. Set to a value between 0 and 1, the worker will accept jobs if the CPU load is below the configured value.', }, + minCriticalWorkerApps: { + type: 'number', + default: 0, + describe: 'Number of Workers reserved for fulfilling playout-critical expectations that will be kept runnini', + }, }) /** CLI-argument-definitions for the "Single" process */ const singleAppArguments = defineArguments({ @@ -378,7 +378,6 @@ export interface PackageManagerConfig { watchFiles: boolean noCore: boolean chaosMonkey: boolean - criticalWorkerPoolSize?: number concurrency?: number } } @@ -406,7 +405,6 @@ export async function getPackageManagerConfig(): Promise { watchFiles: argv.watchFiles, noCore: argv.noCore, chaosMonkey: argv.chaosMonkey, - criticalWorkerPoolSize: argv.criticalWorkerPoolSize, concurrency: argv.concurrency, }, } @@ -422,6 +420,7 @@ export interface WorkerConfig { networkIds: string[] costMultiplier: number considerCPULoad: number | null + pickUpCriticalExpectationsOnly: boolean } & WorkerAgentConfig } export async function getWorkerConfig(): Promise { @@ -447,6 +446,8 @@ export async function getWorkerConfig(): Promise { considerCPULoad: (typeof argv.considerCPULoad === 'string' ? parseFloat(argv.considerCPULoad) : argv.considerCPULoad) || null, + pickUpCriticalExpectationsOnly: + (typeof argv.pickUpCriticalExpectationsOnly === 'string' ? true : false) || false, }, } } @@ -473,6 +474,7 @@ export async function getAppContainerConfig(): Promise & { + statusReport: Omit & { /** Set to true to enable reporting back statuses to Core */ sendReport: boolean } @@ -400,6 +400,8 @@ export namespace Expectation { usesCPUCount?: number /** If set, removes the target package if the expectation becomes unfulfilled. */ removePackageOnUnFulfill?: boolean + /** If set, the expectation is required for playout and therefore has the highest priority */ + requiredForPlayout?: boolean } export interface RemoveDelay { /** When removing, wait a duration of time before actually removing it (milliseconds). If not set, package is removed right away. */ diff --git a/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts b/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts index 41642b5b..49bf9f6b 100644 --- a/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts +++ b/shared/packages/expectationManager/src/evaluationRunner/evaluateExpectationStates/new.ts @@ -16,8 +16,7 @@ export async function evaluateExpectationStateNew({ manager, tracker, trackedExp trackedExp.status = {} const { hasQueriedAnyone, workerCount } = await manager.workerAgents.updateAvailableWorkersForExpectation( - trackedExp, - manager.criticalWorkerPoolSize + trackedExp ) const availableWorkersCount = trackedExp.availableWorkers.size diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index 59157ad8..f4ff9f3a 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -136,7 +136,6 @@ export class ExpectationManager { export interface ExpectationManagerOptions { constants?: Partial chaosMonkey?: boolean - criticalWorkerPoolSize?: number } export type ExpectationManagerServerOptions = diff --git a/shared/packages/expectationManager/src/internalManager/internalManager.ts b/shared/packages/expectationManager/src/internalManager/internalManager.ts index cc467b27..4270f49b 100644 --- a/shared/packages/expectationManager/src/internalManager/internalManager.ts +++ b/shared/packages/expectationManager/src/internalManager/internalManager.ts @@ -43,7 +43,6 @@ export class InternalManager { public statuses: ManagerStatusReporter private enableChaosMonkey = false - public criticalWorkerPoolSize = 0 private managerWatchdog: ManagerStatusWatchdog public statusReport: StatusReportCache @@ -84,7 +83,6 @@ export class InternalManager { this.statusReport = new StatusReportCache(this) this.enableChaosMonkey = options?.chaosMonkey ?? false - this.criticalWorkerPoolSize = options?.criticalWorkerPoolSize ?? 0 } /** Initialize the ExpectationManager. This method is should be called shortly after the class has been instantiated. */ diff --git a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts index a972667a..88ee2513 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts @@ -36,10 +36,7 @@ export class TrackedWorkerAgents { * Asks the Workers if they support a certain Expectation. * Updates trackedExp.availableWorkers to reflect the result. */ - public async updateAvailableWorkersForExpectation( - trackedExp: TrackedExpectation, - criticalWorkerPoolRatio: number - ): Promise<{ + public async updateAvailableWorkersForExpectation(trackedExp: TrackedExpectation): Promise<{ hasQueriedAnyone: boolean workerCount: number }> { @@ -49,19 +46,9 @@ export class TrackedWorkerAgents { // workers, but instead just ask until we have got an enough number of available workers. let hasQueriedAnyone = false - const criticalWorkerPool = (1 - criticalWorkerPoolRatio) * workerAgents.length - const criticalWorkerPoolFirstIndex = Math.floor(criticalWorkerPool) await Promise.all( - workerAgents.map(async ({ workerId, workerAgent }, workerIndex) => { + workerAgents.map(async ({ workerId, workerAgent }) => { if (!workerAgent.connected) return - if (!trackedExp.exp.statusReport.requiredForPlayout && workerIndex >= criticalWorkerPoolFirstIndex) { - trackedExp.availableWorkers.delete(workerId) - trackedExp.noAvailableWorkersReason = { - user: 'Worker reserved for requiredForPlayout expectations', - tech: `Worker "${workerId}" is within the pool of critical expectation workers, currently that size is ${criticalWorkerPool} out of ${workerAgents.length} total`, - } - return - } // Only ask each worker once, or after a certain time has passed: const queriedWorker = trackedExp.queriedWorkers.get(workerId) @@ -117,10 +104,6 @@ export class TrackedWorkerAgents { const workerIds = Array.from(trackedExp.availableWorkers.keys()) - if (trackedExp.exp.statusReport.requiredForPlayout) { - // TODO: decimate the workerIds - } - let noCostReason: Reason = { user: `${workerIds.length} workers are currently busy`, tech: `${workerIds.length} busy, ${trackedExp.queriedWorkers.size} queried`, diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index 70384922..08999a02 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -82,6 +82,7 @@ export class WorkerAgent { private initWorkForceAPIPromise?: { resolve: () => void; reject: (reason?: any) => void } private initAppContainerAPIPromise?: { resolve: () => void; reject: (reason?: any) => void } private cpuTracker = new CPUTracker() + private isOnlyForCriticalExpectations = false private logger: LoggerInstance private workerStorageDeferRead = deferGets(async (dataId: DataId) => { @@ -147,6 +148,7 @@ export class WorkerAgent { : { type: 'internal', } + this.isOnlyForCriticalExpectations = this.config.worker.pickUpCriticalExpectationsOnly // Todo: Different types of workers: this._worker = new GenericWorker( this.logger, @@ -221,9 +223,9 @@ export class WorkerAgent { setLogLevel: async (logLevel: LogLevel) => this.setLogLevel(logLevel), _debugKill: async () => this._debugKill(), - doYouSupportExpectation: async (exp: Expectation.Any) => this.doYouSupportExpectation(exp), + doYouSupportExpectation: async (exp: Expectation.Any) => this.doesWorkerSupportExpectation(exp), doYouSupportPackageContainer: async (packageContainer: PackageContainerExpectation) => - this.doYouSupportPackageContainer(packageContainer), + this.doesWorkerSupportPackageContainer(packageContainer), setSpinDownTime: async (spinDownTime: number) => this.setSpinDownTime(spinDownTime), }) // Wait for this.appContainerAPI to be ready before continuing: @@ -300,17 +302,27 @@ export class WorkerAgent { // isFree(): boolean { // return this._busyMethodCount === 0 // } - async doYouSupportExpectation(exp: Expectation.Any): Promise { + private async doesWorkerSupportExpectation(exp: Expectation.Any): Promise { this.IDidSomeWork() + if (this.isOnlyForCriticalExpectations && !exp.workOptions.requiredForPlayout) { + return { + support: false, + reason: { + tech: 'Worker is reserved for `workOptions.requiredForPlayout` expectations', + user: 'Worker is reserved for playout-critical operations', + }, + } + } + return this._worker.doYouSupportExpectation(exp) } - async doYouSupportPackageContainer( + private async doesWorkerSupportPackageContainer( packageContainer: PackageContainerExpectation ): Promise { this.IDidSomeWork() return this._worker.doYouSupportPackageContainer(packageContainer) } - async expectationManagerAvailable(managerId: ExpectationManagerId, url: string): Promise { + private async expectationManagerAvailable(managerId: ExpectationManagerId, url: string): Promise { const existing = this.expectationManagers.get(managerId) if (existing) { existing.api.terminate() @@ -318,7 +330,7 @@ export class WorkerAgent { await this.connectToExpectationManager(managerId, url) } - async expectationManagerGone(managerId: ExpectationManagerId): Promise { + private async expectationManagerGone(managerId: ExpectationManagerId): Promise { this.expectationManagers.delete(managerId) } public async setLogLevel(logLevel: LogLevel): Promise { @@ -390,6 +402,197 @@ export class WorkerAgent { } } + private async createNewJobForExpectation( + managerId: ExpectationManagerId, + exp: Expectation.Any, + cost: ExpectationManagerWorkerAgent.ExpectationCost, + /** Timeout, cancels the job if no updates are received in this time [ms] */ + timeout: number + ): Promise { + const expectationManager = this.expectationManagers.get(managerId) + if (!expectationManager) { + this.logger.error( + `Worker "${this.id}" could not start job for expectation (${exp.id}), because it could not find expecation manager "${managerId}"` + ) + + throw new Error(`ExpectationManager "${managerId}" not found`) + } + + // Tmp: we're only allowing one work per worker + if (this.currentJobs.length > 0) { + this.logger.warn( + `createNewJobForExpectation called, even though there are ${ + this.currentJobs.length + } current jobs. Startcost now: ${this.getStartCost(exp).cost}, spcified cost=${ + cost.cost + }, specified startCost=${cost.startCost}` + ) + } + + const currentJob: CurrentJob = { + cost: cost, + cancelled: false, + lastUpdated: Date.now(), + progress: 0, + wipId: this.getNextWipId(), + workInProgress: null, + timeoutInterval: setInterval(() => { + if (currentJob.cancelled && currentJob.timeoutInterval) { + clearInterval(currentJob.timeoutInterval) + currentJob.timeoutInterval = null + return + } + const timeSinceLastUpdate = Date.now() - currentJob.lastUpdated + + if (timeSinceLastUpdate > timeout) { + // The job seems to have timed out. + // Expectation Manager will clean up on it's side, we have to do the same here. + + this.logger.warn( + `WorkerAgent: Cancelling job "${currentJob.workInProgress?.properties.workLabel}" (${currentJob.wipId}) due to timeout (${timeSinceLastUpdate} > ${timeout})` + ) + if (currentJob.timeoutInterval) { + clearInterval(currentJob.timeoutInterval) + currentJob.timeoutInterval = null + } + + // Ensure that the job is removed, so that it won't block others: + this.removeJob(currentJob) + + Promise.race([ + this.cancelJob(currentJob), + new Promise((_, reject) => { + setTimeout( + () => + reject( + `Timeout when cancelling job "${currentJob.workInProgress?.properties.workLabel}" (${currentJob.wipId})` + ), + 1000 + ) + }), + ]).catch((error) => { + // Not much we can do about that error.. + this.logger.error( + `WorkerAgent: timeout watch: Error in cancelJob (${currentJob.wipId}) ${stringifyError( + error + )}` + ) + }) + } + }, 1000), + } + this.currentJobs.push(currentJob) + this.logger.debug( + `Worker "${this.id}" starting job ${currentJob.wipId}, (${exp.id}). (${this.currentJobs.length})` + ) + + try { + const workInProgress = await this.makeWorkerWorkOnJobForExpecation(managerId, currentJob, exp, timeout) + + return { + wipId: currentJob.wipId, + properties: workInProgress.properties, + } + } catch (err) { + // makeWorkerWorkOnExpecation() / _worker.workOnExpectation() failed. + + this.removeJob(currentJob) + this.logger.warn( + `Worker "${this.id}" stopped job ${currentJob.wipId}, (${exp.id}), due to initial error. (${this.currentJobs.length})` + ) + + throw err + } + } + + private async makeWorkerWorkOnJobForExpecation( + managerId: ExpectationManagerId, + job: CurrentJob, + exp: Expectation.Any, + /** Timeout, cancels the job if no updates are received in this time [ms] */ + timeout: number + ): Promise { + const workInProgress = await this._worker.workOnExpectation(exp, timeout) + + job.workInProgress = workInProgress + + workInProgress.on('progress', (actualVersionHash, progress: number) => { + this.IDidSomeWork() + if (job.cancelled) return // Don't send updates on cancelled work + job.lastUpdated = Date.now() + job.progress = progress + + const expectationManager = this.expectationManagers.get(managerId) + if (!expectationManager) { + this.logger.warn( + `Could not report work progress to Expectation Manager "${managerId}", because the manager could not be found.` + ) + return + } + + expectationManager?.api.wipEventProgress(job.wipId, actualVersionHash, progress).catch((err) => { + if (!this.terminated) { + this.logger.error(`Error in wipEventProgress: ${stringifyError(err)}`) + } + }) + }) + workInProgress.on('error', (error: string) => { + this.IDidSomeWork() + if (job.cancelled) return // Don't send updates on cancelled work + job.lastUpdated = Date.now() + this.removeJob(job) + this.logger.warn( + `Worker "${this.id}" stopped job ${job.wipId}, (${exp.id}), due to error: (${ + this.currentJobs.length + }): ${stringifyError(error)}` + ) + + const expectationManager = this.expectationManagers.get(managerId) + if (!expectationManager) { + this.logger.warn( + `Could not report work error to Expectation Manager "${managerId}", because the manager could not be found.` + ) + return + } + + expectationManager?.api + .wipEventError(job.wipId, { + user: 'Work aborted due to an error', + tech: error, + }) + .catch((err) => { + if (!this.terminated) { + this.logger.error(`Error in wipEventError: ${stringifyError(err)}`) + } + }) + }) + workInProgress.on('done', (actualVersionHash, reason, result) => { + this.IDidSomeWork() + if (job.cancelled) return // Don't send updates on cancelled work + job.lastUpdated = Date.now() + this.removeJob(job) + this.logger.debug( + `Worker "${this.id}" stopped job ${job.wipId}, (${exp.id}), done. (${this.currentJobs.length})` + ) + + const expectationManager = this.expectationManagers.get(managerId) + if (!expectationManager) { + this.logger.warn( + `Could not report work done to Expectation Manager "${managerId}", because the manager could not be found.` + ) + return + } + + expectationManager?.api.wipEventDone(job.wipId, actualVersionHash, reason, result).catch((err) => { + if (!this.terminated) { + this.logger.error(`Error in wipEventDone: ${stringifyError(err)}`) + } + }) + }) + + return workInProgress + } + private async connectToExpectationManager(managerId: ExpectationManagerId, url: string): Promise { this.logger.info(`Worker: Connecting to Expectation Manager "${managerId}" at url "${url}"`) const expectationManager = { @@ -409,7 +612,7 @@ export class WorkerAgent { const methods = literal>({ doYouSupportExpectation: async (exp: Expectation.Any): Promise => { - return this._worker.doYouSupportExpectation(exp) + return this.doesWorkerSupportExpectation(exp) }, getCostForExpectation: async ( exp: Expectation.Any @@ -453,150 +656,7 @@ export class WorkerAgent { timeout: number ): Promise => { this.IDidSomeWork() - - // Tmp: we're only allowing one work per worker - if (this.currentJobs.length > 0) { - this.logger.warn( - `workOnExpectation called, even though there are ${ - this.currentJobs.length - } current jobs. Startcost now: ${this.getStartCost(exp).cost}, spcified cost=${ - cost.cost - }, specified startCost=${cost.startCost}` - ) - } - - const currentJob: CurrentJob = { - cost: cost, - cancelled: false, - lastUpdated: Date.now(), - progress: 0, - wipId: this.getNextWipId(), - workInProgress: null, - timeoutInterval: setInterval(() => { - if (currentJob.cancelled && currentJob.timeoutInterval) { - clearInterval(currentJob.timeoutInterval) - currentJob.timeoutInterval = null - return - } - const timeSinceLastUpdate = Date.now() - currentJob.lastUpdated - - if (timeSinceLastUpdate > timeout) { - // The job seems to have timed out. - // Expectation Manager will clean up on it's side, we have to do the same here. - - this.logger.warn( - `WorkerAgent: Cancelling job "${currentJob.workInProgress?.properties.workLabel}" (${currentJob.wipId}) due to timeout (${timeSinceLastUpdate} > ${timeout})` - ) - if (currentJob.timeoutInterval) { - clearInterval(currentJob.timeoutInterval) - currentJob.timeoutInterval = null - } - - // Ensure that the job is removed, so that it won't block others: - this.removeJob(currentJob) - - Promise.race([ - this.cancelJob(currentJob), - new Promise((_, reject) => { - setTimeout( - () => - reject( - `Timeout when cancelling job "${currentJob.workInProgress?.properties.workLabel}" (${currentJob.wipId})` - ), - 1000 - ) - }), - ]).catch((error) => { - // Not much we can do about that error.. - this.logger.error( - `WorkerAgent: timeout watch: Error in cancelJob (${ - currentJob.wipId - }) ${stringifyError(error)}` - ) - }) - } - }, 1000), - } - this.currentJobs.push(currentJob) - this.logger.debug( - `Worker "${this.id}" starting job ${currentJob.wipId}, (${exp.id}). (${this.currentJobs.length})` - ) - - try { - const workInProgress = await this._worker.workOnExpectation(exp, timeout) - - currentJob.workInProgress = workInProgress - - workInProgress.on('progress', (actualVersionHash, progress: number) => { - this.IDidSomeWork() - if (currentJob.cancelled) return // Don't send updates on cancelled work - currentJob.lastUpdated = Date.now() - currentJob.progress = progress - expectationManager.api - .wipEventProgress(currentJob.wipId, actualVersionHash, progress) - .catch((err) => { - if (!this.terminated) { - this.logger.error(`Error in wipEventProgress: ${stringifyError(err)}`) - } - }) - }) - workInProgress.on('error', (error: string) => { - this.IDidSomeWork() - if (currentJob.cancelled) return // Don't send updates on cancelled work - currentJob.lastUpdated = Date.now() - this.currentJobs = this.currentJobs.filter((job) => job !== currentJob) - this.logger.warn( - `Worker "${this.id}" stopped job ${currentJob.wipId}, (${exp.id}), due to error: (${ - this.currentJobs.length - }): ${stringifyError(error)}` - ) - - expectationManager.api - .wipEventError(currentJob.wipId, { - user: 'Work aborted due to an error', - tech: error, - }) - .catch((err) => { - if (!this.terminated) { - this.logger.error(`Error in wipEventError: ${stringifyError(err)}`) - } - }) - - this.removeJob(currentJob) - }) - workInProgress.on('done', (actualVersionHash, reason, result) => { - this.IDidSomeWork() - if (currentJob.cancelled) return // Don't send updates on cancelled work - currentJob.lastUpdated = Date.now() - this.currentJobs = this.currentJobs.filter((job) => job !== currentJob) - this.logger.debug( - `Worker "${this.id}" stopped job ${currentJob.wipId}, (${exp.id}), done. (${this.currentJobs.length})` - ) - - expectationManager.api - .wipEventDone(currentJob.wipId, actualVersionHash, reason, result) - .catch((err) => { - if (!this.terminated) { - this.logger.error(`Error in wipEventDone: ${stringifyError(err)}`) - } - }) - this.removeJob(currentJob) - }) - - return { - wipId: currentJob.wipId, - properties: workInProgress.properties, - } - } catch (err) { - // worker.workOnExpectation() failed. - - this.removeJob(currentJob) - this.logger.warn( - `Worker "${this.id}" stopped job ${currentJob.wipId}, (${exp.id}), due to initial error. (${this.currentJobs.length})` - ) - - throw err - } + return this.createNewJobForExpectation(managerId, exp, cost, timeout) }, removeExpectation: async (exp: Expectation.Any): Promise => { this.IDidSomeWork() @@ -609,8 +669,7 @@ export class WorkerAgent { doYouSupportPackageContainer: async ( packageContainer: PackageContainerExpectation ): Promise => { - this.IDidSomeWork() - return this._worker.doYouSupportPackageContainer(packageContainer) + return this.doesWorkerSupportPackageContainer(packageContainer) }, runPackageContainerCronJob: async ( packageContainer: PackageContainerExpectation From e13f8c5083afcb78887eb0871149e36efc1c685a Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Tue, 26 Mar 2024 16:21:56 +0100 Subject: [PATCH 09/30] fix: tests --- .../src/__tests__/basic.spec.ts | 15 ++++++++------ .../src/__tests__/issues.spec.ts | 5 +++-- .../src/__tests__/quantel.spec.ts | 20 +++++++++++-------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts index f1294fba..b8866126 100644 --- a/tests/internal-tests/src/__tests__/basic.spec.ts +++ b/tests/internal-tests/src/__tests__/basic.spec.ts @@ -83,7 +83,6 @@ describeForAllPlatforms( statusReport: { label: `Copy file0`, description: `Copy file0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -97,7 +96,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.FILE_ON_DISK }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) @@ -132,7 +133,6 @@ describeForAllPlatforms( statusReport: { label: `Copy file0`, description: `Copy file0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -146,7 +146,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.FILE_ON_DISK }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) @@ -188,7 +190,6 @@ describeForAllPlatforms( statusReport: { label: `Copy quantel clip0`, description: `Copy clip0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -202,7 +203,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts index 5f4ad641..f2b109c8 100644 --- a/tests/internal-tests/src/__tests__/issues.spec.ts +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -549,7 +549,6 @@ function addCopyFileExpectation( statusReport: { label: `Copy file0`, description: `Copy file0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -563,7 +562,9 @@ function addCopyFileExpectation( }, version: { type: Expectation.Version.Type.FILE_ON_DISK }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) } diff --git a/tests/internal-tests/src/__tests__/quantel.spec.ts b/tests/internal-tests/src/__tests__/quantel.spec.ts index ed087d05..5be65bdc 100644 --- a/tests/internal-tests/src/__tests__/quantel.spec.ts +++ b/tests/internal-tests/src/__tests__/quantel.spec.ts @@ -57,7 +57,6 @@ describeForAllPlatforms( statusReport: { label: `Copy quantel clip0`, description: `Copy clip0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -71,7 +70,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) @@ -112,7 +113,6 @@ describeForAllPlatforms( statusReport: { label: `Copy quantel clip0`, description: `Copy clip0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -126,7 +126,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) @@ -202,7 +204,6 @@ describeForAllPlatforms( statusReport: { label: `Copy quantel clip0`, description: `Copy clip0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -216,7 +217,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) @@ -263,7 +266,6 @@ describeForAllPlatforms( statusReport: { label: `Copy quantel clip0`, description: `Copy clip0 because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -277,7 +279,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }), }) From 9625276ab130eb14487128e9b6907eb96ab7b10b Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Tue, 26 Mar 2024 16:29:32 +0100 Subject: [PATCH 10/30] chore: fix test pt.2 --- tests/internal-tests/src/__tests__/lib/setupEnv.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index e901a756..b12a9776 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -81,6 +81,7 @@ const defaultTestConfig: SingleAppConfig = { sourcePackageStabilityThreshold: 0, // Disabling this to speed up the tests costMultiplier: 1, considerCPULoad: null, + pickUpCriticalExpectationsOnly: false, }, quantelHTTPTransformerProxy: { port: 0, @@ -94,6 +95,7 @@ const defaultTestConfig: SingleAppConfig = { minRunningApps: 1, spinDownTime: 0, maxAppKeepalive: 6 * 3600 * 1000, // 6 hrs, the default + minCriticalWorkerApps: 0, worker: { resourceId: '', networkIds: [], @@ -259,8 +261,7 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise | null ) => { - if (debugLogging) - console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) + if (debugLogging) console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) let container = containerStatuses[containerId] if (!container) { From 85906186fdfa7d0051638d49f3f600bbba950b0c Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Tue, 26 Mar 2024 16:35:20 +0100 Subject: [PATCH 11/30] chore: remove comment --- apps/appcontainer-node/packages/generic/src/appContainer.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 9eead331..a7198088 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -55,7 +55,6 @@ export class AppContainer { private busyPorts = new Set() private apps: Map< - // <- TODO: Needs to understand which apps are special and which aren't AppId, { process: cp.ChildProcess @@ -76,7 +75,7 @@ export class AppContainer { start: number } > = new Map() - private availableApps: Map = new Map() // <- needs to be smarter + private availableApps: Map = new Map() private websocketServer?: WebsocketServer private monitorAppsTimer: NodeJS.Timer | undefined From 912a59df62e2c3911179ade7865f2789c584cec9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:15:55 +0000 Subject: [PATCH 12/30] chore(deps): bump @supercharge/promise-pool from 2.4.0 to 3.2.0 Bumps [@supercharge/promise-pool](https://github.com/superchargejs/promise-pool) from 2.4.0 to 3.2.0. - [Changelog](https://github.com/supercharge/promise-pool/blob/main/CHANGELOG.md) - [Commits](https://github.com/superchargejs/promise-pool/compare/v2.4.0...v3.2.0) --- updated-dependencies: - dependency-name: "@supercharge/promise-pool" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- shared/packages/expectationManager/package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/packages/expectationManager/package.json b/shared/packages/expectationManager/package.json index 63c9c900..f3c87827 100644 --- a/shared/packages/expectationManager/package.json +++ b/shared/packages/expectationManager/package.json @@ -20,7 +20,7 @@ "dependencies": { "@sofie-package-manager/api": "1.50.2", "@sofie-package-manager/worker": "1.50.2", - "@supercharge/promise-pool": "^2.4.0", + "@supercharge/promise-pool": "^3.2.0", "underscore": "^1.12.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", diff --git a/yarn.lock b/yarn.lock index 74b2e8b3..49d19c48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2009,7 +2009,7 @@ __metadata: dependencies: "@sofie-package-manager/api": "npm:1.50.2" "@sofie-package-manager/worker": "npm:1.50.2" - "@supercharge/promise-pool": "npm:^2.4.0" + "@supercharge/promise-pool": "npm:^3.2.0" jest: "npm:*" rimraf: "npm:^5.0.5" type-fest: "npm:3.13.1" @@ -2056,10 +2056,10 @@ __metadata: languageName: unknown linkType: soft -"@supercharge/promise-pool@npm:^2.4.0": - version: 2.4.0 - resolution: "@supercharge/promise-pool@npm:2.4.0" - checksum: 10/bcd55a44a09575cc9c54845112587abab4a90d965b389534b02be1b3dda61190553f874bbe7bff0fccd2ef74493912bf093540de3ebb9ad5b145f9124b69a391 +"@supercharge/promise-pool@npm:^3.2.0": + version: 3.2.0 + resolution: "@supercharge/promise-pool@npm:3.2.0" + checksum: 10/c6c653a7a56cd93c103266673afe2ea0887091a286f766f9777ef5e3a416cb0faf1c5cad64f3f68de903dbfeb82ecd4e7edae5a35c4957158aa8b1730bc0c48a languageName: node linkType: hard From 39978a96030ce5757db2941f72a7efac61240928 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:16:44 +0000 Subject: [PATCH 13/30] chore(deps): bump atem-connection from 3.2.0 to 3.5.0 Bumps [atem-connection](https://github.com/nrkno/sofie-atem-connection) from 3.2.0 to 3.5.0. - [Changelog](https://github.com/nrkno/sofie-atem-connection/blob/master/CHANGELOG.md) - [Commits](https://github.com/nrkno/sofie-atem-connection/compare/v3.2.0...v3.5.0) --- updated-dependencies: - dependency-name: atem-connection dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 74b2e8b3..647c083f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -989,14 +989,14 @@ __metadata: languageName: node linkType: hard -"@julusian/freetype2@npm:^1.1.0": - version: 1.1.1 - resolution: "@julusian/freetype2@npm:1.1.1" +"@julusian/freetype2@npm:^1.1.2": + version: 1.1.2 + resolution: "@julusian/freetype2@npm:1.1.2" dependencies: node-addon-api: "npm:^5.0.0" node-gyp: "npm:latest" pkg-prebuilds: "npm:^0.2.1" - checksum: 10/b1119074a62a4562169f3fb5cb03d01b773697b66da71ef3ae92a5e14c4a21a6b94a7f88307cf89b081a9fe7f8dd2b044672746496921f81f3fffc34b87b9448 + checksum: 10/7134fc9593c9dbfdb89e5eca0d11b96ae244de284b750681790c5bb61725721cceb39b488e86bd8ac2b1bc7e9ae65cf4dc5a3e52af59126b80df6e026448363a languageName: node linkType: hard @@ -3246,20 +3246,20 @@ __metadata: linkType: hard "atem-connection@npm:^3.2.0": - version: 3.2.0 - resolution: "atem-connection@npm:3.2.0" + version: 3.5.0 + resolution: "atem-connection@npm:3.5.0" dependencies: - "@julusian/freetype2": "npm:^1.1.0" + "@julusian/freetype2": "npm:^1.1.2" debug: "npm:^4.3.4" eventemitter3: "npm:^4.0.7" exit-hook: "npm:^2.2.1" nanotimer: "npm:^0.3.15" p-lazy: "npm:^3.1.0" p-queue: "npm:^6.6.2" - threadedclass: "npm:^1.1.1" - tslib: "npm:^2.3.1" - wavefile: "npm:^8.4.4" - checksum: 10/ef2ed51728d36fab3f54681577a423c3e369b9964be17a2187e97cb6585f9ef3a16ed5a23c71f4c029c3d8da73f2492ab4f0f316536f8b68a650265b10dd5918 + threadedclass: "npm:^1.2.1" + tslib: "npm:^2.6.2" + wavefile: "npm:^8.4.6" + checksum: 10/42a60b31dd0ef704231d00f8e8144a7af62b69ccda47559fed8b56c05b0c6d783112f82c60ca0d2eb7a8317f964b0eee494014718ee3730db6d70eb94b831147 languageName: node linkType: hard @@ -11619,7 +11619,7 @@ __metadata: languageName: node linkType: hard -"threadedclass@npm:^1.1.1": +"threadedclass@npm:^1.2.1": version: 1.2.1 resolution: "threadedclass@npm:1.2.1" dependencies: @@ -11833,7 +11833,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.1": +"tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.4.0, tslib@npm:^2.5.1, tslib@npm:^2.6.2": version: 2.6.2 resolution: "tslib@npm:2.6.2" checksum: 10/bd26c22d36736513980091a1e356378e8b662ded04204453d353a7f34a4c21ed0afc59b5f90719d4ba756e581a162ecbf93118dc9c6be5acf70aa309188166ca @@ -12259,7 +12259,7 @@ __metadata: languageName: node linkType: hard -"wavefile@npm:^8.4.4": +"wavefile@npm:^8.4.6": version: 8.4.6 resolution: "wavefile@npm:8.4.6" dependencies: From 92f7b12ee1e9f82ce208992c827f02d63b6c7a1f Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 28 Mar 2024 14:13:01 +0100 Subject: [PATCH 14/30] chore: create test suite for scaling control inside AppContainer --- .../packages/generic/package.json | 4 +- .../__mocks__/@sofie-package-manager/api.ts | 57 +++++ .../generic/src/__mocks__/child_process.ts | 74 ++++++ .../generic/src/__mocks__/workerAgentApi.ts | 45 ++++ .../generic/src/__mocks__/workforceApi.ts | 31 +++ .../generic/src/__tests__/autoScaling.spec.ts | 231 ++++++++++++++++++ .../generic/src/__tests__/lib/containers.ts | 109 +++++++++ .../generic/src/__tests__/lib/setupEnv.ts | 75 ++++++ 8 files changed, 624 insertions(+), 2 deletions(-) create mode 100644 apps/appcontainer-node/packages/generic/src/__mocks__/@sofie-package-manager/api.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__mocks__/child_process.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__mocks__/workforceApi.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts create mode 100644 apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts diff --git a/apps/appcontainer-node/packages/generic/package.json b/apps/appcontainer-node/packages/generic/package.json index 8b3414ba..40d52c0a 100644 --- a/apps/appcontainer-node/packages/generic/package.json +++ b/apps/appcontainer-node/packages/generic/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "run -T rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "__test": "jest" + "test": "jest" }, "peerDependencies": { "@sofie-automation/shared-lib": "*" @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/@sofie-package-manager/api.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/@sofie-package-manager/api.ts new file mode 100644 index 00000000..6756f9b4 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/@sofie-package-manager/api.ts @@ -0,0 +1,57 @@ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import EventEmitter from 'events' + +const packageManagerAPI: any = jest.createMockFromModule('@sofie-package-manager/api') +const realPackageManagerAPI = jest.requireActual('@sofie-package-manager/api') + +type ClientType = 'N/A' | 'workerAgent' | 'expectationManager' | 'appContainer' + +class MockClientConnection extends EventEmitter { + constructor() { + super() + } + + public clientType: ClientType = 'N/A' + public clientId = '' +} + +export class WebsocketServer extends EventEmitter { + constructor(public _port: number, public _logger: any, connectionClb: (client: MockClientConnection) => void) { + super() + WebsocketServer.connectionClb = connectionClb + } + static connectionClb: (connection: MockClientConnection) => void + + static openConnections: MockClientConnection[] = [] + + static mockNewConnection(clientId: string, clientType: ClientType): MockClientConnection { + const newConnection = new MockClientConnection() + newConnection.clientId = clientId + newConnection.clientType = clientType + WebsocketServer.openConnections.push(newConnection) + WebsocketServer.connectionClb(newConnection) + return newConnection + } + + terminate() { + WebsocketServer.openConnections.forEach((connection) => { + connection.emit('close') + }) + this.emit('close') + } +} +packageManagerAPI.WebsocketServer = WebsocketServer + +// these are various utilities, not really a part of the API +packageManagerAPI.initializeLogger = realPackageManagerAPI.initializeLogger +packageManagerAPI.setupLogger = realPackageManagerAPI.setupLogger +packageManagerAPI.protectString = realPackageManagerAPI.protectString +packageManagerAPI.unprotectString = realPackageManagerAPI.unprotectString +packageManagerAPI.literal = realPackageManagerAPI.literal +packageManagerAPI.mapEntries = realPackageManagerAPI.mapEntries +packageManagerAPI.findValue = realPackageManagerAPI.findValue +packageManagerAPI.DataStore = realPackageManagerAPI.DataStore +packageManagerAPI.stringifyError = realPackageManagerAPI.stringifyError +packageManagerAPI.waitTime = realPackageManagerAPI.waitTime + +module.exports = packageManagerAPI diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/child_process.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/child_process.ts new file mode 100644 index 00000000..9984c5c4 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/child_process.ts @@ -0,0 +1,74 @@ +import EventEmitter from 'events' + +/* eslint-disable no-console */ + +const child_process: any = jest.createMockFromModule('child_process') + +async function pExec(_commandString: string, _options: any): Promise<{ stdout: string; stderr: string }> { + const NOOP = { stdout: '', stderr: '' } + return NOOP +} +function exec( + commandString: string, + options?: any, + cb?: (error: any | null, result: { stdout: string; stderr: string } | null) => void +): void { + if (typeof options === 'function' && cb === undefined) { + cb = options + options = {} + } + pExec(commandString, options) + .then((result) => cb?.(null, result)) + .catch((err) => cb?.(err, null)) +} +child_process.exec = exec + +const allProcesses: SpawnedProcess[] = [] +let mockOnNewProcessClb: null | ((process: SpawnedProcess) => void) = null +function spawn(command: string, args: string[] = []) { + const spawned = new SpawnedProcess(command, args) + mockOnNewProcessClb?.(spawned) + allProcesses.push(spawned) + + spawned.on('exit', () => { + const index = allProcesses.indexOf(spawned) + allProcesses.splice(index, 1) + }) + return spawned +} +child_process.spawn = spawn + +function mockOnNewProcess(clb: (process: SpawnedProcess) => void) { + mockOnNewProcessClb = clb +} +child_process.mockOnNewProcess = mockOnNewProcess + +function mockListAllProcesses(): SpawnedProcess[] { + return allProcesses +} +child_process.mockListAllProcesses = mockListAllProcesses + +function mockClearAllProcesses(): void { + allProcesses.length = 0 +} +child_process.mockClearAllProcesses = mockClearAllProcesses + +class SpawnedProcess extends EventEmitter { + public stdout = new EventEmitter() + public stderr = new EventEmitter() + public pid: number + + constructor(public command: string, public args: string[]) { + super() + this.pid = Date.now() + } + + kill() { + this.emit('exit') + this.stdout.emit('end') + this.stderr.emit('end') + return true + } +} + +module.exports = child_process diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts new file mode 100644 index 00000000..13b70cc6 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts @@ -0,0 +1,45 @@ +import { + AppContainerWorkerAgent, + AdapterServerOptions, + LogLevel, + Expectation, + ReturnTypeDoYouSupportExpectation, + PackageContainerExpectation, + AppContainerId, +} from '@sofie-package-manager/api' + +export class WorkerAgentAPI implements AppContainerWorkerAgent.WorkerAgent { + constructor( + public id: AppContainerId, + methods: AppContainerWorkerAgent.AppContainer, + _options: AdapterServerOptions + ) { + WorkerAgentAPI.mockAppContainer = methods + } + + static mockAppContainer: AppContainerWorkerAgent.AppContainer | null = null + + type = '' + + async setLogLevel(_logLevel: LogLevel): Promise { + return + } + async _debugKill(): Promise { + return + } + async doYouSupportExpectation(_exp: Expectation.Any): Promise { + return { + support: true, + } + } + async doYouSupportPackageContainer( + _packageContainer: PackageContainerExpectation + ): Promise { + return { + support: true, + } + } + async setSpinDownTime(_spinDownTime: number): Promise { + return + } +} diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/workforceApi.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/workforceApi.ts new file mode 100644 index 00000000..0a94b393 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/workforceApi.ts @@ -0,0 +1,31 @@ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { AppContainerId, LoggerInstance, WorkForceAppContainer } from '@sofie-package-manager/api' +import { EventEmitter } from 'events' + +export class WorkforceAPI extends EventEmitter implements WorkForceAppContainer.WorkForce { + constructor(public id: AppContainerId, _loger: LoggerInstance) { + super() + } + + connected = false + public static mockAvailableApps: AppDesc[] = [] + public static mockMethods: Record Promise> = {} + + registerAvailableApps = async (availableApps: AppDesc[]): Promise => { + WorkforceAPI.mockAvailableApps = availableApps + return + } + init = async (_connectionOptions: any, methods: any) => { + WorkforceAPI.mockMethods = methods + this.connected = true + setImmediate(() => { + this.emit('connected') + }) + } + + terminate = () => { + this.emit('disconnected') + } +} + +type AppDesc = { appType: `@@protectedString/AppType/${string}` } diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts new file mode 100644 index 00000000..3f98b25d --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts @@ -0,0 +1,231 @@ +jest.mock('child_process') +jest.mock('../workforceApi.ts') +jest.mock('../workerAgentApi.ts') + +//@ts-ignore mock +import { mockListAllProcesses, mockClearAllProcesses } from 'child_process' +import { prepareTestEnviromnent, setupAppContainer, setupWorkers } from './lib/setupEnv' +import { WorkforceAPI } from '../workforceApi' +import { getFileCopyExpectation, getPackageContainerExpectation } from './lib/containers' +import { sleep } from '@sofie-automation/server-core-integration' +import { WorkerAgentAPI } from '../workerAgentApi' + +jest.setTimeout(10000) + +describe('Auto-scaling', () => { + beforeAll(async () => { + await prepareTestEnviromnent(false) + }) + + afterEach(async () => { + mockClearAllProcesses() + }) + + it('Spins up 2 as the minimal amount of workers', async () => { + const MIN_RUNNING_APPS = 2 + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + }) + + await appContainer.init() + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Spins up 0 as the minimal amount of workers', async () => { + const MIN_RUNNING_APPS = 0 + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + }) + + await appContainer.init() + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Responds to requests to spin up new workers if needed for an expectation', async () => { + const MIN_RUNNING_APPS = 0 + const MAX_RUNNING_APPS = 5 + const TARGET_RUNNING_APPS = 1 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + }) + + await appContainer.init() + + const expectation0 = getFileCopyExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) + + expect(result0.success).toBeTruthy() + } + + expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Responds to requests to spin up new workers if needed for a package container', async () => { + const MIN_RUNNING_APPS = 0 + const MAX_RUNNING_APPS = 5 + const TARGET_RUNNING_APPS = 1 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + }) + + await appContainer.init() + + const expectation0 = getPackageContainerExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForPackageContainer(expectation0) + + expect(result0.success).toBeTruthy() + } + + expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Refuses to start up new workers if already at `maxRunningApps`', async () => { + const MIN_RUNNING_APPS = 0 + const MAX_RUNNING_APPS = 0 + const TARGET_RUNNING_APPS = 0 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + }) + + await appContainer.init() + + const expectation0 = getFileCopyExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) + + expect(result0.success).toBeFalsy() + } + + expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Spins down workers if requested and more than `minRunningApps`', async () => { + const MIN_RUNNING_APPS = 0 + const MAX_RUNNING_APPS = 5 + const INTERMEDIATE_RUNNING_APPS = 1 + const TARGET_RUNNING_APPS = 0 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + }) + + await appContainer.init() + + const expectation0 = getFileCopyExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) + + expect(result0.success).toBeTruthy() + } + + expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) + + //@ts-ignore mock + await WorkerAgentAPI.mockAppContainer.requestSpinDown() + + expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) + + appContainer.terminate() + }) + + it('Refuses to spin down workers if requested and less than `minRunningApps`', async () => { + const MIN_RUNNING_APPS = 1 + const MAX_RUNNING_APPS = 5 + const INTERMEDIATE_RUNNING_APPS = 1 + const TARGET_RUNNING_APPS = 1 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + }) + + await appContainer.init() + + const expectation0 = getFileCopyExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) + + expect(result0.success).toBeTruthy() + } + + expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) + + //@ts-ignore mock + await WorkerAgentAPI.mockAppContainer.requestSpinDown() + + expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) + + appContainer.terminate() + }) +}) + +function getWorkerCount() { + const processes = mockListAllProcesses() + return processes.filter((item: any) => item.args.find((arg: string) => arg.match(/--workerId/))).length +} diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts new file mode 100644 index 00000000..4754bede --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts @@ -0,0 +1,109 @@ +import { + Accessor, + AccessorId, + AccessorOnPackage, + Expectation, + ExpectationId, + ExpectationManagerId, + ExpectedPackageId, + PackageContainerExpectation, + PackageContainerId, + literal, + protectString, +} from '@sofie-package-manager/api' +export const STEP0 = protectString('step0') +export const MANAGER0 = protectString('manager0') +export const PACKAGE0 = protectString('package0') +export const LOCAL0 = protectString('local0') +export const SOURCE0 = protectString('source0') +export const TARGET0 = protectString('target0') + +export function getAccessors(containerId: PackageContainerId): { [accessorId: AccessorId]: Accessor.Any } { + return { + [LOCAL0]: literal({ + type: Accessor.AccessType.LOCAL_FOLDER, + folderPath: `/sources/${containerId}/`, + allowWrite: false, + allowRead: true, + label: 'Test', + }), + } +} + +export function getLocalSource( + containerId: PackageContainerId, + filePath: string +): Expectation.SpecificPackageContainerOnPackage.FileSource { + return { + containerId: containerId, + label: `Label ${containerId}`, + accessors: { + [LOCAL0]: literal({ + type: Accessor.AccessType.LOCAL_FOLDER, + filePath: filePath, + folderPath: `/sources/${containerId}/`, + allowRead: true, + }), + }, + } +} +export function getLocalTarget( + containerId: PackageContainerId, + filePath: string +): Expectation.SpecificPackageContainerOnPackage.FileTarget { + return { + containerId: containerId, + label: `Label ${containerId}`, + accessors: { + [LOCAL0]: literal({ + type: Accessor.AccessType.LOCAL_FOLDER, + filePath: filePath, + folderPath: `/targets/${containerId}/`, + allowRead: true, + allowWrite: true, + }), + }, + } +} + +export function getFileCopyExpectation(): Expectation.FileCopy { + return literal({ + id: STEP0, + priority: 0, + managerId: MANAGER0, + fromPackages: [{ id: PACKAGE0, expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: '', + sendReport: true, + }, + startRequirement: { + sources: [getLocalSource(SOURCE0, 'file0Source.mp4')], + }, + endRequirement: { + targets: [getLocalTarget(TARGET0, 'myFolder/file0Target.mp4')], + content: { + filePath: 'file0Target.mp4', + }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }) +} + +export function getPackageContainerExpectation(): PackageContainerExpectation { + return literal({ + id: SOURCE0, + accessors: getAccessors(SOURCE0), + cronjobs: {}, + label: 'Mock Expectation', + managerId: MANAGER0, + monitors: { + packages: { + label: 'Mock Package Monitor', + targetLayers: ['layer0'], + }, + }, + }) +} diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts new file mode 100644 index 00000000..4ed21bc6 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts @@ -0,0 +1,75 @@ +import { + AppContainerConfig, + AppContainerProcessConfig, + LogLevel, + ProcessConfig, + initializeLogger, + literal, + protectString, + setupLogger, + WebsocketServer, +} from '@sofie-package-manager/api' +// @ts-ignore mock +import { mockOnNewProcess } from 'child_process' +import { AppContainer } from '../../appContainer' +import deepExtend from 'deep-extend' + +export async function prepareTestEnviromnent(debugLogging: boolean): Promise { + const config: { process: ProcessConfig } = { + process: { + certificates: [], + logPath: undefined, + unsafeSSL: false, + logLevel: debugLogging ? LogLevel.DEBUG : LogLevel.INFO, + }, + } + + initializeLogger(config) +} + +export async function setupAppContainer(partialAppContainerConfig: Partial): Promise { + const config = literal({ + appContainer: deepExtend( + { + appContainerId: protectString('app0'), + maxAppKeepalive: 1000, + maxRunningApps: 10, + minRunningApps: 1, + port: 9090, + spinDownTime: 1000, + minCriticalWorkerApps: 0, + worker: { + considerCPULoad: null, + costMultiplier: 1, + networkIds: [], + resourceId: '', + windowsDriveLetters: [], + }, + workforceURL: null, + }, + partialAppContainerConfig + ), + process: { + certificates: [], + logLevel: undefined, + logPath: undefined, + unsafeSSL: false, + }, + }) + + const logger = setupLogger(config, '', undefined, undefined, undefined, undefined) + + return new AppContainer(logger, config) +} + +export async function setupWorkers(): Promise { + mockOnNewProcess((cp: any) => { + setImmediate(() => { + const match = cp.args.find((arg: string) => arg.match(/--workerId=(\w+)/)) + expect(match).toBeTruthy() + const workerIdMatch = match.match(/--workerId=(\w+)/) + // @ts-ignore mock + WebsocketServer.mockNewConnection(workerIdMatch[1], 'workerAgent') + }) + }) +} From 88193dbf06b9be905218e729da6dbb74dd2ba543 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 28 Mar 2024 15:04:47 +0100 Subject: [PATCH 15/30] chore: add test suite for minCriticalWorkerApps --- .../__tests__/criticalExpectations.spec.ts | 99 +++++++++++++++++++ .../generic/src/__tests__/lib/containers.ts | 4 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts new file mode 100644 index 00000000..19611289 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts @@ -0,0 +1,99 @@ +jest.mock('child_process') +jest.mock('../workforceApi.ts') +jest.mock('../workerAgentApi.ts') + +//@ts-ignore mock +import { mockListAllProcesses, mockClearAllProcesses } from 'child_process' +import { prepareTestEnviromnent, setupAppContainer, setupWorkers } from './lib/setupEnv' +import { WorkforceAPI } from '../workforceApi' +import { getFileCopyExpectation } from './lib/containers' +import { sleep } from '@sofie-automation/server-core-integration' +import { WorkerAgentAPI } from '../workerAgentApi' + +jest.setTimeout(10000) + +describe('Critical worker Apps', () => { + beforeAll(async () => { + await prepareTestEnviromnent(false) + }) + + afterEach(async () => { + mockClearAllProcesses() + }) + + it('Spins up 2 critical expectaiton workers', async () => { + const MIN_RUNNING_APPS = 0 + const MIN_CRITICAL_WORKER_APPS = 2 + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, + }) + + await appContainer.init() + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS + MIN_CRITICAL_WORKER_APPS) + + appContainer.terminate() + }) + + it('Spins up 2 critical expectaiton workers and one regular worker', async () => { + const MIN_RUNNING_APPS = 1 + const MIN_CRITICAL_WORKER_APPS = 2 + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, + }) + + await appContainer.init() + + expect(getWorkerCount()).toBe(MIN_RUNNING_APPS + MIN_CRITICAL_WORKER_APPS) + + appContainer.terminate() + }) + + it('Refuses to spin down critical workers', async () => { + const MIN_RUNNING_APPS = 0 + const MAX_RUNNING_APPS = 5 + const MIN_CRITICAL_WORKER_APPS = 1 + + const appContainer = await setupAppContainer({ + minRunningApps: MIN_RUNNING_APPS, + maxRunningApps: MAX_RUNNING_APPS, + minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, + }) + + await appContainer.init() + + const expectation0 = getFileCopyExpectation() + + await setupWorkers() + + // Ensure that the initial state has settled + await sleep(500) + + expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) + + { + // @ts-ignore + const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) + + console.log(result0) + + expect(result0.success).toBeTruthy() + } + + expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) + + //@ts-ignore mock + await WorkerAgentAPI.mockAppContainer.requestSpinDown() + + expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) + + appContainer.terminate() + }) +}) + +function getWorkerCount() { + const processes = mockListAllProcesses() + return processes.filter((item: any) => item.args.find((arg: string) => arg.match(/--workerId/))).length +} diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts index 4754bede..1072a710 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/containers.ts @@ -88,7 +88,9 @@ export function getFileCopyExpectation(): Expectation.FileCopy { }, version: { type: Expectation.Version.Type.FILE_ON_DISK }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }) } From 097600d5c8475d5df79f11449c7d476f259fb525 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 28 Mar 2024 17:17:22 +0100 Subject: [PATCH 16/30] fix: non-auto-scaling critical expecation workers --- .../generic/src/__mocks__/workerAgentApi.ts | 6 +- .../generic/src/__tests__/autoScaling.spec.ts | 24 ++-- .../__tests__/criticalExpectations.spec.ts | 27 ++--- .../generic/src/__tests__/lib/setupEnv.ts | 1 + .../packages/generic/src/appContainer.ts | 109 ++++++++---------- 5 files changed, 75 insertions(+), 92 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts index 13b70cc6..ebd61806 100644 --- a/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts @@ -6,6 +6,7 @@ import { ReturnTypeDoYouSupportExpectation, PackageContainerExpectation, AppContainerId, + WorkerAgentId, } from '@sofie-package-manager/api' export class WorkerAgentAPI implements AppContainerWorkerAgent.WorkerAgent { @@ -14,10 +15,11 @@ export class WorkerAgentAPI implements AppContainerWorkerAgent.WorkerAgent { methods: AppContainerWorkerAgent.AppContainer, _options: AdapterServerOptions ) { - WorkerAgentAPI.mockAppContainer = methods + console.log(methods.id, methods) + WorkerAgentAPI.mockAppContainer[methods.id] = methods } - static mockAppContainer: AppContainerWorkerAgent.AppContainer | null = null + public static mockAppContainer: Record = {} type = '' diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts index 3f98b25d..17eda374 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts @@ -27,6 +27,8 @@ describe('Auto-scaling', () => { minRunningApps: MIN_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) @@ -40,6 +42,8 @@ describe('Auto-scaling', () => { minRunningApps: MIN_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() expect(getWorkerCount()).toBe(MIN_RUNNING_APPS) @@ -57,12 +61,12 @@ describe('Auto-scaling', () => { maxRunningApps: MAX_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() const expectation0 = getFileCopyExpectation() - await setupWorkers() - // Ensure that the initial state has settled await sleep(500) @@ -90,12 +94,12 @@ describe('Auto-scaling', () => { maxRunningApps: MAX_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() const expectation0 = getPackageContainerExpectation() - await setupWorkers() - // Ensure that the initial state has settled await sleep(500) @@ -123,12 +127,12 @@ describe('Auto-scaling', () => { maxRunningApps: MAX_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() const expectation0 = getFileCopyExpectation() - await setupWorkers() - // Ensure that the initial state has settled await sleep(500) @@ -157,12 +161,12 @@ describe('Auto-scaling', () => { maxRunningApps: MAX_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() const expectation0 = getFileCopyExpectation() - await setupWorkers() - // Ensure that the initial state has settled await sleep(500) @@ -196,12 +200,12 @@ describe('Auto-scaling', () => { maxRunningApps: MAX_RUNNING_APPS, }) + await setupWorkers() + await appContainer.init() const expectation0 = getFileCopyExpectation() - await setupWorkers() - // Ensure that the initial state has settled await sleep(500) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts index 19611289..6b81cb85 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts @@ -5,8 +5,6 @@ jest.mock('../workerAgentApi.ts') //@ts-ignore mock import { mockListAllProcesses, mockClearAllProcesses } from 'child_process' import { prepareTestEnviromnent, setupAppContainer, setupWorkers } from './lib/setupEnv' -import { WorkforceAPI } from '../workforceApi' -import { getFileCopyExpectation } from './lib/containers' import { sleep } from '@sofie-automation/server-core-integration' import { WorkerAgentAPI } from '../workerAgentApi' @@ -29,6 +27,8 @@ describe('Critical worker Apps', () => { minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, }) + await setupWorkers() + await appContainer.init() expect(getWorkerCount()).toBe(MIN_RUNNING_APPS + MIN_CRITICAL_WORKER_APPS) @@ -44,6 +44,8 @@ describe('Critical worker Apps', () => { minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, }) + await setupWorkers() + await appContainer.init() expect(getWorkerCount()).toBe(MIN_RUNNING_APPS + MIN_CRITICAL_WORKER_APPS) @@ -62,30 +64,17 @@ describe('Critical worker Apps', () => { minCriticalWorkerApps: MIN_CRITICAL_WORKER_APPS, }) - await appContainer.init() - - const expectation0 = getFileCopyExpectation() - await setupWorkers() - // Ensure that the initial state has settled - await sleep(500) - - expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) - - { - // @ts-ignore - const result0 = await WorkforceAPI.mockMethods.requestAppTypeForExpectation(expectation0) - - console.log(result0) + await appContainer.init() - expect(result0.success).toBeTruthy() - } + // Ensure that the initial state has settled + await sleep(5000) expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer.requestSpinDown() + await WorkerAgentAPI.mockAppContainer['app0_1'].requestSpinDown() expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts index 4ed21bc6..aeba1391 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts @@ -70,6 +70,7 @@ export async function setupWorkers(): Promise { const workerIdMatch = match.match(/--workerId=(\w+)/) // @ts-ignore mock WebsocketServer.mockNewConnection(workerIdMatch[1], 'workerAgent') + console.log('New worker: ', workerIdMatch[1]) }) }) } diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index a7198088..adea18e5 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -54,27 +54,7 @@ export class AppContainer { private usedInspectPorts = new Set() private busyPorts = new Set() - private apps: Map< - AppId, - { - process: cp.ChildProcess - appType: AppType - /** Set to true if app should be considered for scaling down */ - isAutoScaling: boolean - /** Set to true if the app is only handling playout-critical expectations */ - isOnlyForCriticalExpectations: boolean - /** Set to true when the process is about to be killed */ - toBeKilled: boolean - restarts: number - lastRestart: number - spinDownTime: number - /** If null, there is no websocket connection to the app */ - workerAgentApi: WorkerAgentAPI | null - monitorPing: boolean - lastPing: number - start: number - } - > = new Map() + private apps: Map = new Map() private availableApps: Map = new Map() private websocketServer?: WebsocketServer @@ -405,27 +385,8 @@ export class AppContainer { } for (const [appType, availableApp] of this.availableApps.entries()) { - // Do we already have any instance of the appType running? - let runningApp = Array.from(this.apps.values()).find((app) => { - return app.appType === appType - }) - - if (!runningApp) { - const newAppId = await this._spinUp(appType, true) // todo: make it not die too soon + const runningApp = await this.getRunningOrSpawnScalingApp(appType) - // wait for the app to connect to us: - await tryAfewTimes(async () => { - const app = this.apps.get(newAppId) - if (!app) throw new Error(`Worker "${newAppId}" not found`) - if (app.workerAgentApi) { - return true - } - await waitTime(200) - return false - }, 10) - runningApp = this.apps.get(newAppId) - if (!runningApp) throw new Error(`Worker "${newAppId}" didn't connect in time`) - } if (runningApp?.workerAgentApi) { const result = await runningApp.workerAgentApi.doYouSupportExpectation(exp) if (result.support) { @@ -471,27 +432,8 @@ export class AppContainer { } for (const [appType, availableApp] of this.availableApps.entries()) { - // Do we already have any instance of the appType running? - let runningApp = findValue(this.apps, (_, app) => { - return app.appType === appType - }) - - if (!runningApp) { - const newAppId = await this._spinUp(appType, true) // todo: make it not die too soon + const runningApp = await this.getRunningOrSpawnScalingApp(appType) - // wait for the app to connect to us: - await tryAfewTimes(async () => { - const app = this.apps.get(newAppId) - if (!app) throw new Error(`Worker "${newAppId}" not found`) - if (app.workerAgentApi) { - return true - } - await waitTime(200) - return false - }, 10) - runningApp = this.apps.get(newAppId) - if (!runningApp) throw new Error(`Worker "${newAppId}" didn't connect in time`) - } if (runningApp?.workerAgentApi) { const result = await runningApp.workerAgentApi.doYouSupportPackageContainer(packageContainer) if (result.support) { @@ -513,6 +455,32 @@ export class AppContainer { }, } } + + private async getRunningOrSpawnScalingApp(appType: AppType): Promise { + // Do we already have any instance of the appType running? + let runningApp = findValue(this.apps, (_, app) => { + if (app.isAutoScaling) return false + return app.appType === appType + }) + + if (!runningApp) { + const newAppId = await this._spinUp(appType, true) // todo: make it not die too soon + + // wait for the app to connect to us: + await tryAfewTimes(async () => { + const app = this.apps.get(newAppId) + if (!app) throw new Error(`Worker "${newAppId}" not found`) + if (app.workerAgentApi) { + return true + } + await waitTime(200) + return false + }, 10) + runningApp = this.apps.get(newAppId) + if (!runningApp) throw new Error(`Worker "${newAppId}" didn't connect in time`) + } + return runningApp + } private getNewAppId(): AppId { const newAppId = protectString(`${this.id}_${this.appId++}`) @@ -821,6 +789,25 @@ interface AvailableAppInfo { cost: number } +interface RunningAppInfo { + process: cp.ChildProcess + appType: AppType + /** Set to true if app should be considered for scaling down */ + isAutoScaling: boolean + /** Set to true if the app is only handling playout-critical expectations */ + isOnlyForCriticalExpectations: boolean + /** Set to true when the process is about to be killed */ + toBeKilled: boolean + restarts: number + lastRestart: number + spinDownTime: number + /** If null, there is no websocket connection to the app */ + workerAgentApi: WorkerAgentAPI | null + monitorPing: boolean + lastPing: number + start: number +} + async function tryAfewTimes(cb: () => Promise, maxTries: number) { for (let i = 0; i < maxTries; i++) { if (await cb()) { From 3b3cad6305fe94b6b2642ed0c98e690cdeccbbd5 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 28 Mar 2024 17:24:53 +0100 Subject: [PATCH 17/30] chore: remove console.logs fron tests --- .../packages/generic/src/__mocks__/workerAgentApi.ts | 5 ++++- .../generic/src/__tests__/criticalExpectations.spec.ts | 4 +++- .../packages/generic/src/__tests__/lib/setupEnv.ts | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts index ebd61806..94c5df88 100644 --- a/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts +++ b/apps/appcontainer-node/packages/generic/src/__mocks__/workerAgentApi.ts @@ -15,12 +15,15 @@ export class WorkerAgentAPI implements AppContainerWorkerAgent.WorkerAgent { methods: AppContainerWorkerAgent.AppContainer, _options: AdapterServerOptions ) { - console.log(methods.id, methods) WorkerAgentAPI.mockAppContainer[methods.id] = methods } public static mockAppContainer: Record = {} + public static mockReset(): void { + WorkerAgentAPI.mockAppContainer = {} + } + type = '' async setLogLevel(_logLevel: LogLevel): Promise { diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts index 6b81cb85..1147549b 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts @@ -17,6 +17,8 @@ describe('Critical worker Apps', () => { afterEach(async () => { mockClearAllProcesses() + //@ts-ignore mock + WorkerAgentAPI.mockReset() }) it('Spins up 2 critical expectaiton workers', async () => { @@ -74,7 +76,7 @@ describe('Critical worker Apps', () => { expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer['app0_1'].requestSpinDown() + await WorkerAgentAPI.mockAppContainer['app0_0'].requestSpinDown() expect(getWorkerCount()).toBe(MIN_CRITICAL_WORKER_APPS) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts index aeba1391..4ed21bc6 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts @@ -70,7 +70,6 @@ export async function setupWorkers(): Promise { const workerIdMatch = match.match(/--workerId=(\w+)/) // @ts-ignore mock WebsocketServer.mockNewConnection(workerIdMatch[1], 'workerAgent') - console.log('New worker: ', workerIdMatch[1]) }) }) } From ba1c75aac7bd9d8f032085e8a5377614ef6459e9 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Thu, 28 Mar 2024 17:37:47 +0100 Subject: [PATCH 18/30] chore: improve autoScaling tests --- .../packages/generic/src/__tests__/autoScaling.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts index 17eda374..0fde56b7 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts @@ -182,7 +182,7 @@ describe('Auto-scaling', () => { expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer.requestSpinDown() + await WorkerAgentAPI.mockAppContainer['app0_0'].requestSpinDown() expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) @@ -221,7 +221,7 @@ describe('Auto-scaling', () => { expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer.requestSpinDown() + await WorkerAgentAPI.mockAppContainer['app0_0'].requestSpinDown() expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) From d1839c59fcdba2e610aec6f01c6c4d82256df0b5 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Fri, 29 Mar 2024 13:55:45 +0100 Subject: [PATCH 19/30] chore: improve setupEnv for appContainer tests --- .../generic/src/__tests__/autoScaling.spec.ts | 28 ++++++++++++------- .../__tests__/criticalExpectations.spec.ts | 13 ++------- .../generic/src/__tests__/lib/setupEnv.ts | 20 ++++++++++++- .../packages/generic/src/appContainer.ts | 2 +- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts index 0fde56b7..03d73640 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/autoScaling.spec.ts @@ -3,8 +3,15 @@ jest.mock('../workforceApi.ts') jest.mock('../workerAgentApi.ts') //@ts-ignore mock -import { mockListAllProcesses, mockClearAllProcesses } from 'child_process' -import { prepareTestEnviromnent, setupAppContainer, setupWorkers } from './lib/setupEnv' +import { mockClearAllProcesses } from 'child_process' +import { + getWorkerCount, + getWorkerId, + prepareTestEnviromnent, + resetMocks, + setupAppContainer, + setupWorkers, +} from './lib/setupEnv' import { WorkforceAPI } from '../workforceApi' import { getFileCopyExpectation, getPackageContainerExpectation } from './lib/containers' import { sleep } from '@sofie-automation/server-core-integration' @@ -18,7 +25,7 @@ describe('Auto-scaling', () => { }) afterEach(async () => { - mockClearAllProcesses() + await resetMocks() }) it('Spins up 2 as the minimal amount of workers', async () => { @@ -181,8 +188,11 @@ describe('Auto-scaling', () => { expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) + const workerId = getWorkerId(0) + expect(workerId).toBeDefined() + //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer['app0_0'].requestSpinDown() + await WorkerAgentAPI.mockAppContainer[workerId].requestSpinDown() expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) @@ -220,16 +230,14 @@ describe('Auto-scaling', () => { expect(getWorkerCount()).toBe(INTERMEDIATE_RUNNING_APPS) + const workerId = getWorkerId(0) + expect(workerId).toBeDefined() + //@ts-ignore mock - await WorkerAgentAPI.mockAppContainer['app0_0'].requestSpinDown() + await WorkerAgentAPI.mockAppContainer[workerId].requestSpinDown() expect(getWorkerCount()).toBe(TARGET_RUNNING_APPS) appContainer.terminate() }) }) - -function getWorkerCount() { - const processes = mockListAllProcesses() - return processes.filter((item: any) => item.args.find((arg: string) => arg.match(/--workerId/))).length -} diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts index 1147549b..ee45e573 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/criticalExpectations.spec.ts @@ -2,9 +2,7 @@ jest.mock('child_process') jest.mock('../workforceApi.ts') jest.mock('../workerAgentApi.ts') -//@ts-ignore mock -import { mockListAllProcesses, mockClearAllProcesses } from 'child_process' -import { prepareTestEnviromnent, setupAppContainer, setupWorkers } from './lib/setupEnv' +import { getWorkerCount, prepareTestEnviromnent, resetMocks, setupAppContainer, setupWorkers } from './lib/setupEnv' import { sleep } from '@sofie-automation/server-core-integration' import { WorkerAgentAPI } from '../workerAgentApi' @@ -16,9 +14,7 @@ describe('Critical worker Apps', () => { }) afterEach(async () => { - mockClearAllProcesses() - //@ts-ignore mock - WorkerAgentAPI.mockReset() + await resetMocks() }) it('Spins up 2 critical expectaiton workers', async () => { @@ -83,8 +79,3 @@ describe('Critical worker Apps', () => { appContainer.terminate() }) }) - -function getWorkerCount() { - const processes = mockListAllProcesses() - return processes.filter((item: any) => item.args.find((arg: string) => arg.match(/--workerId/))).length -} diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts index 4ed21bc6..591cad45 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts @@ -8,11 +8,13 @@ import { protectString, setupLogger, WebsocketServer, + WorkerAgentId, } from '@sofie-package-manager/api' // @ts-ignore mock -import { mockOnNewProcess } from 'child_process' +import { mockOnNewProcess, mockListAllProcesses, mockClearAllProcesses } from 'child_process' import { AppContainer } from '../../appContainer' import deepExtend from 'deep-extend' +import { WorkerAgentAPI } from '../../workerAgentApi' export async function prepareTestEnviromnent(debugLogging: boolean): Promise { const config: { process: ProcessConfig } = { @@ -73,3 +75,19 @@ export async function setupWorkers(): Promise { }) }) } + +export function getWorkerCount() { + const processes = mockListAllProcesses() + return processes.filter((item: any) => item.args.find((arg: string) => arg.match(/--workerId/))).length +} + +export function getWorkerId(index: number): WorkerAgentId | undefined { + //@ts-ignore mock + return Object.keys(WorkerAgentAPI.mockAppContainer)[index] +} + +export async function resetMocks(): Promise { + mockClearAllProcesses() + //@ts-ignore mock + WorkerAgentAPI.mockReset() +} diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index adea18e5..8233b88e 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -459,7 +459,7 @@ export class AppContainer { private async getRunningOrSpawnScalingApp(appType: AppType): Promise { // Do we already have any instance of the appType running? let runningApp = findValue(this.apps, (_, app) => { - if (app.isAutoScaling) return false + if (!app.isAutoScaling) return false return app.appType === appType }) From 4d785b2bd5e6383126276cdfbb12e636db502bf8 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Fri, 29 Mar 2024 14:03:32 +0100 Subject: [PATCH 20/30] chore(AppContainer/test): filter out non-error messages from module under test --- .../packages/generic/src/__tests__/lib/setupEnv.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts index 591cad45..5f584d99 100644 --- a/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts +++ b/apps/appcontainer-node/packages/generic/src/__tests__/lib/setupEnv.ts @@ -59,7 +59,7 @@ export async function setupAppContainer(partialAppContainerConfig: Partial level === LogLevel.ERROR) return new AppContainer(logger, config) } From 62ed3f765d2e6160e981014d7ac36b09436aefc4 Mon Sep 17 00:00:00 2001 From: Jan Starzak Date: Tue, 2 Apr 2024 10:54:54 +0200 Subject: [PATCH 21/30] chore: requiredForPlayout moved to workOptions --- tests/internal-tests/src/__tests__/basic.spec.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts index ad413443..648e4cfb 100644 --- a/tests/internal-tests/src/__tests__/basic.spec.ts +++ b/tests/internal-tests/src/__tests__/basic.spec.ts @@ -325,7 +325,6 @@ describeForAllPlatforms( statusReport: { label: `Copy file${i}`, description: `Copy file${i} because test`, - requiredForPlayout: true, displayRank: 0, sendReport: true, }, @@ -339,7 +338,9 @@ describeForAllPlatforms( }, version: { type: Expectation.Version.Type.FILE_ON_DISK }, }, - workOptions: {}, + workOptions: { + requiredForPlayout: true, + }, }) } // console.log(fs.__printAllFiles()) @@ -359,10 +360,8 @@ describeForAllPlatforms( for (const exp of Object.values(expectations)) { const PACKAGE = exp.fromPackages[0].id - packageStatuses.actual[PACKAGE] = - env.containerStatuses[TARGET0].packages[PACKAGE]?.packageStatus?.status - packageStatuses.expected[PACKAGE] = - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + packageStatuses.actual[PACKAGE] = env.containerStatuses[TARGET0].packages[PACKAGE]?.packageStatus?.status + packageStatuses.expected[PACKAGE] = ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY } expect(packageStatuses.actual).toMatchObject(packageStatuses.expected) }, 1000 + env.WAIT_JOB_TIME * 2 + COPY_TIME * 10) From bd5d116bd4b3cf24b18ff9ddb463ddd5f26b9aa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:50 +0000 Subject: [PATCH 22/30] chore(deps): bump @parcel/watcher from 2.3.0 to 2.4.1 Bumps [@parcel/watcher](https://github.com/parcel-bundler/watcher) from 2.3.0 to 2.4.1. - [Release notes](https://github.com/parcel-bundler/watcher/releases) - [Commits](https://github.com/parcel-bundler/watcher/commits) --- updated-dependencies: - dependency-name: "@parcel/watcher" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 104 +++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/yarn.lock b/yarn.lock index 212ee162..b11334ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1666,86 +1666,86 @@ __metadata: languageName: unknown linkType: soft -"@parcel/watcher-android-arm64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-android-arm64@npm:2.3.0" +"@parcel/watcher-android-arm64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-android-arm64@npm:2.4.1" conditions: os=android & cpu=arm64 languageName: node linkType: hard -"@parcel/watcher-darwin-arm64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-darwin-arm64@npm:2.3.0" +"@parcel/watcher-darwin-arm64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-darwin-arm64@npm:2.4.1" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@parcel/watcher-darwin-x64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-darwin-x64@npm:2.3.0" +"@parcel/watcher-darwin-x64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-darwin-x64@npm:2.4.1" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@parcel/watcher-freebsd-x64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-freebsd-x64@npm:2.3.0" +"@parcel/watcher-freebsd-x64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-freebsd-x64@npm:2.4.1" conditions: os=freebsd & cpu=x64 languageName: node linkType: hard -"@parcel/watcher-linux-arm-glibc@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-linux-arm-glibc@npm:2.3.0" +"@parcel/watcher-linux-arm-glibc@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-linux-arm-glibc@npm:2.4.1" conditions: os=linux & cpu=arm & libc=glibc languageName: node linkType: hard -"@parcel/watcher-linux-arm64-glibc@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-linux-arm64-glibc@npm:2.3.0" +"@parcel/watcher-linux-arm64-glibc@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-linux-arm64-glibc@npm:2.4.1" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@parcel/watcher-linux-arm64-musl@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-linux-arm64-musl@npm:2.3.0" +"@parcel/watcher-linux-arm64-musl@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-linux-arm64-musl@npm:2.4.1" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@parcel/watcher-linux-x64-glibc@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-linux-x64-glibc@npm:2.3.0" +"@parcel/watcher-linux-x64-glibc@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-linux-x64-glibc@npm:2.4.1" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@parcel/watcher-linux-x64-musl@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-linux-x64-musl@npm:2.3.0" +"@parcel/watcher-linux-x64-musl@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-linux-x64-musl@npm:2.4.1" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@parcel/watcher-win32-arm64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-win32-arm64@npm:2.3.0" +"@parcel/watcher-win32-arm64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-win32-arm64@npm:2.4.1" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@parcel/watcher-win32-ia32@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-win32-ia32@npm:2.3.0" +"@parcel/watcher-win32-ia32@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-win32-ia32@npm:2.4.1" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@parcel/watcher-win32-x64@npm:2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher-win32-x64@npm:2.3.0" +"@parcel/watcher-win32-x64@npm:2.4.1": + version: 2.4.1 + resolution: "@parcel/watcher-win32-x64@npm:2.4.1" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -1762,21 +1762,21 @@ __metadata: linkType: hard "@parcel/watcher@npm:^2.3.0": - version: 2.3.0 - resolution: "@parcel/watcher@npm:2.3.0" - dependencies: - "@parcel/watcher-android-arm64": "npm:2.3.0" - "@parcel/watcher-darwin-arm64": "npm:2.3.0" - "@parcel/watcher-darwin-x64": "npm:2.3.0" - "@parcel/watcher-freebsd-x64": "npm:2.3.0" - "@parcel/watcher-linux-arm-glibc": "npm:2.3.0" - "@parcel/watcher-linux-arm64-glibc": "npm:2.3.0" - "@parcel/watcher-linux-arm64-musl": "npm:2.3.0" - "@parcel/watcher-linux-x64-glibc": "npm:2.3.0" - "@parcel/watcher-linux-x64-musl": "npm:2.3.0" - "@parcel/watcher-win32-arm64": "npm:2.3.0" - "@parcel/watcher-win32-ia32": "npm:2.3.0" - "@parcel/watcher-win32-x64": "npm:2.3.0" + version: 2.4.1 + resolution: "@parcel/watcher@npm:2.4.1" + dependencies: + "@parcel/watcher-android-arm64": "npm:2.4.1" + "@parcel/watcher-darwin-arm64": "npm:2.4.1" + "@parcel/watcher-darwin-x64": "npm:2.4.1" + "@parcel/watcher-freebsd-x64": "npm:2.4.1" + "@parcel/watcher-linux-arm-glibc": "npm:2.4.1" + "@parcel/watcher-linux-arm64-glibc": "npm:2.4.1" + "@parcel/watcher-linux-arm64-musl": "npm:2.4.1" + "@parcel/watcher-linux-x64-glibc": "npm:2.4.1" + "@parcel/watcher-linux-x64-musl": "npm:2.4.1" + "@parcel/watcher-win32-arm64": "npm:2.4.1" + "@parcel/watcher-win32-ia32": "npm:2.4.1" + "@parcel/watcher-win32-x64": "npm:2.4.1" detect-libc: "npm:^1.0.3" is-glob: "npm:^4.0.3" micromatch: "npm:^4.0.5" @@ -1807,7 +1807,7 @@ __metadata: optional: true "@parcel/watcher-win32-x64": optional: true - checksum: 10/5ba2be3337153f0c26b4a0b3a4f78ee728a96c37855c1cd39a573ac60b68e3116e657404c61b121b3f77f5227ab3d2c94679a816e42e90d1a476d7c783225368 + checksum: 10/c163dff1828fa249c00f24931332dea5a8f2fcd1bfdd0e304ccdf7619c58bff044526fa39241fd2121d2a2141f71775ce3117450d78c4df3070d152282017644 languageName: node linkType: hard From ee251bc3b65d11037f137d35d1f1e2883b65e361 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 07:57:41 +0200 Subject: [PATCH 23/30] chore: minor refactor, doc --- .../packages/generic/src/appContainer.ts | 27 ++++++++++++++----- shared/packages/api/src/appContainer.ts | 3 ++- shared/packages/api/src/config.ts | 4 +-- shared/packages/worker/src/workerAgent.ts | 4 ++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 64f34c2f..5c69b70c 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -225,7 +225,7 @@ export class AppContainer { requestSpinDown: async (): Promise => { const app = this.apps.get(clientId) if (!app || !app.isAutoScaling) return - if (this.getScalingAppCount(app.appType) > this.config.appContainer.minRunningApps) { + if (this.getAutoScalingAppCount(app.appType) > this.config.appContainer.minRunningApps) { this.spinDown(clientId, `Requested by app`).catch((error) => { this.logger.error(`Error when spinning down app "${clientId}": ${stringifyError(error)}`) }) @@ -249,13 +249,15 @@ export class AppContainer { } } - private getScalingAppCount(appType: AppType): number { + /** Returns the number of **auto-scaling** apps */ + private getAutoScalingAppCount(appType: AppType): number { let count = 0 for (const app of this.apps.values()) { if (app.appType === appType && app.isAutoScaling) count++ } return count } + /** Returns the number of playout-critical apps */ private getCriticalExpectationAppCount(appType: AppType): number { let count = 0 for (const app of this.apps.values()) { @@ -504,8 +506,8 @@ export class AppContainer { return newAppId } - async spinUp(appType: AppType, longSpinDownTime = false): Promise { - return this._spinUp(appType, longSpinDownTime) + async spinUp(appType: AppType): Promise { + return this._spinUp(appType) } private async _spinUp( appType: AppType, @@ -526,6 +528,10 @@ export class AppContainer { isAutoScaling = false } + let spinDownTime = this.config.appContainer.spinDownTime + if (longSpinDownTime) { + spinDownTime *= 10 + } this.apps.set(appId, { process: child, appType: appType, @@ -536,7 +542,7 @@ export class AppContainer { isAutoScaling: isAutoScaling, isOnlyForCriticalExpectations: isOnlyForCriticalExpectations, lastPing: Date.now(), - spinDownTime: this.config.appContainer.spinDownTime * (longSpinDownTime ? 10 : 1), + spinDownTime: spinDownTime, workerAgentApi: null, start: Date.now(), }) @@ -695,6 +701,7 @@ export class AppContainer { if (this.config.appContainer.minCriticalWorkerApps !== null) { for (const [appType, appInfo] of this.availableApps.entries()) { if (!appInfo.canRunInCriticalExpectationsOnlyMode) continue + while (this.getCriticalExpectationAppCount(appType) < this.config.appContainer.minCriticalWorkerApps) { await this._spinUp(appType, false, true) } @@ -702,7 +709,7 @@ export class AppContainer { } for (const appType of this.availableApps.keys()) { - while (this.getScalingAppCount(appType) < this.config.appContainer.minRunningApps) { + while (this.getAutoScalingAppCount(appType) < this.config.appContainer.minRunningApps) { await this._spinUp(appType) } } @@ -788,8 +795,9 @@ export class AppContainer { interface AvailableAppInfo { file: string getExecArgs: (appId: AppId, useCriticalOnlyMode: boolean) => string[] - /** Some kind of value, how much it costs to run it, per minute */ + /** Whether the application can be spun up as a critical worker */ canRunInCriticalExpectationsOnlyMode: boolean + /** Some kind of value, how much it costs to run it, per minute */ cost: number } @@ -804,6 +812,11 @@ interface RunningAppInfo { toBeKilled: boolean restarts: number lastRestart: number + /** + * When an App has been idle for longer than the spinDownTime, if might request to be spun down + * (set to 0 to disable) + * [milliseconds] + */ spinDownTime: number /** If null, there is no websocket connection to the app */ workerAgentApi: WorkerAgentAPI | null diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts index e03686b5..1a5e3f84 100644 --- a/shared/packages/api/src/appContainer.ts +++ b/shared/packages/api/src/appContainer.ts @@ -15,7 +15,8 @@ export interface AppContainerConfig { maxRunningApps: number maxAppKeepalive: number spinDownTime: number - minCriticalWorkerApps: number | null + /** Minimum number of critical workers to spin up */ + minCriticalWorkerApps: number worker: { resourceId: string diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index d0c0676d..65deb2bb 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -199,7 +199,7 @@ const appContainerArguments = defineArguments({ }, spinDownTime: { type: 'number', - default: parseInt(process.env.APP_CONTAINER_SPIN_DOWN_TIME || '', 10) || 60 * 1000, + default: parseInt(process.env.APP_CONTAINER_SPIN_DOWN_TIME || '', 10) || 60 * 1000, // ms (1 minute) describe: 'How long a Worker should stay idle before attempting to be spun down', }, @@ -233,7 +233,7 @@ const appContainerArguments = defineArguments({ minCriticalWorkerApps: { type: 'number', default: 0, - describe: 'Number of Workers reserved for fulfilling playout-critical expectations that will be kept runnini', + describe: 'Number of Workers reserved for fulfilling playout-critical expectations that will be kept running', }, }) /** CLI-argument-definitions for the "Single" process */ diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index d135bdae..ff9249ab 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -82,6 +82,7 @@ export class WorkerAgent { private initWorkForceAPIPromise?: { resolve: () => void; reject: (reason?: any) => void } private initAppContainerAPIPromise?: { resolve: () => void; reject: (reason?: any) => void } private cpuTracker = new CPUTracker() + /** When true, this worker should only accept expectation that are critical for playout */ private isOnlyForCriticalExpectations = false private logger: LoggerInstance @@ -307,8 +308,8 @@ export class WorkerAgent { return { support: false, reason: { - tech: 'Worker is reserved for `workOptions.requiredForPlayout` expectations', user: 'Worker is reserved for playout-critical operations', + tech: 'Worker is reserved for `workOptions.requiredForPlayout` expectations', }, } } @@ -373,6 +374,7 @@ export class WorkerAgent { })), } } + /** Set the SpinDown Time [ms] */ public async setSpinDownTime(spinDownTime: number): Promise { this.spinDownTime = spinDownTime this.setupIntervalCheck() From e2e496477fa4ef66f1743e23bfd7292de8cab8b2 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 09:45:27 +0200 Subject: [PATCH 24/30] fix: add missing CLI argument definition --- .../packages/generic/src/appContainer.ts | 2 +- shared/packages/api/src/config.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 5c69b70c..026ead73 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -273,7 +273,7 @@ export class AppContainer { `--logLevel=${getLogLevel()}`, `--workerId=${appId}`, - pickUpCriticalExpectationsOnly ? `--pickUpCriticalExpectationsOnly=1` : '', + pickUpCriticalExpectationsOnly ? `--pickUpCriticalExpectationsOnly=true` : '', `--workforceURL=${this.config.appContainer.workforceURL}`, `--appContainerURL=${'ws://127.0.0.1:' + this.websocketServer?.port}`, diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 65deb2bb..a88007fa 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -164,6 +164,11 @@ const workerArguments = defineArguments({ describe: 'If set, the worker will consider the CPU load of the system it runs on before it accepts jobs. Set to a value between 0 and 1, the worker will accept jobs if the CPU load is below the configured value.', }, + pickUpCriticalExpectationsOnly: { + type: 'boolean', + default: process.env.WORKER_PICK_UP_CRITICAL_EXPECTATIONS_ONLY === '1' || false, + describe: 'If set to 1, the worker will only pick up expectations that are marked as critical for playout.', + }, }) /** CLI-argument-definitions for the AppContainer process */ const appContainerArguments = defineArguments({ @@ -446,8 +451,7 @@ export async function getWorkerConfig(): Promise { considerCPULoad: (typeof argv.considerCPULoad === 'string' ? parseFloat(argv.considerCPULoad) : argv.considerCPULoad) || null, - pickUpCriticalExpectationsOnly: - (typeof argv.pickUpCriticalExpectationsOnly === 'string' ? true : false) || false, + pickUpCriticalExpectationsOnly: argv.pickUpCriticalExpectationsOnly, }, } } From d3683265a5736ab03cf3df8c7b7c3efbda706261 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 09:48:07 +0200 Subject: [PATCH 25/30] chore: minor optimization: set spinDownTime to 0 for not autoscaling apps. Since when isAutoScaling is false, requests for spinDown are ignored anyway. --- .../packages/generic/src/appContainer.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index 026ead73..dbb10751 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -532,6 +532,12 @@ export class AppContainer { if (longSpinDownTime) { spinDownTime *= 10 } + if (!isAutoScaling) { + // If not auto-scaling, disable the spinDownTime + // (to reduce chatter and unnecessary requestSpinDown() calls ) + spinDownTime = 0 + } + this.apps.set(appId, { process: child, appType: appType, @@ -698,7 +704,7 @@ export class AppContainer { } private async spinUpMinimumApps(): Promise { - if (this.config.appContainer.minCriticalWorkerApps !== null) { + if (this.config.appContainer.minCriticalWorkerApps > 0) { for (const [appType, appInfo] of this.availableApps.entries()) { if (!appInfo.canRunInCriticalExpectationsOnlyMode) continue From 15759df5a7461678d894485fa4631c389272cac5 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 09:49:36 +0200 Subject: [PATCH 26/30] chore: fix mkdir mock issue in tests --- .../lib/trackedExpectationAPI.ts | 2 +- tests/internal-tests/src/__mocks__/fs.ts | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/shared/packages/expectationManager/src/expectationTracker/lib/trackedExpectationAPI.ts b/shared/packages/expectationManager/src/expectationTracker/lib/trackedExpectationAPI.ts index 6cb16ca4..ce02c0eb 100644 --- a/shared/packages/expectationManager/src/expectationTracker/lib/trackedExpectationAPI.ts +++ b/shared/packages/expectationManager/src/expectationTracker/lib/trackedExpectationAPI.ts @@ -206,7 +206,7 @@ export class TrackedExpectationAPI { if (!trackedExp.noWorkerAssignedTime) { const now = Date.now() - this.logger.error( + this.logger.debug( `Setting trackedExp.noWorkerAssignedTime of "${expLabel(trackedExp)}" to ${now} (reason: ${ noAssignedWorkerReason.tech })` diff --git a/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts index cfc7e921..a80ed77c 100644 --- a/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -187,6 +187,15 @@ function deleteMock(path: string, orgPath?: string, dir?: MockDirectory): void { delete dir.content[fileName] } } +function existsMock(path: string): boolean { + try { + getMock(path) + return true + } catch (err) { + if ((err as any).code === 'ENOENT') return false + throw err + } +} export function __printAllFiles(): string { const getPaths = (dir: MockDirectory, indent: string): string => { const strs: any[] = [] @@ -368,14 +377,35 @@ export function mkdir( optsOrCallback: { recursive?: boolean } | ((error: any, result?: any) => void), callback?: (error: any, result?: any) => void ): void { + let opts: { recursive?: boolean } if (typeof optsOrCallback === 'function') { callback = optsOrCallback + opts = {} + } else { + opts = optsOrCallback } path = fixPath(path) if (DEBUG_LOG) console.log('fs.mkdir', path) fsMockEmitter.emit('mkdir', path) + try { + // Handle if the directory already exists: + if (existsMock(path)) { + const existing = getMock(path) + if (existing.isDirectory && opts.recursive) { + // don't do anything + return callback?.(undefined, null) + } else { + throw Object.assign(new Error(`EEXIST: file already exists, mkdir "${path}"`), { + errno: 0, + code: 'EEXIST', + syscall: 'mock', + path: path, + }) + } + } + setMock( path, { @@ -384,7 +414,7 @@ export function mkdir( isDirectory: true, content: {}, }, - false + opts.recursive ?? false ) return callback?.(undefined, null) From 7e64cd0b1c7cefd2bd6a6708bc939b8328a1c4a1 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 10:26:22 +0200 Subject: [PATCH 27/30] chore: fixes after dependency updates --- .../packages/generic/src/appContainer.ts | 2 +- .../http-server/packages/generic/package.json | 2 +- .../packages/generic/package.json | 2 +- .../workers/genericWorker/genericWorker.ts | 2 +- shared/packages/worker/src/workerAgent.ts | 2 +- tsconfig.build.json | 4 ++-- yarn.lock | 21 +++++++++++++------ 7 files changed, 22 insertions(+), 13 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index dbb10751..bcf9da3e 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -59,7 +59,7 @@ export class AppContainer { private availableApps: Map = new Map() private websocketServer?: WebsocketServer - private monitorAppsTimer: NodeJS.Timer | undefined + private monitorAppsTimer: NodeJS.Timeout | undefined private initWorkForceApiPromise?: { resolve: () => void; reject: (reason: any) => void } /** diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index 5a45a461..fd5ded6d 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -31,7 +31,7 @@ "@types/koa-router": "^7.4.0", "@types/koa__cors": "^4.0.0", "@types/mime-types": "^2.1.0", - "@types/node": "^14.14.31", + "@types/node": "^18", "@types/underscore": "^1.10.24", "@types/yargs": "^17.0.24", "rimraf": "^5.0.5" diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index 51faab74..94da4e1b 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -34,7 +34,7 @@ "@types/koa-router": "^7.4.4", "@types/koa__cors": "^4.0.0", "@types/mime-types": "^2.1.0", - "@types/node": "^14.14.31", + "@types/node": "^18", "@types/underscore": "^1.10.24", "@types/xml2js": "^0.4.7", "@types/yargs": "^17.0.24", diff --git a/shared/packages/worker/src/worker/workers/genericWorker/genericWorker.ts b/shared/packages/worker/src/worker/workers/genericWorker/genericWorker.ts index 5277dbd2..164980c7 100644 --- a/shared/packages/worker/src/worker/workers/genericWorker/genericWorker.ts +++ b/shared/packages/worker/src/worker/workers/genericWorker/genericWorker.ts @@ -43,7 +43,7 @@ export class GenericWorker extends BaseWorker { /** Contains the result of testing the FFProbe executable. null = all is well, otherwise contains error message */ public testFFProbe: null | string = 'Not initialized' - private monitor: NodeJS.Timer | undefined + private monitor: NodeJS.Timeout | undefined constructor( logger: LoggerInstance, diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index ff9249ab..806ab353 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -76,7 +76,7 @@ export class WorkerAgent { > = new Map() private terminated = false private spinDownTime = 0 - private intervalCheckTimer: NodeJS.Timer | null = null + private intervalCheckTimer: NodeJS.Timeout | null = null private lastWorkTime = 0 private activeMonitors: Map> = new Map() private initWorkForceAPIPromise?: { resolve: () => void; reject: (reason?: any) => void } diff --git a/tsconfig.build.json b/tsconfig.build.json index a0e9a56d..44cbef64 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,7 +1,7 @@ { "compilerOptions": { "module": "commonjs", - "target": "es2018", + "target": "es2020", "noImplicitAny": true, "moduleResolution": "node", "sourceMap": true, @@ -12,7 +12,7 @@ "traceResolution": false, "pretty": true, "lib": [ - "es2018" + "es2020" ], "types": [ "node", diff --git a/yarn.lock b/yarn.lock index b1655cde..22abcbf9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -611,7 +611,7 @@ __metadata: "@types/koa-router": "npm:^7.4.0" "@types/koa__cors": "npm:^4.0.0" "@types/mime-types": "npm:^2.1.0" - "@types/node": "npm:^14.14.31" + "@types/node": "npm:^18" "@types/underscore": "npm:^1.10.24" "@types/yargs": "npm:^17.0.24" koa: "npm:^2.14.1" @@ -1839,7 +1839,7 @@ __metadata: "@types/koa-router": "npm:^7.4.4" "@types/koa__cors": "npm:^4.0.0" "@types/mime-types": "npm:^2.1.0" - "@types/node": "npm:^14.14.31" + "@types/node": "npm:^18" "@types/underscore": "npm:^1.10.24" "@types/xml2js": "npm:^0.4.7" "@types/yargs": "npm:^17.0.24" @@ -2499,10 +2499,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^14.14.31": - version: 14.18.21 - resolution: "@types/node@npm:14.18.21" - checksum: 10/e6f333886c3e9ff455bebdf3fccc719514d76a5a2453f04824dcdd305a779fe29eca8370d66d754c8e1d9f014b9063ef33a26139d50352db18a064e309abc842 +"@types/node@npm:^18": + version: 18.19.29 + resolution: "@types/node@npm:18.19.29" + dependencies: + undici-types: "npm:~5.26.4" + checksum: 10/9a3572b488f875ca1b545cc96980f1cb54dd05da16b2dc0cc3c3cb49ceafc3a5e417f4741c711c7bb81a67a0ddd29f546dcb077e4cb9b98a492fbaf373b1fbdc languageName: node linkType: hard @@ -12024,6 +12026,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: 10/0097779d94bc0fd26f0418b3a05472410408877279141ded2bd449167be1aed7ea5b76f756562cb3586a07f251b90799bab22d9019ceba49c037c76445f7cddd + languageName: node + linkType: hard + "unique-filename@npm:^2.0.0": version: 2.0.1 resolution: "unique-filename@npm:2.0.1" From 4afad993880581e916ff35afe4ef4a798d1ea27b Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 10:33:42 +0200 Subject: [PATCH 28/30] v1.50.5-alpha.0 --- apps/appcontainer-node/app/package.json | 4 +- .../packages/generic/package.json | 6 +- apps/http-server/app/package.json | 4 +- .../http-server/packages/generic/package.json | 4 +- apps/package-manager/app/package.json | 4 +- .../packages/generic/package.json | 8 +- .../app/package.json | 4 +- .../packages/generic/package.json | 4 +- apps/single-app/app/package.json | 16 ++-- apps/worker/app/package.json | 4 +- apps/worker/packages/generic/package.json | 6 +- apps/workforce/app/package.json | 4 +- apps/workforce/packages/generic/package.json | 6 +- lerna.json | 2 +- shared/packages/api/package.json | 2 +- .../packages/expectationManager/package.json | 6 +- shared/packages/worker/package.json | 4 +- shared/packages/workforce/package.json | 4 +- tests/internal-tests/package.json | 14 +-- yarn.lock | 88 +++++++++---------- 20 files changed, 97 insertions(+), 97 deletions(-) diff --git a/apps/appcontainer-node/app/package.json b/apps/appcontainer-node/app/package.json index 2d502bc5..c3e1efee 100644 --- a/apps/appcontainer-node/app/package.json +++ b/apps/appcontainer-node/app/package.json @@ -1,6 +1,6 @@ { "name": "@appcontainer-node/app", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "description": "AppContainer-Node.js", "private": true, "scripts": { @@ -11,7 +11,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@appcontainer-node/generic": "1.50.2" + "@appcontainer-node/generic": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/appcontainer-node/packages/generic/package.json b/apps/appcontainer-node/packages/generic/package.json index a1750163..be399568 100644 --- a/apps/appcontainer-node/packages/generic/package.json +++ b/apps/appcontainer-node/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@appcontainer-node/generic", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -13,8 +13,8 @@ "@sofie-automation/shared-lib": "*" }, "dependencies": { - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/worker": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0", "underscore": "^1.12.0" }, "devDependencies": { diff --git a/apps/http-server/app/package.json b/apps/http-server/app/package.json index 49ccc7ea..18e17ea2 100644 --- a/apps/http-server/app/package.json +++ b/apps/http-server/app/package.json @@ -1,6 +1,6 @@ { "name": "@http-server/app", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "description": "Upload to and serve proxies of packages", "private": true, "scripts": { @@ -11,7 +11,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@http-server/generic": "1.50.2", + "@http-server/generic": "1.50.5-alpha.0", "rimraf": "^5.0.5" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index fd5ded6d..0c75bc03 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@http-server/generic", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -11,7 +11,7 @@ }, "dependencies": { "@koa/cors": "^5.0.0", - "@sofie-package-manager/api": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", "koa": "^2.14.1", "koa-bodyparser": "^4.3.0", "koa-range": "^0.3.0", diff --git a/apps/package-manager/app/package.json b/apps/package-manager/app/package.json index ed1e3b2d..3ce21a58 100644 --- a/apps/package-manager/app/package.json +++ b/apps/package-manager/app/package.json @@ -1,6 +1,6 @@ { "name": "@package-manager/app", - "version": "1.50.4", + "version": "1.50.5-alpha.0", "private": true, "scripts": { "build": "yarn rimraf dist && yarn build:main", @@ -10,7 +10,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@package-manager/generic": "1.50.4" + "@package-manager/generic": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/package-manager/packages/generic/package.json b/apps/package-manager/packages/generic/package.json index d29cfa9d..c5488b37 100644 --- a/apps/package-manager/packages/generic/package.json +++ b/apps/package-manager/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@package-manager/generic", - "version": "1.50.4", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -15,9 +15,9 @@ }, "dependencies": { "@parcel/watcher": "^2.3.0", - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/expectation-manager": "1.50.4", - "@sofie-package-manager/worker": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/expectation-manager": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0", "data-store": "^4.0.3", "deep-extend": "^0.6.0", "fast-clone": "^1.5.13", diff --git a/apps/quantel-http-transformer-proxy/app/package.json b/apps/quantel-http-transformer-proxy/app/package.json index 56c3826d..156418c5 100644 --- a/apps/quantel-http-transformer-proxy/app/package.json +++ b/apps/quantel-http-transformer-proxy/app/package.json @@ -1,6 +1,6 @@ { "name": "@quantel-http-transformer-proxy/app", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "description": "Proxy for a Quantel HTTP Transformer", "private": true, "scripts": { @@ -10,7 +10,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@quantel-http-transformer-proxy/generic": "1.50.2" + "@quantel-http-transformer-proxy/generic": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index 94da4e1b..3c62c6f5 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@quantel-http-transformer-proxy/generic", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -11,7 +11,7 @@ }, "dependencies": { "@koa/cors": "^5.0.0", - "@sofie-package-manager/api": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", "got": "^11.8.6", "koa": "^2.14.1", "koa-bodyparser": "^4.3.0", diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index eb64e468..a8cf7cc5 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -1,6 +1,6 @@ { "name": "@single-app/app", - "version": "1.50.4", + "version": "1.50.5-alpha.0", "description": "Package Manager, http-proxy etc.. all in one application", "private": true, "scripts": { @@ -11,13 +11,13 @@ "start": "node --inspect dist/index.js" }, "dependencies": { - "@appcontainer-node/generic": "1.50.2", - "@http-server/generic": "1.50.2", - "@package-manager/generic": "1.50.4", - "@quantel-http-transformer-proxy/generic": "1.50.2", - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/worker": "1.50.2", - "@sofie-package-manager/workforce": "1.50.2", + "@appcontainer-node/generic": "1.50.5-alpha.0", + "@http-server/generic": "1.50.5-alpha.0", + "@package-manager/generic": "1.50.5-alpha.0", + "@quantel-http-transformer-proxy/generic": "1.50.5-alpha.0", + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0", + "@sofie-package-manager/workforce": "1.50.5-alpha.0", "underscore": "^1.12.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", diff --git a/apps/worker/app/package.json b/apps/worker/app/package.json index d594944f..0af26852 100644 --- a/apps/worker/app/package.json +++ b/apps/worker/app/package.json @@ -1,6 +1,6 @@ { "name": "@worker/app", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "description": "Boilerplace", "private": true, "scripts": { @@ -12,7 +12,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@worker/generic": "1.50.2" + "@worker/generic": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/worker/packages/generic/package.json b/apps/worker/packages/generic/package.json index b66f7590..70c19534 100644 --- a/apps/worker/packages/generic/package.json +++ b/apps/worker/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@worker/generic", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -10,8 +10,8 @@ "__test": "jest" }, "dependencies": { - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/worker": "1.50.2" + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/workforce/app/package.json b/apps/workforce/app/package.json index ba3d37de..e601395a 100644 --- a/apps/workforce/app/package.json +++ b/apps/workforce/app/package.json @@ -1,6 +1,6 @@ { "name": "@workforce/app", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "description": "Boilerplace", "private": true, "scripts": { @@ -11,7 +11,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@workforce/generic": "1.50.2" + "@workforce/generic": "1.50.5-alpha.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { diff --git a/apps/workforce/packages/generic/package.json b/apps/workforce/packages/generic/package.json index bc43a6b5..d5a67d18 100644 --- a/apps/workforce/packages/generic/package.json +++ b/apps/workforce/packages/generic/package.json @@ -1,6 +1,6 @@ { "name": "@workforce/generic", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", @@ -10,8 +10,8 @@ "__test": "jest" }, "dependencies": { - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/workforce": "1.50.2" + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/workforce": "1.50.5-alpha.0" }, "devDependencies": { "rimraf": "^5.0.5" diff --git a/lerna.json b/lerna.json index 03ce0b58..ec649f94 100644 --- a/lerna.json +++ b/lerna.json @@ -4,6 +4,6 @@ "apps/**", "tests/**" ], - "version": "1.50.4", + "version": "1.50.5-alpha.0", "npmClient": "yarn" } diff --git a/shared/packages/api/package.json b/shared/packages/api/package.json index b08e29e3..baccd92e 100644 --- a/shared/packages/api/package.json +++ b/shared/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@sofie-package-manager/api", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "main": "dist/index", "types": "dist/index", "files": [ diff --git a/shared/packages/expectationManager/package.json b/shared/packages/expectationManager/package.json index 54299bac..07c8ada4 100644 --- a/shared/packages/expectationManager/package.json +++ b/shared/packages/expectationManager/package.json @@ -1,6 +1,6 @@ { "name": "@sofie-package-manager/expectation-manager", - "version": "1.50.4", + "version": "1.50.5-alpha.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", @@ -18,8 +18,8 @@ "type-fest": "3.13.1" }, "dependencies": { - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/worker": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0", "@supercharge/promise-pool": "^3.2.0", "underscore": "^1.12.0" }, diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json index 76bf8ba6..750bc19c 100644 --- a/shared/packages/worker/package.json +++ b/shared/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@sofie-package-manager/worker", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", @@ -26,7 +26,7 @@ }, "dependencies": { "@parcel/watcher": "^2.3.0", - "@sofie-package-manager/api": "1.50.2", + "@sofie-package-manager/api": "1.50.5-alpha.0", "abort-controller": "^3.0.0", "atem-connection": "^3.2.0", "deep-diff": "^1.0.2", diff --git a/shared/packages/workforce/package.json b/shared/packages/workforce/package.json index f768b664..00edf6e3 100644 --- a/shared/packages/workforce/package.json +++ b/shared/packages/workforce/package.json @@ -1,6 +1,6 @@ { "name": "@sofie-package-manager/workforce", - "version": "1.50.2", + "version": "1.50.5-alpha.0", "main": "dist/index", "types": "dist/index", "files": [ @@ -13,7 +13,7 @@ "__test": "jest" }, "dependencies": { - "@sofie-package-manager/api": "1.50.2" + "@sofie-package-manager/api": "1.50.5-alpha.0" }, "devDependencies": { "rimraf": "^5.0.5" diff --git a/tests/internal-tests/package.json b/tests/internal-tests/package.json index e8586a6a..60a36583 100644 --- a/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -1,6 +1,6 @@ { "name": "@tests/internal-tests", - "version": "1.50.4", + "version": "1.50.5-alpha.0", "description": "Internal tests", "private": true, "scripts": { @@ -14,12 +14,12 @@ "tv-automation-quantel-gateway-client": "^3.1.7" }, "dependencies": { - "@http-server/generic": "1.50.2", - "@package-manager/generic": "1.50.4", - "@sofie-package-manager/api": "1.50.2", - "@sofie-package-manager/expectation-manager": "1.50.4", - "@sofie-package-manager/worker": "1.50.2", - "@sofie-package-manager/workforce": "1.50.2", + "@http-server/generic": "1.50.5-alpha.0", + "@package-manager/generic": "1.50.5-alpha.0", + "@sofie-package-manager/api": "1.50.5-alpha.0", + "@sofie-package-manager/expectation-manager": "1.50.5-alpha.0", + "@sofie-package-manager/worker": "1.50.5-alpha.0", + "@sofie-package-manager/workforce": "1.50.5-alpha.0", "underscore": "^1.12.0", "windows-network-drive": "^4.0.1" }, diff --git a/yarn.lock b/yarn.lock index 22abcbf9..bf1dafda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -38,18 +38,18 @@ __metadata: version: 0.0.0-use.local resolution: "@appcontainer-node/app@workspace:apps/appcontainer-node/app" dependencies: - "@appcontainer-node/generic": "npm:1.50.2" + "@appcontainer-node/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@appcontainer-node/generic@npm:1.50.2, @appcontainer-node/generic@workspace:apps/appcontainer-node/packages/generic": +"@appcontainer-node/generic@npm:1.50.5-alpha.0, @appcontainer-node/generic@workspace:apps/appcontainer-node/packages/generic": version: 0.0.0-use.local resolution: "@appcontainer-node/generic@workspace:apps/appcontainer-node/packages/generic" dependencies: - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/worker": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" "@types/underscore": "npm:^1.10.24" rimraf: "npm:^5.0.5" underscore: "npm:^1.12.0" @@ -592,18 +592,18 @@ __metadata: version: 0.0.0-use.local resolution: "@http-server/app@workspace:apps/http-server/app" dependencies: - "@http-server/generic": "npm:1.50.2" + "@http-server/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@http-server/generic@npm:1.50.2, @http-server/generic@workspace:apps/http-server/packages/generic": +"@http-server/generic@npm:1.50.5-alpha.0, @http-server/generic@workspace:apps/http-server/packages/generic": version: 0.0.0-use.local resolution: "@http-server/generic@workspace:apps/http-server/packages/generic" dependencies: "@koa/cors": "npm:^5.0.0" - "@sofie-package-manager/api": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" "@types/busboy": "npm:^1.5.0" "@types/koa": "npm:^2.13.5" "@types/koa-bodyparser": "npm:^4.3.12" @@ -1635,20 +1635,20 @@ __metadata: version: 0.0.0-use.local resolution: "@package-manager/app@workspace:apps/package-manager/app" dependencies: - "@package-manager/generic": "npm:1.50.4" + "@package-manager/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@package-manager/generic@npm:1.50.4, @package-manager/generic@workspace:apps/package-manager/packages/generic": +"@package-manager/generic@npm:1.50.5-alpha.0, @package-manager/generic@workspace:apps/package-manager/packages/generic": version: 0.0.0-use.local resolution: "@package-manager/generic@workspace:apps/package-manager/packages/generic" dependencies: "@parcel/watcher": "npm:^2.3.0" - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/expectation-manager": "npm:1.50.4" - "@sofie-package-manager/worker": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/expectation-manager": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" "@types/deep-extend": "npm:0.4.31" "@types/underscore": "npm:^1.10.24" data-store: "npm:^4.0.3" @@ -1820,18 +1820,18 @@ __metadata: version: 0.0.0-use.local resolution: "@quantel-http-transformer-proxy/app@workspace:apps/quantel-http-transformer-proxy/app" dependencies: - "@quantel-http-transformer-proxy/generic": "npm:1.50.2" + "@quantel-http-transformer-proxy/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@quantel-http-transformer-proxy/generic@npm:1.50.2, @quantel-http-transformer-proxy/generic@workspace:apps/quantel-http-transformer-proxy/packages/generic": +"@quantel-http-transformer-proxy/generic@npm:1.50.5-alpha.0, @quantel-http-transformer-proxy/generic@workspace:apps/quantel-http-transformer-proxy/packages/generic": version: 0.0.0-use.local resolution: "@quantel-http-transformer-proxy/generic@workspace:apps/quantel-http-transformer-proxy/packages/generic" dependencies: "@koa/cors": "npm:^5.0.0" - "@sofie-package-manager/api": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" "@types/koa": "npm:^2.13.5" "@types/koa-bodyparser": "npm:^4.3.0" "@types/koa-range": "npm:^0.3.2" @@ -1884,13 +1884,13 @@ __metadata: version: 0.0.0-use.local resolution: "@single-app/app@workspace:apps/single-app/app" dependencies: - "@appcontainer-node/generic": "npm:1.50.2" - "@http-server/generic": "npm:1.50.2" - "@package-manager/generic": "npm:1.50.4" - "@quantel-http-transformer-proxy/generic": "npm:1.50.2" - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/worker": "npm:1.50.2" - "@sofie-package-manager/workforce": "npm:1.50.2" + "@appcontainer-node/generic": "npm:1.50.5-alpha.0" + "@http-server/generic": "npm:1.50.5-alpha.0" + "@package-manager/generic": "npm:1.50.5-alpha.0" + "@quantel-http-transformer-proxy/generic": "npm:1.50.5-alpha.0" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" + "@sofie-package-manager/workforce": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" underscore: "npm:^1.12.0" @@ -1982,7 +1982,7 @@ __metadata: languageName: node linkType: hard -"@sofie-package-manager/api@npm:1.50.2, @sofie-package-manager/api@workspace:shared/packages/api": +"@sofie-package-manager/api@npm:1.50.5-alpha.0, @sofie-package-manager/api@workspace:shared/packages/api": version: 0.0.0-use.local resolution: "@sofie-package-manager/api@workspace:shared/packages/api" dependencies: @@ -1999,12 +1999,12 @@ __metadata: languageName: unknown linkType: soft -"@sofie-package-manager/expectation-manager@npm:1.50.4, @sofie-package-manager/expectation-manager@workspace:shared/packages/expectationManager": +"@sofie-package-manager/expectation-manager@npm:1.50.5-alpha.0, @sofie-package-manager/expectation-manager@workspace:shared/packages/expectationManager": version: 0.0.0-use.local resolution: "@sofie-package-manager/expectation-manager@workspace:shared/packages/expectationManager" dependencies: - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/worker": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" "@supercharge/promise-pool": "npm:^3.2.0" jest: "npm:*" rimraf: "npm:^5.0.5" @@ -2013,12 +2013,12 @@ __metadata: languageName: unknown linkType: soft -"@sofie-package-manager/worker@npm:1.50.2, @sofie-package-manager/worker@workspace:shared/packages/worker": +"@sofie-package-manager/worker@npm:1.50.5-alpha.0, @sofie-package-manager/worker@workspace:shared/packages/worker": version: 0.0.0-use.local resolution: "@sofie-package-manager/worker@workspace:shared/packages/worker" dependencies: "@parcel/watcher": "npm:^2.3.0" - "@sofie-package-manager/api": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" "@types/deep-diff": "npm:^1.0.0" "@types/node-fetch": "npm:^2.5.8" "@types/proper-lockfile": "npm:^4.1.4" @@ -2042,11 +2042,11 @@ __metadata: languageName: unknown linkType: soft -"@sofie-package-manager/workforce@npm:1.50.2, @sofie-package-manager/workforce@workspace:shared/packages/workforce": +"@sofie-package-manager/workforce@npm:1.50.5-alpha.0, @sofie-package-manager/workforce@workspace:shared/packages/workforce": version: 0.0.0-use.local resolution: "@sofie-package-manager/workforce@workspace:shared/packages/workforce" dependencies: - "@sofie-package-manager/api": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft @@ -2071,12 +2071,12 @@ __metadata: version: 0.0.0-use.local resolution: "@tests/internal-tests@workspace:tests/internal-tests" dependencies: - "@http-server/generic": "npm:1.50.2" - "@package-manager/generic": "npm:1.50.4" - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/expectation-manager": "npm:1.50.4" - "@sofie-package-manager/worker": "npm:1.50.2" - "@sofie-package-manager/workforce": "npm:1.50.2" + "@http-server/generic": "npm:1.50.5-alpha.0" + "@package-manager/generic": "npm:1.50.5-alpha.0" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/expectation-manager": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" + "@sofie-package-manager/workforce": "npm:1.50.5-alpha.0" deep-extend: "npm:^0.6.0" jest: "npm:*" tv-automation-quantel-gateway-client: "npm:^3.1.7" @@ -2784,18 +2784,18 @@ __metadata: version: 0.0.0-use.local resolution: "@worker/app@workspace:apps/worker/app" dependencies: - "@worker/generic": "npm:1.50.2" + "@worker/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@worker/generic@npm:1.50.2, @worker/generic@workspace:apps/worker/packages/generic": +"@worker/generic@npm:1.50.5-alpha.0, @worker/generic@workspace:apps/worker/packages/generic": version: 0.0.0-use.local resolution: "@worker/generic@workspace:apps/worker/packages/generic" dependencies: - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/worker": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/worker": "npm:1.50.5-alpha.0" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft @@ -2804,18 +2804,18 @@ __metadata: version: 0.0.0-use.local resolution: "@workforce/app@workspace:apps/workforce/app" dependencies: - "@workforce/generic": "npm:1.50.2" + "@workforce/generic": "npm:1.50.5-alpha.0" lerna: "npm:^6.6.1" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft -"@workforce/generic@npm:1.50.2, @workforce/generic@workspace:apps/workforce/packages/generic": +"@workforce/generic@npm:1.50.5-alpha.0, @workforce/generic@workspace:apps/workforce/packages/generic": version: 0.0.0-use.local resolution: "@workforce/generic@workspace:apps/workforce/packages/generic" dependencies: - "@sofie-package-manager/api": "npm:1.50.2" - "@sofie-package-manager/workforce": "npm:1.50.2" + "@sofie-package-manager/api": "npm:1.50.5-alpha.0" + "@sofie-package-manager/workforce": "npm:1.50.5-alpha.0" rimraf: "npm:^5.0.5" languageName: unknown linkType: soft From 474ed6bf32a12678b8d3ea9012ad97d9d6e625bd Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 14:26:22 +0200 Subject: [PATCH 29/30] fix: issue with missing CLI argument (when spinning up workers from AppContainer) --- .../packages/generic/src/appContainer.ts | 11 +++++++---- shared/packages/api/src/config.ts | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index bcf9da3e..752ed44f 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -307,8 +307,11 @@ export class AppContainer { const appType = protectString('worker') this.availableApps.set(appType, { file: process.execPath, - getExecArgs: (appId: AppId) => { - return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId, false)] + getExecArgs: (appId: AppId, useCriticalOnlyMode: boolean) => { + return [ + path.resolve('.', '../../worker/app/dist/index.js'), + ...getWorkerArgs(appId, useCriticalOnlyMode), + ] }, canRunInCriticalExpectationsOnlyMode: true, cost: 0, @@ -326,8 +329,8 @@ export class AppContainer { const appType: AppType = protectString(fileName) this.availableApps.set(appType, { file: path.join(dirPath, fileName), - getExecArgs: (appId: AppId) => { - return [...getWorkerArgs(appId, false)] + getExecArgs: (appId: AppId, useCriticalOnlyMode: boolean) => { + return [...getWorkerArgs(appId, useCriticalOnlyMode)] }, canRunInCriticalExpectationsOnlyMode: true, cost: 0, diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index a88007fa..8652dbf0 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -207,6 +207,11 @@ const appContainerArguments = defineArguments({ default: parseInt(process.env.APP_CONTAINER_SPIN_DOWN_TIME || '', 10) || 60 * 1000, // ms (1 minute) describe: 'How long a Worker should stay idle before attempting to be spun down', }, + minCriticalWorkerApps: { + type: 'number', + default: 0, + describe: 'Number of Workers reserved for fulfilling playout-critical expectations that will be kept running', + }, // These are passed-through to the spun-up workers: resourceId: { @@ -235,11 +240,6 @@ const appContainerArguments = defineArguments({ describe: 'If set, the worker will consider the CPU load of the system it runs on before it accepts jobs. Set to a value between 0 and 1, the worker will accept jobs if the CPU load is below the configured value.', }, - minCriticalWorkerApps: { - type: 'number', - default: 0, - describe: 'Number of Workers reserved for fulfilling playout-critical expectations that will be kept running', - }, }) /** CLI-argument-definitions for the "Single" process */ const singleAppArguments = defineArguments({ From 1a9874b09ff6ed576e47b49daea2206052d252d1 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 5 Apr 2024 14:27:07 +0200 Subject: [PATCH 30/30] fix: ensure that expectations can be picked up by newly spun up workers --- .../src/internalManager/lib/trackedWorkerAgents.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts index 3d5cf0c7..73f64afb 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/trackedWorkerAgents.ts @@ -202,6 +202,12 @@ export class TrackedWorkerAgents { } } + if (trackedExp.waitingForWorkerTime !== null) { + // If the expectation is waiting for a worker, it might be a good idea to update the list of available workers: + // (This can be useful for example if a new worker has just been registered) + await this.updateAvailableWorkersForExpectation(trackedExp) + } + if (!trackedExp.availableWorkers.size) { session.noAssignedWorkerReason = { user: `No workers available`, tech: `No workers available` } }