diff --git a/.eslintrc.js b/.eslintrc.js index 28c1e159..82ab5170 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -8,4 +8,7 @@ module.exports = { "**/dist/**/*", "**/__tests__/**/*", ], + "rules": { + "no-console": "warn" + } }; diff --git a/FOR_DEVELOPERS.md b/FOR_DEVELOPERS.md index 670a9a75..85fc9154 100644 --- a/FOR_DEVELOPERS.md +++ b/FOR_DEVELOPERS.md @@ -1,23 +1,20 @@ - This document contains documentation intended for developers of this repo. - # Key concepts -![System overview](./images/System-overview.png "System overview") +![System overview](./images/System-overview.png 'System overview') ## Workforce -*Note: There can be only one (1) Workforce in a setup.* +_Note: There can be only one (1) Workforce in a setup._ The Workforce keeps track of which `ExpectationManagers` and `Workers` are online, and mediates the contact between the two. _Future functionality: The Workforce is responsible for tracking the total workload and spin up/down workers accordingly._ - ## Package Manager -*Note: There can be multiple Package Managers in a setup* +_Note: There can be multiple Package Managers in a setup_ The Package Manager receives [Packages](#packages) from [Sofie Core](https://github.com/nrkno/tv-automation-server-core) and generates [Expectations](#expectations) from them. @@ -35,22 +32,20 @@ A typical lifetime of an Expectation is: 4. `WORKING`: Intermediary state while the Worker is working. 5. `FULFILLED`: From time-to-time, the Expectation is re-checked if it still is `FULFILLED` - ## Worker -*Note: There can be multiple Workers in a setup* +_Note: There can be multiple Workers in a setup_ The Worker is the process which actually does the work. It exposes an API with methods for the `ExpectationManager` to call in order to check status of Packages and perform the work. The Worker is (almost completely) **stateless**, which allows it to expose a lambda-like API. This allows for there being a pool of Workers where the workload can be easilly shared between the Workers. - _Future functionality: There are multiple different types of Workers. Some are running on a Windows-machine with direct access to that maching. Some are running in Linux/Docker containers, or even off-premise._ ### ExpectationHandlers & AccessorHandlers -![Expectation and Accessor handlers](./images/handlers.png "Expectation and Accessor handlers") +![Expectation and Accessor handlers](./images/handlers.png 'Expectation and Accessor handlers') Internally, the Worker is structured by separating the `ExpectationHandlers` and the `AccessorHandlers`. @@ -59,7 +54,6 @@ The `ExpectationHandlers` handles the high-level functionality required for a ce For example, when [copying a file](shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts) from one folder to another, the `ExpectationHandler` will handle things like "check if the source package exists", "check if the target package already exists" but it's never touching the files directly, only talking to the the `AccessorHandler`. The `AccessorHandler` [exposes a few generic methods](./shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts), like "check if we can read from a Package" (ie does a file exist), etc. - ## HTTP-server The HTTP-server application is a simple HTTP-server which allows for uploading and downloading of Packages with a RESTful API. @@ -94,3 +88,66 @@ A PackageContainer is separated from an **Accessor**, which is the "way to acces _See [expectationApi.ts](shared/packages/api/src/expectationApi.ts)._ An Expectation is what the PackageManager uses to figure out what should be done and how to do it. One example is "Copy a file from a source into a target". + +# Examples: + +Below are some examples of the data: + +## Example A, a file is to be copied + +```javascript +// The input to Package Manager (from Sofie-Core): + +const packageContainers = { + source0: { + label: 'Source 0', + accessors: { + local: { + type: 'local_folder', + folderPath: 'D:\\media\\source0', + allowRead: true, + }, + }, + }, + target0: { + label: 'Target 0', + accessors: { + local: { + type: 'local_folder', + folderPath: 'D:\\media\\target0', + allowRead: true, + allowWrite: true, + }, + }, + }, +} +const expectedPackages = [ + { + type: 'media_file', + _id: 'test', + contentVersionHash: 'abc1234', + content: { + filePath: 'myFocalFolder\\amb.mp4', + }, + version: {}, + sources: [ + { + containerId: 'source0', + accessors: { + local: { + type: 'local_folder', + filePath: 'amb.mp4', + }, + }, + }, + ], + layers: ['target0'], + sideEffect: { + previewContainerId: null, + previewPackageSettings: null, + thumbnailContainerId: null, + thumbnailPackageSettings: null, + }, + }, +] +``` diff --git a/README.md b/README.md index ed896df6..89df24ff 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,26 @@ The packages in [shared/packages](shared/packages) are helper libraries, used by The packages in [apps/](apps/) can be run as individual applications. +The packages in [tests/](tests/) contain unit/integration tests. + ### Applications -| Name | Location | Description | -| ----- | -------- | ----------- | -| **Workforce** | [apps/workforce/app](apps/workforce/app) | Mediates connections between the Workers and the Package Managers. _(Later: Will handle spin-up/down of workers according to the current need.)_ | +| Name | Location | Description | +| ------------------- | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Workforce** | [apps/workforce/app](apps/workforce/app) | Mediates connections between the Workers and the Package Managers. _(Later: Will handle spin-up/down of workers according to the current need.)_ | | **Package Manager** | [apps/package-manager/app](apps/package-manager/app) | The Package Manager receives `expectedPackages` from a [Sofie Core](https://github.com/nrkno/tv-automation-server-core), converts them into `Expectations`. Keeps track of work statues and distributes the work to the Workers. | -| **Worker** | [apps/worker/app](apps/worker/app) | Executes work orders from the Package Manager | -| **HTTP-server** | [apps/http-server/app](apps/http-server/app) | A simple HTTP server, where files can be uploaded to and served from. (Often used for thumbnails & previews) | -| **Single-app** | [apps/single-app/app](apps/single-app/app) | Runs one of each of the above in a single application. | +| **Worker** | [apps/worker/app](apps/worker/app) | Executes work orders from the Package Manager | +| **HTTP-server** | [apps/http-server/app](apps/http-server/app) | A simple HTTP server, where files can be uploaded to and served from. (Often used for thumbnails & previews) | +| **Single-app** | [apps/single-app/app](apps/single-app/app) | Runs one of each of the above in a single application. | ### Packages (Libraries) -| Name | Location | Description | -| -- | -- | -- | -| **API** | [shared/packages/api](shared/packages/api) | Various interfaces used by the other libraries | +| Name | Location | Description | +| ---------------------- | ------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| **API** | [shared/packages/api](shared/packages/api) | Various interfaces used by the other libraries | | **ExpectationManager** | [shared/packages/expectationManager](shared/packages/expectationManager) | The ExpectationManager class is used by the Package Manager application | -| **Worker** | [shared/packages/worker](shared/packages/worker) | The Worker class is used by the Worker application | -| **Workforce** | [shared/packages/Workforce](shared/packages/Workforce) | The Workforce class is used by the Worker application | +| **Worker** | [shared/packages/worker](shared/packages/worker) | The Worker class is used by the Worker application | +| **Workforce** | [shared/packages/Workforce](shared/packages/Workforce) | The Workforce class is used by the Worker application | ## For Developers diff --git a/apps/_boilerplate/app/package.json b/apps/_boilerplate/app/package.json index 2d30537a..9a5ed7ce 100644 --- a/apps/_boilerplate/app/package.json +++ b/apps/_boilerplate/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/boilerplate.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/boilerplate.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js boilerplate.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/_boilerplate/app/scripts/copy-natives.js b/apps/_boilerplate/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/_boilerplate/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/_boilerplate/app/src/index.ts b/apps/_boilerplate/app/src/index.ts index 18a80d0a..dffb2283 100644 --- a/apps/_boilerplate/app/src/index.ts +++ b/apps/_boilerplate/app/src/index.ts @@ -1,3 +1,4 @@ import { startProcess } from '@boilerplate/generic' +/* eslint-disable no-console */ startProcess().catch(console.error) diff --git a/apps/_boilerplate/packages/generic/package.json b/apps/_boilerplate/packages/generic/package.json index 57a24903..6bc658c2 100644 --- a/apps/_boilerplate/packages/generic/package.json +++ b/apps/_boilerplate/packages/generic/package.json @@ -19,7 +19,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -29,4 +29,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/_boilerplate/packages/generic/src/index.ts b/apps/_boilerplate/packages/generic/src/index.ts index 7b7f3be2..9375a134 100644 --- a/apps/_boilerplate/packages/generic/src/index.ts +++ b/apps/_boilerplate/packages/generic/src/index.ts @@ -1,4 +1,5 @@ // boilerplate export async function startProcess(): Promise { + // eslint-disable-next-line no-console console.log('hello world!') } diff --git a/apps/appcontainer-node/app/.gitignore b/apps/appcontainer-node/app/.gitignore new file mode 100644 index 00000000..d11e3a5a --- /dev/null +++ b/apps/appcontainer-node/app/.gitignore @@ -0,0 +1,2 @@ +log.log +deploy/* diff --git a/apps/tests/internal-tests/jest.config.js b/apps/appcontainer-node/app/jest.config.js similarity index 100% rename from apps/tests/internal-tests/jest.config.js rename to apps/appcontainer-node/app/jest.config.js diff --git a/apps/appcontainer-node/app/package.json b/apps/appcontainer-node/app/package.json new file mode 100644 index 00000000..652e517d --- /dev/null +++ b/apps/appcontainer-node/app/package.json @@ -0,0 +1,36 @@ +{ + "name": "@appcontainer-node/app", + "version": "1.0.0", + "description": "AppContainer-Node.js", + "private": true, + "scripts": { + "build": "rimraf dist && yarn build:main", + "build:main": "tsc -p tsconfig.json", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js appContainer-node.exe && node ../../../scripts/copy-natives.js win32-x64", + "__test": "jest", + "start": "node dist/index.js", + "precommit": "lint-staged" + }, + "devDependencies": { + "nexe": "^3.3.7", + "lint-staged": "^7.2.0" + }, + "dependencies": { + "@appcontainer-node/generic": "*" + }, + "peerDependencies": { + "@sofie-automation/blueprints-integration": "*" + }, + "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", + "engines": { + "node": ">=12.20.0" + }, + "lint-staged": { + "*.{js,css,json,md,scss}": [ + "prettier" + ], + "*.{ts,tsx}": [ + "eslint" + ] + } +} diff --git a/apps/appcontainer-node/app/src/__tests__/test.spec.ts b/apps/appcontainer-node/app/src/__tests__/test.spec.ts new file mode 100644 index 00000000..b65fc1f7 --- /dev/null +++ b/apps/appcontainer-node/app/src/__tests__/test.spec.ts @@ -0,0 +1,7 @@ +describe('tmp', () => { + test('tmp', () => { + // Note: To enable tests in this package, ensure that the "test" script is present in package.json + expect(1).toEqual(1) + }) +}) +export {} diff --git a/apps/appcontainer-node/app/src/index.ts b/apps/appcontainer-node/app/src/index.ts new file mode 100644 index 00000000..fc2a51a1 --- /dev/null +++ b/apps/appcontainer-node/app/src/index.ts @@ -0,0 +1,5 @@ +import { startProcess } from '@appcontainer-node/generic' +/* eslint-disable no-console */ + +console.log('process started') // This is a message all Sofie processes log upon startup +startProcess().catch(console.error) diff --git a/apps/tests/internal-tests/tsconfig.json b/apps/appcontainer-node/app/tsconfig.json similarity index 100% rename from apps/tests/internal-tests/tsconfig.json rename to apps/appcontainer-node/app/tsconfig.json diff --git a/apps/appcontainer-node/packages/generic/jest.config.js b/apps/appcontainer-node/packages/generic/jest.config.js new file mode 100644 index 00000000..269ab60e --- /dev/null +++ b/apps/appcontainer-node/packages/generic/jest.config.js @@ -0,0 +1,8 @@ +const base = require('../../../../jest.config.base') +const packageJson = require('./package') + +module.exports = { + ...base, + name: packageJson.name, + displayName: packageJson.name, +} diff --git a/apps/appcontainer-node/packages/generic/package.json b/apps/appcontainer-node/packages/generic/package.json new file mode 100644 index 00000000..2e3986b7 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/package.json @@ -0,0 +1,35 @@ +{ + "name": "@appcontainer-node/generic", + "version": "1.0.0", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "rimraf dist && yarn build:main", + "build:main": "tsc -p tsconfig.json", + "__test": "jest", + "precommit": "lint-staged" + }, + "peerDependencies": { + "@sofie-automation/blueprints-integration": "*" + }, + "dependencies": { + "@shared/api": "*", + "@shared/worker": "*" + }, + "devDependencies": { + "lint-staged": "^7.2.0" + }, + "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", + "engines": { + "node": ">=12.20.0" + }, + "lint-staged": { + "*.{js,css,json,md,scss}": [ + "prettier" + ], + "*.{ts,tsx}": [ + "eslint" + ] + } +} \ No newline at end of file diff --git a/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts b/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts new file mode 100644 index 00000000..e62eae89 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/__tests__/temp.spec.ts @@ -0,0 +1,6 @@ +describe('Temp', () => { + // This is just a placeholder, to be replaced with real tests later on + test('basic math', () => { + expect(1 + 1).toEqual(2) + }) +}) diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts new file mode 100644 index 00000000..0eb5a08e --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -0,0 +1,385 @@ +import * as ChildProcess from 'child_process' +import * as path from 'path' +import * as fs from 'fs' +import { + LoggerInstance, + AppContainerProcessConfig, + ClientConnectionOptions, + LogLevel, + WebsocketServer, + ClientConnection, + AppContainerWorkerAgent, + assertNever, + Expectation, + waitTime, + APPCONTAINER_PING_TIME, +} from '@shared/api' +import { WorkforceAPI } from './workforceApi' +import { WorkerAgentAPI } from './workerAgentApi' + +/** Mimimum time between app restarts */ +const RESTART_COOLDOWN = 60 * 1000 // ms + +export class AppContainer { + private workforceAPI: WorkforceAPI + private id: string + private workForceConnectionOptions: ClientConnectionOptions + private appId = 0 + + private apps: { + [appId: string]: { + process: ChildProcess.ChildProcess + appType: string + toBeKilled: boolean + restarts: number + lastRestart: number + spinDownTime: number + workerAgentApi?: WorkerAgentAPI + monitorPing: boolean + lastPing: number + } + } = {} + private availableApps: { + [appType: string]: AvailableAppInfo + } = {} + private websocketServer?: WebsocketServer + + private monitorAppsTimer: NodeJS.Timer | undefined + + constructor(private logger: LoggerInstance, private config: AppContainerProcessConfig) { + if (config.appContainer.port !== null) { + this.websocketServer = new WebsocketServer(config.appContainer.port, (client: ClientConnection) => { + // A new client has connected + + this.logger.debug(`AppContainer: New client "${client.clientType}" connected, id "${client.clientId}"`) + + switch (client.clientType) { + case 'workerAgent': { + const workForceMethods = this.getWorkerAgentAPI(client.clientId) + const api = new WorkerAgentAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + const app = this.apps[client.clientId] + if (!app) { + throw new Error(`Unknown app "${client.clientId}" just connected to the appContainer`) + } + app.workerAgentApi = api + client.on('close', () => { + delete app.workerAgentApi + }) + // Set upp the app for pinging and automatic spin-down: + app.monitorPing = true + app.lastPing = Date.now() + api.setSpinDownTime(app.spinDownTime) + break + } + case 'expectationManager': + case 'appContainer': + case 'N/A': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: + assertNever(client.clientType) + throw new Error(`Workforce: Unknown clientType "${client.clientType}"`) + } + }) + } + + this.workforceAPI = new WorkforceAPI(this.logger) + + this.id = config.appContainer.appContainerId + this.workForceConnectionOptions = this.config.appContainer.workforceURL + ? { + type: 'websocket', + url: this.config.appContainer.workforceURL, + } + : { + type: 'internal', + } + } + async init(): Promise { + if (this.workForceConnectionOptions.type === 'websocket') { + this.logger.info(`AppContainer: Connecting to Workforce at "${this.workForceConnectionOptions.url}"`) + } + + await this.setupAvailableApps() + + await this.workforceAPI.init(this.id, this.workForceConnectionOptions, this) + + await this.workforceAPI.registerAvailableApps( + Object.entries(this.availableApps).map((o) => { + const appType = o[0] as string + return { + appType: appType, + } + }) + ) + this.monitorAppsTimer = setInterval(() => { + this.monitorApps() + }, APPCONTAINER_PING_TIME) + this.monitorApps() // Also run right away + } + /** Return the API-methods that the AppContainer exposes to the WorkerAgent */ + private getWorkerAgentAPI(clientId: string): AppContainerWorkerAgent.AppContainer { + return { + ping: async (): Promise => { + this.apps[clientId].lastPing = Date.now() + }, + requestSpinDown: async (): Promise => { + const app = this.apps[clientId] + if (app) { + if (this.getAppCount(app.appType) > this.config.appContainer.minRunningApps) { + this.spinDown(clientId).catch((error) => { + this.logger.error(`AppContainer: Error when spinning down app "${clientId}"`) + this.logger.error(error) + }) + } + } + }, + } + } + private getAppCount(appType: string): number { + let count = 0 + for (const app of Object.values(this.apps)) { + if (app.appType === appType) count++ + } + return count + } + private async setupAvailableApps() { + const getWorkerArgs = (appId: string): string[] => { + return [ + `--workerId=${appId}`, + `--workforceURL=${this.config.appContainer.workforceURL}`, + `--appContainerURL=${'ws://127.0.0.1:' + this.websocketServer?.port}`, + this.config.appContainer.windowsDriveLetters + ? `--windowsDriveLetters=${this.config.appContainer.windowsDriveLetters?.join(';')}` + : '', + this.config.appContainer.resourceId ? `--resourceId=${this.config.appContainer.resourceId}` : '', + this.config.appContainer.networkIds.length + ? `--networkIds=${this.config.appContainer.networkIds.join(';')}` + : '', + ] + } + if (process.execPath.match(/node.exe$/)) { + // Process runs as a node process, we're probably in development mode. + this.availableApps['worker'] = { + file: process.execPath, + args: (appId: string) => { + return [path.resolve('.', '../../worker/app/dist/index.js'), ...getWorkerArgs(appId)] + }, + cost: 0, + } + } else { + // Process is a compiled executable + // Look for the worker executable in the same folder: + + const dirPath = path.dirname(process.execPath) + // Note: nexe causes issues with its virtual file system: https://github.com/nexe/nexe/issues/613#issuecomment-579107593 + + ;(await fs.promises.readdir(dirPath)).forEach((fileName) => { + if (fileName.match(/worker/i)) { + this.availableApps[fileName] = { + file: path.join(dirPath, fileName), + args: (appId: string) => { + return [...getWorkerArgs(appId)] + }, + cost: 0, + } + } + }) + } + this.logger.info(`AppContainer: Available apps`) + for (const [appType, availableApp] of Object.entries(this.availableApps)) { + this.logger.info(`${appType}: ${availableApp.file}`) + } + } + terminate(): void { + this.workforceAPI.terminate() + this.websocketServer?.terminate() + + if (this.monitorAppsTimer) { + clearInterval(this.monitorAppsTimer) + delete this.monitorAppsTimer + } + + // kill child processes + } + async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } + async _debugKill(): Promise { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } + + async requestAppTypeForExpectation(exp: Expectation.Any): Promise<{ appType: string; cost: number } | null> { + this.logger.debug(`AppContainer: Got request for resources, for exp "${exp.id}"`) + if (Object.keys(this.apps).length >= this.config.appContainer.maxRunningApps) { + this.logger.debug(`AppContainer: Is already at our limit, no more resources available`) + // If we're at our limit, we can't possibly run anything else + return null + } + + for (const [appType, availableApp] of Object.entries(this.availableApps)) { + // Do we already have any instance of the appType running? + let runningApp = Object.values(this.apps).find((app) => { + return app.appType === appType + }) + + if (!runningApp) { + const newAppId = await this.spinUp(appType, true) // todo: make it not die too soon + + // wait for the app to connect to us: + tryAfewTimes(async () => { + if (this.apps[newAppId].workerAgentApi) { + return true + } + await waitTime(200) + return false + }, 10) + runningApp = this.apps[newAppId] + if (!runningApp) throw new Error(`AppContainer: Worker "${newAppId}" didn't connect in time`) + } + if (runningApp?.workerAgentApi) { + const result = await runningApp.workerAgentApi.doYouSupportExpectation(exp) + if (result.support) { + return { + appType: appType, + cost: availableApp.cost, + } + } + } else { + this.logger.debug(`AppContainer: appType "${appType}" not available`) + } + } + return null + } + async spinUp(appType: string, longSpinDownTime = false): Promise { + const availableApp = this.availableApps[appType] + if (!availableApp) throw new Error(`Unknown appType "${appType}"`) + + const appId = `${this.id}_${this.appId++}` + + const child = this.setupChildProcess(appType, appId, availableApp) + this.apps[appId] = { + process: child, + appType: appType, + toBeKilled: false, + restarts: 0, + lastRestart: 0, + monitorPing: false, + lastPing: Date.now(), + spinDownTime: this.config.appContainer.spinDownTime * (longSpinDownTime ? 10 : 1), + } + return appId + } + async spinDown(appId: string): Promise { + const app = this.apps[appId] + if (!app) throw new Error(`App "${appId}" not found`) + + this.logger.debug(`AppContainer: Spinning down app "${appId}"`) + + app.toBeKilled = true + const success = app.process.kill() + if (!success) throw new Error(`Internal error: Killing of process "${app.process.pid}" failed`) + + app.process.removeAllListeners() + delete this.apps[appId] + } + async getRunningApps(): Promise<{ appId: string; appType: string }[]> { + return Object.entries(this.apps).map((o) => { + const [appId, app] = o + + return { + appId: appId, + appType: app.appType, + } + }) + } + private setupChildProcess( + appType: string, + appId: string, + availableApp: AvailableAppInfo + ): ChildProcess.ChildProcess { + this.logger.debug(`Starting process "${appId}" (${appType}): "${availableApp.file}"`) + const cwd = process.execPath.match(/node.exe$/) + ? undefined // Process runs as a node process, we're probably in development mode. + : path.dirname(process.execPath) // Process runs as a node process, we're probably in development mode. + + const child = ChildProcess.execFile(availableApp.file, availableApp.args(appId), { + cwd: cwd, + }) + + child.stdout?.on('data', (data) => { + this.logger.debug(`${appId} stdout: ${data}`) + }) + child.stderr?.on('data', (data) => { + this.logger.debug(`${appId} stderr: ${data}`) + }) + child.once('close', (code) => { + const app = this.apps[appId] + if (app && !app.toBeKilled) { + // Try to restart the application + + const timeUntilRestart = Math.max(0, app.lastRestart - Date.now() + RESTART_COOLDOWN) + this.logger.warn( + `App ${app.process.pid} (${appType}) closed with code (${code}), trying to restart in ${timeUntilRestart} ms (restarts: ${app.restarts})` + ) + + setTimeout(() => { + app.lastRestart = Date.now() + app.restarts++ + + app.process.removeAllListeners() + + const newChild = this.setupChildProcess(appType, appId, availableApp) + + app.process = newChild + }, timeUntilRestart) + } + }) + + return child + } + private monitorApps() { + for (const [appId, app] of Object.entries(this.apps)) { + if (app.monitorPing) { + if (Date.now() - app.lastPing > APPCONTAINER_PING_TIME * 2.5) { + // The app seems to have crashed. + this.spinDown(appId).catch((error) => { + this.logger.error(`AppContainer: Error when spinning down app "${appId}"`) + this.logger.error(error) + }) + } + } + } + this.spinUpMinimumApps().catch((error) => { + this.logger.error(`AppContainer: Error in spinUpMinimumApps`) + this.logger.error(error) + }) + } + private async spinUpMinimumApps(): Promise { + for (const appType of Object.keys(this.availableApps)) { + while (this.getAppCount(appType) < this.config.appContainer.minRunningApps) { + await this.spinUp(appType) + } + } + } +} +interface AvailableAppInfo { + file: string + args: (appId: string) => string[] + /** Some kind of value, how much it costs to run it, per minute */ + cost: number +} + +async function tryAfewTimes(cb: () => Promise, maxTries: number) { + for (let i = 0; i < maxTries; i++) { + if (await cb()) { + break + } + } +} diff --git a/apps/appcontainer-node/packages/generic/src/index.ts b/apps/appcontainer-node/packages/generic/src/index.ts new file mode 100644 index 00000000..dc4c39fe --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/index.ts @@ -0,0 +1,21 @@ +import { getAppContainerConfig, ProcessHandler, setupLogging } from '@shared/api' +import { AppContainer } from './appContainer' + +export { AppContainer } from './appContainer' + +export async function startProcess(): Promise { + const config = getAppContainerConfig() + + const logger = setupLogging(config) + + logger.info('------------------------------------------------------------------') + logger.info('Starting AppContainer') + logger.info('------------------------------------------------------------------') + + const process = new ProcessHandler(logger) + process.init(config.process) + + const appContainer = new AppContainer(logger, config) + + appContainer.init().catch(logger.error) +} diff --git a/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts new file mode 100644 index 00000000..03b04632 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/workerAgentApi.ts @@ -0,0 +1,37 @@ +import { + AppContainerWorkerAgent, + AdapterServer, + AdapterServerOptions, + LogLevel, + Expectation, + ReturnTypeDoYouSupportExpectation, +} from '@shared/api' + +/** + * Exposes the API-methods of a WorkerAgent, to be called from the AppContainer + * Note: The WorkerAgent connects to the AppContainer, therefore the AppContainer is the AdapterServer here. + * The corresponding other side is implemented at shared/packages/worker/src/appContainerApi.ts + */ +export class WorkerAgentAPI + extends AdapterServer + implements AppContainerWorkerAgent.WorkerAgent { + constructor( + methods: AppContainerWorkerAgent.AppContainer, + options: AdapterServerOptions + ) { + super(methods, options) + } + + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + async doYouSupportExpectation(exp: Expectation.Any): Promise { + return this._sendMessage('doYouSupportExpectation', exp) + } + async setSpinDownTime(spinDownTime: number): Promise { + return this._sendMessage('setSpinDownTime', spinDownTime) + } +} diff --git a/apps/appcontainer-node/packages/generic/src/workforceApi.ts b/apps/appcontainer-node/packages/generic/src/workforceApi.ts new file mode 100644 index 00000000..6a810595 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/src/workforceApi.ts @@ -0,0 +1,17 @@ +import { AdapterClient, LoggerInstance, WorkForceAppContainer } from '@shared/api' + +/** + * Exposes the API-methods of a Workforce, to be called from the AppContainer + * Note: The AppContainer connects to the Workforce, therefore the AppContainer is the AdapterClient here. + * The corresponding other side is implemented at shared/packages/workforce/src/appContainerApi.ts + */ +export class WorkforceAPI + extends AdapterClient + implements WorkForceAppContainer.WorkForce { + constructor(logger: LoggerInstance) { + super(logger, 'appContainer') + } + async registerAvailableApps(availableApps: { appType: string }[]): Promise { + return this._sendMessage('registerAvailableApps', availableApps) + } +} diff --git a/apps/appcontainer-node/packages/generic/tsconfig.json b/apps/appcontainer-node/packages/generic/tsconfig.json new file mode 100644 index 00000000..b07e3e02 --- /dev/null +++ b/apps/appcontainer-node/packages/generic/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + } +} diff --git a/apps/http-server/app/package.json b/apps/http-server/app/package.json index 0ce9aa87..eac1c1b4 100644 --- a/apps/http-server/app/package.json +++ b/apps/http-server/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/http-server.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/http-server.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js http-server.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/http-server/app/scripts/copy-natives.js b/apps/http-server/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/http-server/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/http-server/app/src/index.ts b/apps/http-server/app/src/index.ts index 35f91ae3..34ec591e 100644 --- a/apps/http-server/app/src/index.ts +++ b/apps/http-server/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@http-server/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/http-server/packages/generic/package.json b/apps/http-server/packages/generic/package.json index 852d7665..7ba4f79e 100644 --- a/apps/http-server/packages/generic/package.json +++ b/apps/http-server/packages/generic/package.json @@ -48,7 +48,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -58,4 +58,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/http-server/packages/generic/src/index.ts b/apps/http-server/packages/generic/src/index.ts index c9a015eb..a6cf4c55 100644 --- a/apps/http-server/packages/generic/src/index.ts +++ b/apps/http-server/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { PackageProxyServer } from './server' -import { getHTTPServerConfig, setupLogging } from '@shared/api' +import { getHTTPServerConfig, ProcessHandler, setupLogging } from '@shared/api' export { PackageProxyServer } export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('------------------------------------------------------------------') logger.info('Starting HTTP Server') + const process = new ProcessHandler(logger) + process.init(config.process) + const app = new PackageProxyServer(logger, config) app.init().catch((e) => { logger.error(e) diff --git a/apps/package-manager/app/package.json b/apps/package-manager/app/package.json index 684baf9f..02bdad8e 100644 --- a/apps/package-manager/app/package.json +++ b/apps/package-manager/app/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/package-manager.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/package-manager.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js package-manager.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/package-manager/app/scripts/copy-natives.js b/apps/package-manager/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/package-manager/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/package-manager/app/src/index.ts b/apps/package-manager/app/src/index.ts index c5d6b5c6..21120ec8 100644 --- a/apps/package-manager/app/src/index.ts +++ b/apps/package-manager/app/src/index.ts @@ -1,3 +1,6 @@ import { startProcess } from '@package-manager/generic' + +// eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup + startProcess() diff --git a/apps/package-manager/packages/generic/package.json b/apps/package-manager/packages/generic/package.json index 419a3fcb..d669586b 100644 --- a/apps/package-manager/packages/generic/package.json +++ b/apps/package-manager/packages/generic/package.json @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/package-manager/packages/generic/src/configManifest.ts b/apps/package-manager/packages/generic/src/configManifest.ts index b9d93477..cdd4192b 100644 --- a/apps/package-manager/packages/generic/src/configManifest.ts +++ b/apps/package-manager/packages/generic/src/configManifest.ts @@ -12,5 +12,10 @@ export const PACKAGE_MANAGER_DEVICE_CONFIG: DeviceConfigManifest = { name: 'Delay removal of packages (milliseconds)', type: ConfigManifestEntryType.INT, }, + { + id: 'useTemporaryFilePath', + name: 'Use temporary file paths when copying', + type: ConfigManifestEntryType.BOOLEAN, + }, ], } diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts index d5fe2516..5449d6c4 100644 --- a/apps/package-manager/packages/generic/src/connector.ts +++ b/apps/package-manager/packages/generic/src/connector.ts @@ -1,8 +1,7 @@ -import { ClientConnectionOptions, LoggerInstance, PackageManagerConfig } from '@shared/api' +import { ClientConnectionOptions, LoggerInstance, PackageManagerConfig, ProcessHandler } from '@shared/api' import { ExpectationManager, ExpectationManagerServerOptions } from '@shared/expectation-manager' import { CoreHandler, CoreConfig } from './coreHandler' import { PackageManagerHandler } from './packageManager' -import { ProcessHandler } from './process' import chokidar from 'chokidar' import fs from 'fs' import { promisify } from 'util' @@ -30,18 +29,21 @@ export interface DeviceConfig { export class Connector { private packageManagerHandler: PackageManagerHandler private coreHandler: CoreHandler - private _process: ProcessHandler - constructor(private _logger: LoggerInstance, private config: PackageManagerConfig) { - this._process = new ProcessHandler(this._logger) + constructor( + private _logger: LoggerInstance, + private config: PackageManagerConfig, + private _process: ProcessHandler + ) { this.coreHandler = new CoreHandler(this._logger, this.config.packageManager) - const packageManagerServerOptions: ExpectationManagerServerOptions = config.packageManager.port - ? { - type: 'websocket', - port: config.packageManager.port, - } - : { type: 'internal' } + const packageManagerServerOptions: ExpectationManagerServerOptions = + config.packageManager.port !== null + ? { + type: 'websocket', + port: config.packageManager.port, + } + : { type: 'internal' } const workForceConnectionOptions: ClientConnectionOptions = config.packageManager.workforceURL ? { @@ -61,10 +63,6 @@ export class Connector { public async init(): Promise { try { - this._logger.info('Initializing Process...') - this._process.init(this.config.process) - this._logger.info('Process initialized') - this._logger.info('Initializing Core...') await this.coreHandler.init(this.config, this._process) this._logger.info('Core initialized') diff --git a/apps/package-manager/packages/generic/src/coreHandler.ts b/apps/package-manager/packages/generic/src/coreHandler.ts index bcacad87..6043f55a 100644 --- a/apps/package-manager/packages/generic/src/coreHandler.ts +++ b/apps/package-manager/packages/generic/src/coreHandler.ts @@ -9,9 +9,7 @@ import { import { DeviceConfig } from './connector' import fs from 'fs' -import { LoggerInstance, PackageManagerConfig } from '@shared/api' - -import { ProcessHandler } from './process' +import { LoggerInstance, PackageManagerConfig, ProcessHandler } from '@shared/api' import { PACKAGE_MANAGER_DEVICE_CONFIG } from './configManifest' import { PackageManagerHandler } from './packageManager' @@ -44,6 +42,7 @@ export class CoreHandler { public deviceSettings: { [key: string]: any } = {} public delayRemoval = 0 + public useTemporaryFilePath = false private _deviceOptions: DeviceConfig private _onConnected?: () => any @@ -231,6 +230,9 @@ export class CoreHandler { if (this.deviceSettings['delayRemoval'] !== this.delayRemoval) { this.delayRemoval = this.deviceSettings['delayRemoval'] } + if (this.deviceSettings['useTemporaryFilePath'] !== this.useTemporaryFilePath) { + this.useTemporaryFilePath = this.deviceSettings['useTemporaryFilePath'] + } if (this._packageManagerHandler) { this._packageManagerHandler.onSettingsChanged() @@ -407,4 +409,13 @@ export class CoreHandler { abortExpectation(workId: string): void { return this._packageManagerHandler?.abortExpectation(workId) } + troubleshoot(): any { + return this._packageManagerHandler?.getDataSnapshot() + } + async getExpetationManagerStatus(): Promise { + return this._packageManagerHandler?.getExpetationManagerStatus() + } + async debugKillApp(appId: string): Promise { + return this._packageManagerHandler?.debugKillApp(appId) + } } diff --git a/apps/package-manager/packages/generic/src/expectationGenerator.ts b/apps/package-manager/packages/generic/src/expectationGenerator.ts index e049f508..d616c131 100644 --- a/apps/package-manager/packages/generic/src/expectationGenerator.ts +++ b/apps/package-manager/packages/generic/src/expectationGenerator.ts @@ -6,14 +6,14 @@ import { PackageContainers, PackageManagerSettings, } from './packageManager' -import { Expectation, hashObj, PackageContainerExpectation } from '@shared/api' +import { Expectation, hashObj, PackageContainerExpectation, literal, LoggerInstance } from '@shared/api' export interface ExpectedPackageWrapMediaFile extends ExpectedPackageWrap { expectedPackage: ExpectedPackage.ExpectedPackageMediaFile sources: { containerId: string label: string - accessors: ExpectedPackage.ExpectedPackageMediaFile['sources'][0]['accessors'] + accessors: NonNullable }[] } export interface ExpectedPackageWrapQuantel extends ExpectedPackageWrap { @@ -21,7 +21,15 @@ export interface ExpectedPackageWrapQuantel extends ExpectedPackageWrap { sources: { containerId: string label: string - accessors: ExpectedPackage.ExpectedPackageQuantelClip['sources'][0]['accessors'] + accessors: NonNullable + }[] +} +export interface ExpectedPackageWrapJSONData extends ExpectedPackageWrap { + expectedPackage: ExpectedPackage.ExpectedPackageJSONData + sources: { + containerId: string + label: string + accessors: NonNullable }[] } @@ -30,6 +38,7 @@ type GenerateExpectation = Expectation.Base & { external?: boolean } export function generateExpectations( + logger: LoggerInstance, managerId: string, packageContainers: PackageContainers, _activePlaylist: ActivePlaylist, @@ -55,63 +64,130 @@ export function generateExpectations( activeRundownMap.set(activeRundown._id, activeRundown) } - for (const expWrap of expectedPackages) { - let exp: Expectation.Any | undefined = undefined - - if (expWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { - exp = generateMediaFileCopy(managerId, expWrap, settings) - } else if (expWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { - exp = generateQuantelCopy(managerId, expWrap) + function prioritizeExpectation(packageWrap: ExpectedPackageWrap, exp: Expectation.Any): void { + // Prioritize + /* + 0: Things that are to be played out like RIGHT NOW + 10: Things that are to be played out pretty soon (things that could be cued anytime now) + 100: Other things that affect users (GUI things) + 1000+: Other things that can be played out + */ + + let prioAdd = 1000 + const activeRundown: ActiveRundown | undefined = packageWrap.expectedPackage.rundownId + ? activeRundownMap.get(packageWrap.expectedPackage.rundownId) + : undefined + + if (activeRundown) { + // The expected package is in an active rundown + prioAdd = 0 + activeRundown._rank // Earlier rundowns should have higher priority } - if (exp) { - // Prioritize - /* - 0: Things that are to be played out like RIGHT NOW - 10: Things that are to be played out pretty soon (things that could be cued anytime now) - 100: Other things that affect users (GUI things) - 1000+: Other things that can be played out - */ - - let prioAdd = 1000 - const activeRundown: ActiveRundown | undefined = expWrap.expectedPackage.rundownId - ? activeRundownMap.get(expWrap.expectedPackage.rundownId) - : undefined - - if (activeRundown) { - // The expected package is in an active rundown - prioAdd = 0 + activeRundown._rank // Earlier rundowns should have higher priority + exp.priority += prioAdd + } + function addExpectation(packageWrap: ExpectedPackageWrap, exp: Expectation.Any) { + const existingExp = expectations[exp.id] + if (existingExp) { + // There is already an expectation pointing at the same place. + + existingExp.priority = Math.min(existingExp.priority, exp.priority) + + const existingPackage = existingExp.fromPackages[0] + const newPackage = exp.fromPackages[0] + + if (existingPackage.expectedContentVersionHash !== newPackage.expectedContentVersionHash) { + // log warning: + logger.warn(`WARNING: 2 expectedPackages have the same content, but have different contentVersions!`) + logger.warn(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) + logger.warn(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) + logger.warn(`${JSON.stringify(exp.startRequirement)}`) + + // TODO: log better warnings! + } else { + existingExp.fromPackages.push(exp.fromPackages[0]) } - exp.priority += prioAdd + } else { + expectations[exp.id] = { + ...exp, + sideEffect: packageWrap.expectedPackage.sideEffect, + external: packageWrap.external, + } + } + } - const existingExp = expectations[exp.id] - if (existingExp) { - // There is already an expectation pointing at the same place. + const smartbullExpectations: ExpectedPackageWrap[] = [] // Hack, Smartbull + let orgSmartbullExpectation: ExpectedPackageWrap | undefined = undefined // Hack, Smartbull - existingExp.priority = Math.min(existingExp.priority, exp.priority) + for (const packageWrap of expectedPackages) { + let exp: Expectation.Any | undefined = undefined - const existingPackage = existingExp.fromPackages[0] - const newPackage = exp.fromPackages[0] + // Temporary hacks: handle smartbull: + if (packageWrap.expectedPackage._id.match(/smartbull_auto_clip/)) { + // hack + orgSmartbullExpectation = packageWrap + continue + } + if ( + packageWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE && + packageWrap.sources.find((source) => source.containerId === 'source-smartbull') + ) { + if ((packageWrap as ExpectedPackageWrapMediaFile).expectedPackage.content.filePath.match(/^smartbull/)) { + // the files are on the form "smartbull_TIMESTAMP.mxf/mp4" + smartbullExpectations.push(packageWrap) + } + // (any other files in the "source-smartbull"-container are to be ignored) + continue + } - if (existingPackage.expectedContentVersionHash !== newPackage.expectedContentVersionHash) { - // log warning: - console.log( - `WARNING: 2 expectedPackages have the same content, but have different contentVersions!` - ) - console.log(`"${existingPackage.id}": ${existingPackage.expectedContentVersionHash}`) - console.log(`"${newPackage.id}": ${newPackage.expectedContentVersionHash}`) - console.log(`${JSON.stringify(exp.startRequirement)}`) + if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { + exp = generateMediaFileCopy(managerId, packageWrap, settings) + } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.QUANTEL_CLIP) { + exp = generateQuantelCopy(managerId, packageWrap) + } else if (packageWrap.expectedPackage.type === ExpectedPackage.PackageType.JSON_DATA) { + exp = generateJsonDataCopy(managerId, packageWrap, settings) + } + if (exp) { + prioritizeExpectation(packageWrap, exp) + addExpectation(packageWrap, exp) + } + } - // TODO: log better warnings! - } else { - existingExp.fromPackages.push(exp.fromPackages[0]) + // hack: handle Smartbull: + if (orgSmartbullExpectation) { + // Sort alphabetically on filePath: + smartbullExpectations.sort((a, b) => { + const expA = a.expectedPackage as ExpectedPackage.ExpectedPackageMediaFile + const expB = b.expectedPackage as ExpectedPackage.ExpectedPackageMediaFile + + // lowest first: + if (expA.content.filePath > expB.content.filePath) return 1 + if (expA.content.filePath < expB.content.filePath) return -1 + + return 0 + }) + // Pick the last one: + const bestSmartbull = smartbullExpectations[smartbullExpectations.length - 1] as + | ExpectedPackageWrapMediaFile + | undefined + if (bestSmartbull) { + if (orgSmartbullExpectation.expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) { + const org = orgSmartbullExpectation as ExpectedPackageWrapMediaFile + + const newPackage: ExpectedPackageWrapMediaFile = { + ...org, + expectedPackage: { + ...org.expectedPackage, + // Take these from bestSmartbull: + content: bestSmartbull.expectedPackage.content, + version: {}, // Don't even use bestSmartbull.expectedPackage.version, + sources: bestSmartbull.expectedPackage.sources, + }, } - } else { - expectations[exp.id] = { - ...exp, - sideEffect: expWrap.expectedPackage.sideEffect, - external: expWrap.external, + const exp = generateMediaFileCopy(managerId, newPackage, settings) + if (exp) { + prioritizeExpectation(newPackage, exp) + addExpectation(newPackage, exp) } - } + } else logger.warn('orgSmartbullExpectation is not a MEDIA_FILE') } } @@ -131,28 +207,35 @@ export function generateExpectations( } if (expectation0.sideEffect?.thumbnailContainerId && expectation0.sideEffect?.thumbnailPackageSettings) { - const packageContainer: PackageContainer = - packageContainers[expectation0.sideEffect.thumbnailContainerId] - - const thumbnail = generateMediaFileThumbnail( - expectation, - expectation0.sideEffect.thumbnailContainerId, - expectation0.sideEffect.thumbnailPackageSettings, - packageContainer - ) - expectations[thumbnail.id] = thumbnail + const packageContainer = packageContainers[expectation0.sideEffect.thumbnailContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const thumbnail = generateMediaFileThumbnail( + expectation, + expectation0.sideEffect.thumbnailContainerId, + expectation0.sideEffect.thumbnailPackageSettings, + packageContainer + ) + expectations[thumbnail.id] = thumbnail + } } if (expectation0.sideEffect?.previewContainerId && expectation0.sideEffect?.previewPackageSettings) { - const packageContainer: PackageContainer = packageContainers[expectation0.sideEffect.previewContainerId] - - const preview = generateMediaFilePreview( - expectation, - expectation0.sideEffect.previewContainerId, - expectation0.sideEffect.previewPackageSettings, - packageContainer - ) - expectations[preview.id] = preview + const packageContainer = packageContainers[expectation0.sideEffect.previewContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const preview = generateMediaFilePreview( + expectation, + expectation0.sideEffect.previewContainerId, + expectation0.sideEffect.previewPackageSettings, + packageContainer + ) + expectations[preview.id] = preview + } } } else if (expectation0.type === Expectation.Type.QUANTEL_CLIP_COPY) { const expectation = expectation0 as Expectation.QuantelClipCopy @@ -168,28 +251,35 @@ export function generateExpectations( } if (expectation0.sideEffect?.thumbnailContainerId && expectation0.sideEffect?.thumbnailPackageSettings) { - const packageContainer: PackageContainer = - packageContainers[expectation0.sideEffect.thumbnailContainerId] - - const thumbnail = generateQuantelClipThumbnail( - expectation, - expectation0.sideEffect.thumbnailContainerId, - expectation0.sideEffect.thumbnailPackageSettings, - packageContainer - ) - expectations[thumbnail.id] = thumbnail + const packageContainer = packageContainers[expectation0.sideEffect.thumbnailContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const thumbnail = generateQuantelClipThumbnail( + expectation, + expectation0.sideEffect.thumbnailContainerId, + expectation0.sideEffect.thumbnailPackageSettings, + packageContainer + ) + expectations[thumbnail.id] = thumbnail + } } if (expectation0.sideEffect?.previewContainerId && expectation0.sideEffect?.previewPackageSettings) { - const packageContainer: PackageContainer = packageContainers[expectation0.sideEffect.previewContainerId] - - const preview = generateQuantelClipPreview( - expectation, - expectation0.sideEffect.previewContainerId, - expectation0.sideEffect.previewPackageSettings, - packageContainer - ) - expectations[preview.id] = preview + const packageContainer = packageContainers[expectation0.sideEffect.previewContainerId] as + | PackageContainer + | undefined + + if (packageContainer) { + const preview = generateQuantelClipPreview( + expectation, + expectation0.sideEffect.previewContainerId, + expectation0.sideEffect.previewPackageSettings, + packageContainer + ) + expectations[preview.id] = preview + } } } } @@ -221,10 +311,10 @@ function generateMediaFileCopy( ], type: Expectation.Type.FILE_COPY, statusReport: { - label: `Copy media "${expWrapMediaFile.expectedPackage.content.filePath}"`, - description: `Copy media "${expWrapMediaFile.expectedPackage.content.filePath}" to the playout-device "${ + label: `Copying media "${expWrapMediaFile.expectedPackage.content.filePath}"`, + description: `Copy media file "${expWrapMediaFile.expectedPackage.content.filePath}" to the device "${ expWrapMediaFile.playoutDeviceId - }", from "${JSON.stringify(expWrapMediaFile.sources)}"`, + }", from ${expWrapMediaFile.sources.map((source) => `"${source.label}"`).join(', ')}`, requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, @@ -244,6 +334,7 @@ function generateMediaFileCopy( }, workOptions: { removeDelay: settings.delayRemoval, + useTemporaryFilePath: settings.useTemporaryFilePath, }, } exp.id = hashObj(exp.endRequirement) @@ -253,6 +344,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E const expWrapQuantelClip = expWrap as ExpectedPackageWrapQuantel const content = expWrapQuantelClip.expectedPackage.content + const label = content.title && content.guid ? `${content.title} (${content.guid})` : content.title || content.guid const exp: Expectation.QuantelClipCopy = { id: '', // set later priority: expWrap.priority * 10 || 0, @@ -266,10 +358,10 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E ], statusReport: { - label: `Copy Quantel clip ${content.title || content.guid}`, + label: `Copy Quantel clip ${label}`, description: `Copy Quantel clip ${content.title || content.guid} to server for "${ expWrapQuantelClip.playoutDeviceId - }", from ${expWrapQuantelClip.sources}`, + }", from ${expWrapQuantelClip.sources.map((source) => `"${source.label}"`).join(', ')}`, requiredForPlayout: true, displayRank: 0, sendReport: !expWrap.external, @@ -296,7 +388,7 @@ function generateQuantelCopy(managerId: string, expWrap: ExpectedPackageWrap): E return exp } function generatePackageScan(expectation: Expectation.FileCopy | Expectation.QuantelClipCopy): Expectation.PackageScan { - const scan: Expectation.PackageScan = { + return literal({ id: expectation.id + '_scan', priority: expectation.priority + 100, managerId: expectation.managerId, @@ -304,8 +396,8 @@ function generatePackageScan(expectation: Expectation.FileCopy | Expectation.Qua fromPackages: expectation.fromPackages, statusReport: { - label: `Scan ${expectation.statusReport.label}`, - description: `Scanning is used to provide Sofie GUI with status about the media`, + label: `Scanning`, + description: `Scanning the media, to provide data to the Sofie GUI`, requiredForPlayout: false, displayRank: 10, sendReport: expectation.statusReport.sendReport, @@ -337,14 +429,12 @@ function generatePackageScan(expectation: Expectation.FileCopy | Expectation.Qua }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return scan + }) } function generatePackageDeepScan( expectation: Expectation.FileCopy | Expectation.QuantelClipCopy ): Expectation.PackageDeepScan { - const deepScan: Expectation.PackageDeepScan = { + return literal({ id: expectation.id + '_deepscan', priority: expectation.priority + 1001, managerId: expectation.managerId, @@ -352,10 +442,10 @@ function generatePackageDeepScan( fromPackages: expectation.fromPackages, statusReport: { - label: `Deep Scan ${expectation.statusReport.label}`, - description: `Deep scanning includes scene-detection, black/freeze frames etc.`, + label: `Deep Scanning`, + description: `Detecting scenes, black frames, freeze frames etc.`, requiredForPlayout: false, - displayRank: 10, + displayRank: 11, sendReport: expectation.statusReport.sendReport, }, @@ -390,9 +480,7 @@ function generatePackageDeepScan( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return deepScan + }) } function generateMediaFileThumbnail( @@ -401,7 +489,7 @@ function generateMediaFileThumbnail( settings: ExpectedPackage.SideEffectThumbnailSettings, packageContainer: PackageContainer ): Expectation.MediaFileThumbnail { - const thumbnail: Expectation.MediaFileThumbnail = { + return literal({ id: expectation.id + '_thumbnail', priority: expectation.priority + 1002, managerId: expectation.managerId, @@ -409,7 +497,7 @@ function generateMediaFileThumbnail( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate thumbnail for ${expectation.statusReport.label}`, + label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, requiredForPlayout: false, displayRank: 11, @@ -444,9 +532,7 @@ function generateMediaFileThumbnail( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return thumbnail + }) } function generateMediaFilePreview( expectation: Expectation.FileCopy, @@ -454,7 +540,7 @@ function generateMediaFilePreview( settings: ExpectedPackage.SideEffectPreviewSettings, packageContainer: PackageContainer ): Expectation.MediaFilePreview { - const preview: Expectation.MediaFilePreview = { + return literal({ id: expectation.id + '_preview', priority: expectation.priority + 1003, managerId: expectation.managerId, @@ -462,7 +548,7 @@ function generateMediaFilePreview( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate preview for ${expectation.statusReport.label}`, + label: `Generating preview`, description: `Preview is used in Sofie GUI`, requiredForPlayout: false, displayRank: 12, @@ -496,8 +582,7 @@ function generateMediaFilePreview( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - return preview + }) } function generateQuantelClipThumbnail( @@ -506,7 +591,7 @@ function generateQuantelClipThumbnail( settings: ExpectedPackage.SideEffectThumbnailSettings, packageContainer: PackageContainer ): Expectation.QuantelClipThumbnail { - const thumbnail: Expectation.QuantelClipThumbnail = { + return literal({ id: expectation.id + '_thumbnail', priority: expectation.priority + 1002, managerId: expectation.managerId, @@ -514,7 +599,7 @@ function generateQuantelClipThumbnail( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate thumbnail for ${expectation.statusReport.label}`, + label: `Generating thumbnail`, description: `Thumbnail is used in Sofie GUI`, requiredForPlayout: false, displayRank: 11, @@ -548,9 +633,7 @@ function generateQuantelClipThumbnail( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], - } - - return thumbnail + }) } function generateQuantelClipPreview( expectation: Expectation.QuantelClipCopy, @@ -558,7 +641,7 @@ function generateQuantelClipPreview( settings: ExpectedPackage.SideEffectPreviewSettings, packageContainer: PackageContainer ): Expectation.QuantelClipPreview { - const preview: Expectation.QuantelClipPreview = { + return literal({ id: expectation.id + '_preview', priority: expectation.priority + 1003, managerId: expectation.managerId, @@ -566,7 +649,7 @@ function generateQuantelClipPreview( fromPackages: expectation.fromPackages, statusReport: { - label: `Generate preview for ${expectation.statusReport.label}`, + label: `Generating preview`, description: `Preview is used in Sofie GUI`, requiredForPlayout: false, displayRank: 12, @@ -594,9 +677,6 @@ function generateQuantelClipPreview( }, version: { type: Expectation.Version.Type.QUANTEL_CLIP_PREVIEW, - // bitrate: string // default: '40k' - // width: number - // height: number }, }, workOptions: { @@ -605,8 +685,56 @@ function generateQuantelClipPreview( }, dependsOnFullfilled: [expectation.id], triggerByFullfilledIds: [expectation.id], + }) +} + +function generateJsonDataCopy( + managerId: string, + expWrap: ExpectedPackageWrap, + settings: PackageManagerSettings +): Expectation.JsonDataCopy { + const expWrapMediaFile = expWrap as ExpectedPackageWrapJSONData + + const exp: Expectation.JsonDataCopy = { + id: '', // set later + priority: expWrap.priority * 10 || 0, + managerId: managerId, + fromPackages: [ + { + id: expWrap.expectedPackage._id, + expectedContentVersionHash: expWrap.expectedPackage.contentVersionHash, + }, + ], + type: Expectation.Type.JSON_DATA_COPY, + statusReport: { + label: `Copying JSON data`, + description: `Copy JSON data "${expWrapMediaFile.expectedPackage.content.path}" from "${JSON.stringify( + expWrapMediaFile.sources + )}"`, + requiredForPlayout: true, + displayRank: 0, + sendReport: !expWrap.external, + }, + + startRequirement: { + sources: expWrapMediaFile.sources, + }, + + endRequirement: { + targets: expWrapMediaFile.targets as [Expectation.SpecificPackageContainerOnPackage.File], + content: expWrapMediaFile.expectedPackage.content, + version: { + type: Expectation.Version.Type.FILE_ON_DISK, + ...expWrapMediaFile.expectedPackage.version, + }, + }, + workOptions: { + removeDelay: settings.delayRemoval, + useTemporaryFilePath: settings.useTemporaryFilePath, + }, } - return preview + exp.id = hashObj(exp.endRequirement) + return exp } // function generateMediaFileHTTPCopy(expectation: Expectation.FileCopy): Expectation.FileCopy { @@ -663,9 +791,9 @@ export function generatePackageContainerExpectations( ): { [id: string]: PackageContainerExpectation } { const o: { [id: string]: PackageContainerExpectation } = {} - // This is temporary, to test/show how the for (const [containerId, packageContainer] of Object.entries(packageContainers)) { - if (containerId === 'source0') { + // This is temporary, to test/show how a monitor would work: + if (containerId === 'source_monitor') { o[containerId] = { ...packageContainer, id: containerId, @@ -680,6 +808,25 @@ export function generatePackageContainerExpectations( }, } } + + // This is a hard-coded hack for the "smartbull" feature, + // to be replaced or moved out later: + if (containerId === 'source-smartbull') { + o[containerId] = { + ...packageContainer, + id: containerId, + managerId: managerId, + cronjobs: {}, + monitors: { + packages: { + targetLayers: ['source-smartbull'], // not used, since the layers of the original smartbull-package are used + usePolling: 2000, + awaitWriteFinishStabilityThreshold: 2000, + }, + }, + } + } } + return o } diff --git a/apps/package-manager/packages/generic/src/index.ts b/apps/package-manager/packages/generic/src/index.ts index 5d13f25d..6df38b98 100644 --- a/apps/package-manager/packages/generic/src/index.ts +++ b/apps/package-manager/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { Connector, Config } from './connector' -import { getPackageManagerConfig, LoggerInstance, setupLogging } from '@shared/api' +import { getPackageManagerConfig, LoggerInstance, ProcessHandler, setupLogging } from '@shared/api' export { Connector, Config } export function startProcess(startInInternalMode?: boolean): { logger: LoggerInstance; connector: Connector } { @@ -15,7 +15,11 @@ export function startProcess(startInInternalMode?: boolean): { logger: LoggerIns config.packageManager.accessUrl = null config.packageManager.workforceURL = null } - const connector = new Connector(logger, config) + + const process = new ProcessHandler(logger) + process.init(config.process) + + const connector = new Connector(logger, config, process) logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) logger.info('------------------------------------------------------------------') diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index c397d1cc..9d457fc3 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -10,7 +10,11 @@ import { PackageContainerOnPackage, } from '@sofie-automation/blueprints-integration' import { generateExpectations, generatePackageContainerExpectations } from './expectationGenerator' -import { ExpectationManager, ExpectationManagerServerOptions } from '@shared/expectation-manager' +import { + ExpectationManager, + ExpectationManagerCallbacks, + ExpectationManagerServerOptions, +} from '@shared/expectation-manager' import { ClientConnectionOptions, Expectation, @@ -19,51 +23,48 @@ import { LoggerInstance, PackageContainerExpectation, literal, + Reason, } from '@shared/api' import deepExtend from 'deep-extend' import clone = require('fast-clone') export class PackageManagerHandler { - private _coreHandler!: CoreHandler + public coreHandler!: CoreHandler private _observers: Array = [] - private _expectationManager: ExpectationManager + public expectationManager: ExpectationManager private expectedPackageCache: { [id: string]: ExpectedPackageWrap } = {} - private packageContainersCache: PackageContainers = {} - - private reportedWorkStatuses: { [id: string]: ExpectedPackageStatusAPI.WorkStatus } = {} - private toReportExpectationStatus: { - [id: string]: { - workStatus: ExpectedPackageStatusAPI.WorkStatus | null - /** If the status is new and needs to be reported to Core */ - isUpdated: boolean - } - } = {} - private sendUpdateExpectationStatusTimeouts: NodeJS.Timeout | undefined - - private reportedPackageStatuses: { [id: string]: ExpectedPackageStatusAPI.PackageContainerPackageStatus } = {} - private toReportPackageStatus: { - [key: string]: { - containerId: string - packageId: string - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - /** If the status is new and needs to be reported to Core */ - isUpdated: boolean - } - } = {} - private sendUpdatePackageContainerPackageStatusTimeouts: NodeJS.Timeout | undefined + public packageContainersCache: PackageContainers = {} private externalData: { packageContainers: PackageContainers; expectedPackages: ExpectedPackageWrap[] } = { packageContainers: {}, expectedPackages: [], } private _triggerUpdatedExpectedPackagesTimeout: NodeJS.Timeout | null = null - private monitoredPackages: { + public monitoredPackages: { [monitorId: string]: ResultingExpectedPackage[] } = {} settings: PackageManagerSettings = { delayRemoval: 0, + useTemporaryFilePath: false, + } + callbacksHandler: ExpectationManagerCallbacksHandler + + private dataSnapshot: { + updated: number + expectedPackages: ResultingExpectedPackage[] + packageContainers: PackageContainers + expectations: { + [id: string]: Expectation.Any + } + packageContainerExpectations: { [id: string]: PackageContainerExpectation } + } = { + updated: 0, + expectedPackages: [], + packageContainers: {}, + expectations: {}, + packageContainerExpectations: {}, } constructor( @@ -73,42 +74,22 @@ export class PackageManagerHandler { private serverAccessUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions ) { - this._expectationManager = new ExpectationManager( + this.callbacksHandler = new ExpectationManagerCallbacksHandler(this) + + this.expectationManager = new ExpectationManager( this.logger, this.managerId, this.serverOptions, this.serverAccessUrl, this.workForceConnectionOptions, - ( - expectationId: string, - expectaction: Expectation.Any | null, - actualVersionHash: string | null, - statusInfo: { - status?: string - progress?: number - statusReason?: string - } - ) => this.updateExpectationStatus(expectationId, expectaction, actualVersionHash, statusInfo), - ( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - ) => this.updatePackageContainerPackageStatus(containerId, packageId, packageStatus), - ( - containerId: string, - packageContainer: PackageContainerExpectation | null, - statusInfo: { - statusReason?: string - } - ) => this.updatePackageContainerStatus(containerId, packageContainer, statusInfo), - (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => this.onMessageFromWorker(message) + this.callbacksHandler ) } async init(_config: PackageManagerConfig, coreHandler: CoreHandler): Promise { - this._coreHandler = coreHandler + this.coreHandler = coreHandler - this._coreHandler.setPackageManagerHandler(this) + this.coreHandler.setPackageManagerHandler(this) this.logger.info('PackageManagerHandler init') @@ -121,99 +102,29 @@ export class PackageManagerHandler { }) this.setupObservers() this.onSettingsChanged() - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() - await this.cleanReportedExpectations() - await this._expectationManager.init() + await this.callbacksHandler.cleanReportedExpectations() + await this.expectationManager.init() this.logger.info('PackageManagerHandler initialized') } onSettingsChanged(): void { this.settings = { - delayRemoval: this._coreHandler.delayRemoval, + delayRemoval: this.coreHandler.delayRemoval, + useTemporaryFilePath: this.coreHandler.useTemporaryFilePath, } + this.triggerUpdatedExpectedPackages() } getExpectationManager(): ExpectationManager { - return this._expectationManager + return this.expectationManager } - private wrapExpectedPackage( - packageContainers: PackageContainers, - expectedPackage: ExpectedPackage.Any - ): ExpectedPackageWrap | undefined { - const combinedSources: PackageContainerOnPackage[] = [] - for (const packageSource of expectedPackage.sources) { - const lookedUpSource: PackageContainer = packageContainers[packageSource.containerId] - if (lookedUpSource) { - // We're going to combine the accessor attributes set on the Package with the ones defined on the source: - const combinedSource: PackageContainerOnPackage = { - ...omit(clone(lookedUpSource), 'accessors'), - accessors: {}, - containerId: packageSource.containerId, - } - - const accessorIds = _.uniq( - Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors)) - ) - - for (const accessorId of accessorIds) { - const sourceAccessor = lookedUpSource.accessors[accessorId] as Accessor.Any | undefined - const packageAccessor = packageSource.accessors[accessorId] as AccessorOnPackage.Any | undefined - - if (packageAccessor && sourceAccessor && packageAccessor.type === sourceAccessor.type) { - combinedSource.accessors[accessorId] = deepExtend({}, sourceAccessor, packageAccessor) - } else if (packageAccessor) { - combinedSource.accessors[accessorId] = clone(packageAccessor) - } else if (sourceAccessor) { - combinedSource.accessors[accessorId] = clone( - sourceAccessor - ) as AccessorOnPackage.Any - } - } - combinedSources.push(combinedSource) - } - } - // Lookup Package targets: - const combinedTargets: PackageContainerOnPackage[] = [] - - for (const layer of expectedPackage.layers) { - // Hack: we use the layer name as a 1-to-1 relation to a target containerId - const packageContainerId: string = layer - - if (packageContainerId) { - const lookedUpTarget = packageContainers[packageContainerId] - if (lookedUpTarget) { - // Todo: should the be any combination of properties here? - combinedTargets.push({ - ...omit(clone(lookedUpTarget), 'accessors'), - accessors: lookedUpTarget.accessors as { - [accessorId: string]: AccessorOnPackage.Any - }, - containerId: packageContainerId, - }) - } - } - } - - if (combinedSources.length) { - if (combinedTargets.length) { - return { - expectedPackage: expectedPackage, - priority: 999, // lowest priority - sources: combinedSources, - targets: combinedTargets, - playoutDeviceId: '', - external: true, - } - } - } - return undefined - } setExternalData(packageContainers: PackageContainers, expectedPackages: ExpectedPackage.Any[]): void { const expectedPackagesWraps: ExpectedPackageWrap[] = [] for (const expectedPackage of expectedPackages) { - const wrap = this.wrapExpectedPackage(packageContainers, expectedPackage) + const wrap = wrapExpectedPackage(packageContainers, expectedPackage) if (wrap) { expectedPackagesWraps.push(wrap) } @@ -223,7 +134,7 @@ export class PackageManagerHandler { packageContainers: packageContainers, expectedPackages: expectedPackagesWraps, } - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } private setupObservers(): void { if (this._observers.length) { @@ -233,23 +144,21 @@ export class PackageManagerHandler { }) this._observers = [] } - this.logger.info('Renewing observers') + this.logger.debug('Renewing observers') - const expectedPackagesObserver = this._coreHandler.core.observe('deviceExpectedPackages') + const expectedPackagesObserver = this.coreHandler.core.observe('deviceExpectedPackages') expectedPackagesObserver.added = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } expectedPackagesObserver.changed = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } expectedPackagesObserver.removed = () => { - this._triggerUpdatedExpectedPackages() + this.triggerUpdatedExpectedPackages() } this._observers.push(expectedPackagesObserver) } - private _triggerUpdatedExpectedPackages() { - this.logger.info('_triggerUpdatedExpectedPackages') - + public triggerUpdatedExpectedPackages(): void { if (this._triggerUpdatedExpectedPackagesTimeout) { clearTimeout(this._triggerUpdatedExpectedPackagesTimeout) this._triggerUpdatedExpectedPackagesTimeout = null @@ -257,12 +166,11 @@ export class PackageManagerHandler { this._triggerUpdatedExpectedPackagesTimeout = setTimeout(() => { this._triggerUpdatedExpectedPackagesTimeout = null - this.logger.info('_triggerUpdatedExpectedPackages inner') const expectedPackages: ExpectedPackageWrap[] = [] const packageContainers: PackageContainers = {} - const objs = this._coreHandler.core.getCollection('deviceExpectedPackages').find(() => true) + const objs = this.coreHandler.core.getCollection('deviceExpectedPackages').find(() => true) const activePlaylistObj = objs.find((o) => o.type === 'active_playlist') if (!activePlaylistObj) { @@ -341,41 +249,151 @@ export class PackageManagerHandler { } } - this.logger.info(`Has ${expectedPackages.length} expectedPackages`) - // this.logger.info(JSON.stringify(expectedPackages, null, 2)) + this.logger.debug(`Has ${expectedPackages.length} expectedPackages`) + // this.logger.debug(JSON.stringify(expectedPackages, null, 2)) + + this.dataSnapshot.expectedPackages = expectedPackages + this.dataSnapshot.packageContainers = this.packageContainersCache // Step 1: Generate expectations: const expectations = generateExpectations( - this._expectationManager.managerId, + this.logger, + this.expectationManager.managerId, this.packageContainersCache, activePlaylist, activeRundowns, expectedPackages, this.settings ) - this.logger.info(`Has ${Object.keys(expectations).length} expectations`) + this.logger.debug(`Has ${Object.keys(expectations).length} expectations`) + // this.logger.debug(JSON.stringify(expectations, null, 2)) + this.dataSnapshot.expectations = expectations + const packageContainerExpectations = generatePackageContainerExpectations( - this._expectationManager.managerId, + this.expectationManager.managerId, this.packageContainersCache, activePlaylist ) - this.logger.info(`Has ${Object.keys(packageContainerExpectations).length} packageContainerExpectations`) + this.logger.debug(`Has ${Object.keys(packageContainerExpectations).length} packageContainerExpectations`) + ;(this.dataSnapshot.packageContainerExpectations = packageContainerExpectations), + (this.dataSnapshot.updated = Date.now()) + this.ensureMandatoryPackageContainerExpectations(packageContainerExpectations) - // this.logger.info(JSON.stringify(expectations, null, 2)) // Step 2: Track and handle new expectations: - this._expectationManager.updatePackageContainerExpectations(packageContainerExpectations) + this.expectationManager.updatePackageContainerExpectations(packageContainerExpectations) - this._expectationManager.updateExpectations(expectations) + this.expectationManager.updateExpectations(expectations) } - public updateExpectationStatus( + public restartExpectation(workId: string): void { + // This method can be called from core + this.expectationManager.restartExpectation(workId) + } + public restartAllExpectations(): void { + // This method can be called from core + this.expectationManager.restartAllExpectations() + } + public abortExpectation(workId: string): void { + // This method can be called from core + this.expectationManager.abortExpectation(workId) + } + public getDataSnapshot(): any { + return this.dataSnapshot + } + public async getExpetationManagerStatus(): Promise { + return { + ...(await this.expectationManager.getStatus()), + packageManager: { + workforceURL: + this.workForceConnectionOptions.type === 'websocket' ? this.workForceConnectionOptions.url : null, + lastUpdated: this.dataSnapshot.updated, + countExpectedPackages: this.dataSnapshot.expectedPackages.length, + countPackageContainers: Object.keys(this.dataSnapshot.packageContainers).length, + countExpectations: Object.keys(this.dataSnapshot.expectations).length, + countPackageContainerExpectations: Object.keys(this.dataSnapshot.packageContainerExpectations).length, + }, + } + } + public async debugKillApp(appId: string): Promise { + return this.expectationManager.debugKillApp(appId) + } + + /** Ensures that the packageContainerExpectations containes the mandatory expectations */ + private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { + [id: string]: PackageContainerExpectation + }): void { + for (const [containerId, packageContainer] of Object.entries(this.packageContainersCache)) { + /** Is the Container writeable */ + let isWriteable = false + for (const accessor of Object.values(packageContainer.accessors)) { + if (accessor.allowWrite) { + isWriteable = true + break + } + } + // All writeable packageContainers should have the clean up cronjob: + if (isWriteable) { + if (!packageContainerExpectations[containerId]) { + // todo: Maybe should not all package-managers monitor, + // this should perhaps be coordinated with the Workforce-manager, who should monitor who? + + // Add default packageContainerExpectation: + packageContainerExpectations[containerId] = literal({ + ...packageContainer, + id: containerId, + managerId: this.expectationManager.managerId, + cronjobs: { + interval: 0, + }, + monitors: {}, + }) + } + packageContainerExpectations[containerId].cronjobs.cleanup = {} // Add cronjob to clean up + } + } + } +} +export function omit(obj: T, ...props: P[]): Omit { + return _.omit(obj, ...(props as string[])) as any +} + +/** This class handles data and requests that comes from ExpectationManager. */ +class ExpectationManagerCallbacksHandler implements ExpectationManagerCallbacks { + reportedWorkStatuses: { [id: string]: ExpectedPackageStatusAPI.WorkStatus } = {} + toReportExpectationStatus: { + [id: string]: { + workStatus: ExpectedPackageStatusAPI.WorkStatus | null + /** If the status is new and needs to be reported to Core */ + isUpdated: boolean + } + } = {} + sendUpdateExpectationStatusTimeouts: NodeJS.Timeout | undefined + + reportedPackageStatuses: { [id: string]: ExpectedPackageStatusAPI.PackageContainerPackageStatus } = {} + toReportPackageStatus: { + [key: string]: { + containerId: string + packageId: string + packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null + /** If the status is new and needs to be reported to Core */ + isUpdated: boolean + } + } = {} + sendUpdatePackageContainerPackageStatusTimeouts: NodeJS.Timeout | undefined + + logger: LoggerInstance + constructor(private packageManager: PackageManagerHandler) { + this.logger = this.packageManager.logger + } + + public reportExpectationStatus( expectationId: string, expectaction: Expectation.Any | null, actualVersionHash: string | null, statusInfo: { - status?: string + status?: ExpectedPackageStatusAPI.WorkStatusState progress?: number - statusReason?: string + statusReason?: Reason } ): void { if (!expectaction) { @@ -385,16 +403,20 @@ export class PackageManagerHandler { } else { if (!expectaction.statusReport.sendReport) return // Don't report the status + const previouslyReported = this.toReportExpectationStatus[expectationId]?.workStatus + const workStatus: ExpectedPackageStatusAPI.WorkStatus = { // Default properties: ...{ - status: 'N/A', + status: ExpectedPackageStatusAPI.WorkStatusState.NEW, + statusChanged: 0, progress: 0, - statusReason: '', + statusReason: { user: '', tech: '' }, }, // Previous properties: - ...(((this.toReportExpectationStatus[expectationId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in const packageStatus more strict - // Updated porperties: + ...((previouslyReported || {}) as Partial), // Intentionally cast to Partial<>, to make typings in const workStatus more strict + + // Updated properties: ...expectaction.statusReport, ...statusInfo, @@ -410,9 +432,101 @@ export class PackageManagerHandler { }), } + // Update statusChanged: + workStatus.statusChanged = previouslyReported?.statusChanged || Date.now() + if ( + workStatus.status !== previouslyReported?.status || + workStatus.progress !== previouslyReported?.progress + // (not checking statusReason, as that should not affect statusChanged) + ) { + workStatus.statusChanged = Date.now() + } + this.triggerSendUpdateExpectationStatus(expectationId, workStatus) } } + public reportPackageContainerPackageStatus( + containerId: string, + packageId: string, + packageStatus: Omit | null + ): void { + const packageContainerPackageId = `${containerId}_${packageId}` + if (!packageStatus) { + this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, null) + } else { + const previouslyReported = this.toReportPackageStatus[packageContainerPackageId]?.packageStatus + + const containerStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { + // Default properties: + ...{ + status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, + progress: 0, + statusChanged: 0, + statusReason: { user: '', tech: '' }, + }, + // pre-existing properties: + ...((previouslyReported || {}) as Partial), // Intentionally cast to Partial<>, to make typings in const containerStatus more strict + // Updated properties: + ...packageStatus, + } + + // Update statusChanged: + containerStatus.statusChanged = previouslyReported?.statusChanged || Date.now() + if ( + containerStatus.status !== previouslyReported?.status || + containerStatus.progress !== previouslyReported?.progress + // (not checking statusReason, as that should not affect statusChanged) + ) { + containerStatus.statusChanged = Date.now() + } + + this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, containerStatus) + } + } + public reportPackageContainerExpectationStatus( + containerId: string, + _packageContainer: PackageContainerExpectation | null, + statusInfo: { statusReason?: Reason } + ): void { + // This is not (yet) reported to Core. + // ...to be implemented... + this.logger.info(`PackageContainerStatus "${containerId}"`) + this.logger.info(statusInfo.statusReason?.tech || '>No reason<') + } + public async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { + switch (message.type) { + case 'fetchPackageInfoMetadata': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.fetchPackageInfoMetadata, + message.arguments + ) + case 'updatePackageInfo': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.updatePackageInfo, + message.arguments + ) + case 'removePackageInfo': + return this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.removePackageInfo, + message.arguments + ) + case 'reportFromMonitorPackages': + this.reportMonitoredPackages(...message.arguments) + break + + default: + // @ts-expect-error message is never + throw new Error(`Unsupported message type "${message.type}"`) + } + } + public async cleanReportedExpectations() { + // Clean out all reported statuses, this is an easy way to sync a clean state with core + this.reportedWorkStatuses = {} + await this.packageManager.coreHandler.core.callMethod( + PeripheralDeviceAPI.methods.removeAllExpectedPackageWorkStatusOfDevice, + [] + ) + } private triggerSendUpdateExpectationStatus( expectationId: string, workStatus: ExpectedPackageStatusAPI.WorkStatus | null @@ -477,7 +591,7 @@ export class PackageManagerHandler { } if (changesTosend.length) { - this._coreHandler.core + this.packageManager.coreHandler.core .callMethod(PeripheralDeviceAPI.methods.updateExpectedPackageWorkStatuses, [changesTosend]) .catch((err) => { this.logger.error('Error when calling method updateExpectedPackageWorkStatuses:') @@ -485,39 +599,6 @@ export class PackageManagerHandler { }) } } - private async cleanReportedExpectations() { - // Clean out all reported statuses, this is an easy way to sync a clean state with core - this.reportedWorkStatuses = {} - await this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.removeAllExpectedPackageWorkStatusOfDevice, - [] - ) - } - public updatePackageContainerPackageStatus( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - ): void { - const packageContainerPackageId = `${containerId}_${packageId}` - if (!packageStatus) { - this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, null) - } else { - const o: ExpectedPackageStatusAPI.PackageContainerPackageStatus = { - // Default properties: - ...{ - status: ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.NOT_READY, - progress: 0, - statusReason: '', - }, - // pre-existing properties: - ...(((this.toReportPackageStatus[packageContainerPackageId] || {}) as any) as Record), // Intentionally cast to Any, to make typings in the outer spread-assignment more strict - // Updated properties: - ...packageStatus, - } - - this.triggerSendUpdatePackageContainerPackageStatus(containerId, packageId, o) - } - } private triggerSendUpdatePackageContainerPackageStatus( containerId: string, packageId: string, @@ -539,7 +620,7 @@ export class PackageManagerHandler { }, 300) } } - public sendUpdatePackageContainerPackageStatus(): void { + private sendUpdatePackageContainerPackageStatus(): void { const changesTosend: UpdatePackageContainerPackageStatusesChanges = [] for (const [key, o] of Object.entries(this.toReportPackageStatus)) { @@ -570,7 +651,7 @@ export class PackageManagerHandler { } if (changesTosend.length) { - this._coreHandler.core + this.packageManager.coreHandler.core .callMethod(PeripheralDeviceAPI.methods.updatePackageContainerPackageStatuses, [changesTosend]) .catch((err) => { this.logger.error('Error when calling method updatePackageContainerPackageStatuses:') @@ -578,105 +659,94 @@ export class PackageManagerHandler { }) } } - public updatePackageContainerStatus( - containerId: string, - _packageContainer: PackageContainerExpectation | null, - statusInfo: { statusReason?: string | undefined } - ): void { - this.logger.info(`PackageContainerStatus "${containerId}"`) - this.logger.info(statusInfo.statusReason || '>No reason<') - } - private async onMessageFromWorker( - message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any - ): Promise { - switch (message.type) { - case 'fetchPackageInfoMetadata': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.fetchPackageInfoMetadata, - message.arguments - ) - case 'updatePackageInfo': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.updatePackageInfo, - message.arguments - ) - case 'removePackageInfo': - return this._coreHandler.core.callMethod( - PeripheralDeviceAPI.methods.removePackageInfo, - message.arguments - ) - case 'reportFromMonitorPackages': - this.reportMonitoredPackages(...message.arguments) - break + private reportMonitoredPackages(_containerId: string, monitorId: string, expectedPackages: ExpectedPackage.Any[]) { + const expectedPackagesWraps: ExpectedPackageWrap[] = [] - default: - // @ts-expect-error message is never - throw new Error(`Unsupported message type "${message.type}"`) + for (const expectedPackage of expectedPackages) { + const wrap = wrapExpectedPackage(this.packageManager.packageContainersCache, expectedPackage) + if (wrap) { + expectedPackagesWraps.push(wrap) + } } + + this.logger.debug( + `reportMonitoredPackages: ${expectedPackages.length} packages, ${expectedPackagesWraps.length} wraps` + ) + + this.packageManager.monitoredPackages[monitorId] = expectedPackagesWraps + + this.packageManager.triggerUpdatedExpectedPackages() } - public restartExpectation(workId: string): void { - // This method can be called from core - this._expectationManager.restartExpectation(workId) - } - public restartAllExpectations(): void { - // This method can be called from core - this._expectationManager.restartAllExpectations() - } - public abortExpectation(workId: string): void { - // This method can be called from core - this._expectationManager.abortExpectation(workId) - } - /** Ensures that the packageContainerExpectations containes the mandatory expectations */ - private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { - [id: string]: PackageContainerExpectation - }): void { - for (const [containerId, packageContainer] of Object.entries(this.packageContainersCache)) { - /** Is the Container writeable */ - let isWriteable = false - for (const accessor of Object.values(packageContainer.accessors)) { - if (accessor.allowWrite) { - isWriteable = true - break - } +} +function wrapExpectedPackage( + packageContainers: PackageContainers, + expectedPackage: ExpectedPackage.Any +): ExpectedPackageWrap | undefined { + const combinedSources: PackageContainerOnPackage[] = [] + for (const packageSource of expectedPackage.sources) { + const lookedUpSource: PackageContainer = packageContainers[packageSource.containerId] + if (lookedUpSource) { + // We're going to combine the accessor attributes set on the Package with the ones defined on the source: + const combinedSource: PackageContainerOnPackage = { + ...omit(clone(lookedUpSource), 'accessors'), + accessors: {}, + containerId: packageSource.containerId, } - // All writeable packageContainers should have the clean up cronjob: - if (isWriteable) { - if (!packageContainerExpectations[containerId]) { - // todo: Maybe should not all package-managers monitor, - // this should perhaps be coordinated with the Workforce-manager, who should monitor who? - // Add default packageContainerExpectation: - packageContainerExpectations[containerId] = literal({ - ...packageContainer, - id: containerId, - managerId: this._expectationManager.managerId, - cronjobs: { - interval: 0, - }, - monitors: {}, - }) + const accessorIds = _.uniq( + Object.keys(lookedUpSource.accessors).concat(Object.keys(packageSource.accessors || {})) + ) + + for (const accessorId of accessorIds) { + const sourceAccessor = lookedUpSource.accessors[accessorId] as Accessor.Any | undefined + + const packageAccessor = packageSource.accessors[accessorId] as AccessorOnPackage.Any | undefined + + if (packageAccessor && sourceAccessor && packageAccessor.type === sourceAccessor.type) { + combinedSource.accessors[accessorId] = deepExtend({}, sourceAccessor, packageAccessor) + } else if (packageAccessor) { + combinedSource.accessors[accessorId] = clone(packageAccessor) + } else if (sourceAccessor) { + combinedSource.accessors[accessorId] = clone(sourceAccessor) as AccessorOnPackage.Any } - packageContainerExpectations[containerId].cronjobs.cleanup = {} // Add cronjob to clean up } + combinedSources.push(combinedSource) } } - private reportMonitoredPackages(_containerId: string, monitorId: string, expectedPackages: ExpectedPackage.Any[]) { - const expectedPackagesWraps: ExpectedPackageWrap[] = [] - - for (const expectedPackage of expectedPackages) { - const wrap = this.wrapExpectedPackage(this.packageContainersCache, expectedPackage) - if (wrap) { - expectedPackagesWraps.push(wrap) + // Lookup Package targets: + const combinedTargets: PackageContainerOnPackage[] = [] + + for (const layer of expectedPackage.layers) { + // Hack: we use the layer name as a 1-to-1 relation to a target containerId + const packageContainerId: string = layer + + if (packageContainerId) { + const lookedUpTarget = packageContainers[packageContainerId] + if (lookedUpTarget) { + combinedTargets.push({ + ...omit(clone(lookedUpTarget), 'accessors'), + accessors: lookedUpTarget.accessors as { + [accessorId: string]: AccessorOnPackage.Any + }, + containerId: packageContainerId, + }) } } + } - this.monitoredPackages[monitorId] = expectedPackagesWraps - - this._triggerUpdatedExpectedPackages() + if (combinedSources.length) { + if (combinedTargets.length) { + return { + expectedPackage: expectedPackage, + priority: 999, // lowest priority + sources: combinedSources, + targets: combinedTargets, + playoutDeviceId: '', + external: true, + } + } } -} -export function omit(obj: T, ...props: P[]): Omit { - return _.omit(obj, ...(props as string[])) as any + return undefined } interface ResultingExpectedPackage { @@ -707,6 +777,7 @@ export interface ActiveRundown { } export interface PackageManagerSettings { delayRemoval: number + useTemporaryFilePath: boolean } /** Note: This is based on the Core method updateExpectedPackageWorkStatuses. */ diff --git a/apps/quantel-http-transformer-proxy/app/package.json b/apps/quantel-http-transformer-proxy/app/package.json index 5d4eeef8..fc45a080 100644 --- a/apps/quantel-http-transformer-proxy/app/package.json +++ b/apps/quantel-http-transformer-proxy/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/quantel-http-transformer-proxy.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/quantel-http-transformer-proxy.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js quantel-http-transformer-proxy.exe && node ../../../scripts/copy-natives.js win32-x64", "start": "node dist/index.js", "precommit": "lint-staged" }, @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} \ No newline at end of file +} diff --git a/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js b/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/quantel-http-transformer-proxy/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/quantel-http-transformer-proxy/app/src/index.ts b/apps/quantel-http-transformer-proxy/app/src/index.ts index f9dd7301..c86ac4ee 100644 --- a/apps/quantel-http-transformer-proxy/app/src/index.ts +++ b/apps/quantel-http-transformer-proxy/app/src/index.ts @@ -1,3 +1,4 @@ import { startProcess } from '@quantel-http-transformer-proxy/generic' +/* eslint-disable no-console */ console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/quantel-http-transformer-proxy/packages/generic/package.json b/apps/quantel-http-transformer-proxy/packages/generic/package.json index 3af4be38..3628c244 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/package.json +++ b/apps/quantel-http-transformer-proxy/packages/generic/package.json @@ -53,7 +53,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -63,4 +63,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts b/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts index 3b22e849..467ddbf8 100644 --- a/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts +++ b/apps/quantel-http-transformer-proxy/packages/generic/src/index.ts @@ -1,5 +1,5 @@ import { QuantelHTTPTransformerProxy } from './server' -import { getQuantelHTTPTransformerProxyConfig, setupLogging } from '@shared/api' +import { getQuantelHTTPTransformerProxyConfig, ProcessHandler, setupLogging } from '@shared/api' export { QuantelHTTPTransformerProxy } export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('------------------------------------------------------------------') logger.info('Starting Quantel HTTP Transformer Proxy Server') + const process = new ProcessHandler(logger) + process.init(config.process) + const app = new QuantelHTTPTransformerProxy(logger, config) app.init().catch((e) => { logger.error(e) diff --git a/apps/single-app/app/package.json b/apps/single-app/app/package.json index 3eb66d4a..d849bb9f 100644 --- a/apps/single-app/app/package.json +++ b/apps/single-app/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & node scripts/build-win32.js && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js package-manager-single-app.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -16,6 +16,7 @@ "lint-staged": "^7.2.0" }, "dependencies": { + "@appcontainer-node/generic": "*", "@http-server/generic": "*", "@package-manager/generic": "*", "@quantel-http-transformer-proxy/generic": "*", @@ -29,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/single-app/app/scripts/build-win32.js b/apps/single-app/app/scripts/build-win32.js deleted file mode 100644 index 91c427dc..00000000 --- a/apps/single-app/app/scripts/build-win32.js +++ /dev/null @@ -1,131 +0,0 @@ -const promisify = require("util").promisify -const cp = require('child_process') -const path = require('path') -const nexe = require('nexe') -const exec = promisify(cp.exec) -const glob = promisify(require("glob")) - -const fse = require('fs-extra'); -const mkdirp = require('mkdirp'); -const rimraf = promisify(require('rimraf')) - -const fseCopy = promisify(fse.copy) - -const packageJson = require('../package.json') - -/* - Due to nexe not taking into account the packages in the mono-repo, we're doing a hack, - copying the packages into node_modules, so that nexe will include them. -*/ - -const outputPath = './deploy/' - - -;(async () => { - - const basePath = './' - - // List all Lerna packages: - const list = await exec( 'yarn lerna list -a --json') - const str = list.stdout - .replace(/^\$.*$/gm, '') - .replace(/^Done in.*$/gm, '') - const packages = JSON.parse(str) - - await mkdirp(basePath + 'node_modules') - - // Copy the packages into node_modules: - const copiedFolders = [] - for (const package of packages) { - if (package.name.match(/boilerplate/)) continue - if (package.name.match(packageJson.name)) continue - - console.log(`Copying: ${package.name}`) - const target = basePath + `node_modules/${package.name}` - await fseCopy(package.location, target) - copiedFolders.push(target) - } - - // Remove things that arent used, to reduce file size: - const copiedFiles = [ - ...await glob(`${basePath}node_modules/@*/app/*`), - ...await glob(`${basePath}node_modules/@*/generic/*`), - ] - for (const file of copiedFiles) { - if ( - // Only keep these: - !file.match(/package.json$/) && - !file.match(/node_modules$/) && - !file.match(/dist$/) - ) { - await rimraf(file) - } - } - - await nexe.compile({ - input: './dist/index.js', - output: outputPath + 'package-manager-single-app.exe', - // build: true, //required to use patches - targets: [ - 'windows-x64-12.18.1' - ], - }) - - // Clean after ourselves: - for (const copiedFolder of copiedFolders) { - await rimraf(copiedFolder) - } - - // const basePath = 'C:\\Users/johan/Desktop/New folder/' - // await nexe.compile({ - // input: './dist/index.js', - // output: basePath + 'package-manager-single-app.exe', - // // build: true, //required to use patches - // targets: [ - // 'windows-x64-12.18.1' - // ], - // }) - - // // Copy node_modules: - // const list = await exec( 'yarn lerna list -a --json') - // const str = list.stdout - // // .replace(/.*(\[.*)\nDone in.*/gs, '$1') - // .replace(/^\$.*$/gm, '') - // .replace(/^Done in.*$/gm, '') - // // console.log('str', str) - // const packages = JSON.parse(str) - - // await mkdirp(basePath + 'node_modules') - - - // for (const package of packages) { - // if (package.name.match(/boilerplate/)) continue - - // console.log(`Copying: ${package.name}`) - // await fseCopy(package.location, basePath + `node_modules/${package.name}`) - // } - - // // remove things that arent used: - // const copiedFiles = [ - // ...await glob(`${basePath}node_modules/@*/app/*`), - // ...await glob(`${basePath}node_modules/@*/generic/*`), - // ] - // console.log(copiedFiles) - // for (const file of copiedFiles) { - // if ( - // file.match(/package.json$/) || - // file.match(/node_modules$/) || - // file.match(/dist$/) - // ) { - // // keep it - // } else { - // await rimraf(file) - // } - // } - - - - - -})().catch(console.error) - diff --git a/apps/single-app/app/scripts/copy-natives.js b/apps/single-app/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/single-app/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/single-app/app/src/index.ts b/apps/single-app/app/src/index.ts index 2f606f77..244930d9 100644 --- a/apps/single-app/app/src/index.ts +++ b/apps/single-app/app/src/index.ts @@ -1,68 +1,6 @@ -import * as HTTPServer from '@http-server/generic' -import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/generic' -import * as PackageManager from '@package-manager/generic' -import * as Workforce from '@shared/workforce' -import * as Worker from '@shared/worker' -import { getSingleAppConfig, setupLogging } from '@shared/api' +import { startSingleApp } from './singleApp' +// eslint-disable-next-line no-console console.log('process started') // This is a message all Sofie processes log upon startup - -const config = getSingleAppConfig() -const logger = setupLogging(config) - -;(async function start() { - // Override some of the arguments, as they arent used in the single-app - config.packageManager.port = null - config.packageManager.accessUrl = null - config.packageManager.workforceURL = null - config.workforce.port = null - config.worker.workforceURL = null - - logger.info('------------------------------------------------------------------') - logger.info('Starting Package Manager - Single App') - - logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) - logger.info('------------------------------------------------------------------') - logger.info(JSON.stringify(config, undefined, 2)) - logger.info('------------------------------------------------------------------') - - const connector = new PackageManager.Connector(logger, config) - const expectationManager = connector.getExpectationManager() - - logger.info('Initializing HTTP proxy Server') - const httpServer = new HTTPServer.PackageProxyServer(logger, config) - await httpServer.init() - - logger.info('Initializing Quantel HTTP Transform proxy Server') - const quantelHTTPTransformerProxy = new QuantelHTTPTransformerProxy.QuantelHTTPTransformerProxy(logger, config) - await quantelHTTPTransformerProxy.init() - - logger.info('Initializing Workforce') - const workforce = new Workforce.Workforce(logger, config) - await workforce.init() - - logger.info('Initializing Package Manager (and Expectation Manager)') - expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) - await connector.init() - - const workerAgents: any[] = [] - for (let i = 0; i < config.singleApp.workerCount; i++) { - logger.info('Initializing worker') - const workerAgent = new Worker.WorkerAgent(logger, { - ...config, - worker: { - ...config.worker, - workerId: config.worker.workerId + '_' + i, - }, - }) - workerAgents.push(workerAgent) - - workerAgent.hookToWorkforce(workforce.getWorkerAgentHook()) - workerAgent.hookToExpectationManager(expectationManager.managerId, expectationManager.getWorkerAgentHook()) - await workerAgent.init() - } - - logger.info('------------------------------------------------------------------') - logger.info('Initialization complete') - logger.info('------------------------------------------------------------------') -})().catch(logger.error) +// eslint-disable-next-line no-console +startSingleApp().catch(console.log) diff --git a/apps/single-app/app/src/singleApp.ts b/apps/single-app/app/src/singleApp.ts new file mode 100644 index 00000000..a5abfcf2 --- /dev/null +++ b/apps/single-app/app/src/singleApp.ts @@ -0,0 +1,65 @@ +// import * as ChildProcess from 'child_process' +// import * as path from 'path' + +import * as HTTPServer from '@http-server/generic' +import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/generic' +import * as PackageManager from '@package-manager/generic' +import * as Workforce from '@shared/workforce' +import * as AppConatainerNode from '@appcontainer-node/generic' +import { getSingleAppConfig, ProcessHandler, setupLogging } from '@shared/api' +// import { MessageToAppContainerSpinUp, MessageToAppContainerType } from './__api' + +export async function startSingleApp(): Promise { + const config = getSingleAppConfig() + const logger = setupLogging(config) + // Override some of the arguments, as they arent used in the single-app + config.packageManager.port = 0 // 0 = Set the packageManager port to whatever is available + config.packageManager.accessUrl = 'ws:127.0.0.1' + config.packageManager.workforceURL = null // Filled in later + config.workforce.port = 0 // 0 = Set the workforce port to whatever is available + + const process = new ProcessHandler(logger) + process.init(config.process) + + logger.info('------------------------------------------------------------------') + logger.info('Starting Package Manager - Single App') + + logger.info('Core: ' + config.packageManager.coreHost + ':' + config.packageManager.corePort) + logger.info('------------------------------------------------------------------') + // eslint-disable-next-line no-console + console.log(JSON.stringify(config, undefined, 2)) + logger.info('------------------------------------------------------------------') + + logger.info('Initializing Workforce') + const workforce = new Workforce.Workforce(logger, config) + await workforce.init() + if (!workforce.getPort()) throw new Error(`Internal Error: Got no workforce port`) + const workforceURL = `ws://127.0.0.1:${workforce.getPort()}` + + config.packageManager.workforceURL = workforceURL + config.appContainer.workforceURL = workforceURL + + logger.info('Initializing AppContainer') + const appContainer = new AppConatainerNode.AppContainer(logger, config) + await appContainer.init() + + logger.info('Initializing Package Manager Connector') + const connector = new PackageManager.Connector(logger, config, process) + const expectationManager = connector.getExpectationManager() + + logger.info('Initializing HTTP proxy Server') + const httpServer = new HTTPServer.PackageProxyServer(logger, config) + await httpServer.init() + + logger.info('Initializing Quantel HTTP Transform proxy Server') + const quantelHTTPTransformerProxy = new QuantelHTTPTransformerProxy.QuantelHTTPTransformerProxy(logger, config) + await quantelHTTPTransformerProxy.init() + + logger.info('Initializing Package Manager (and Expectation Manager)') + expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) + await connector.init() + + logger.info('------------------------------------------------------------------') + logger.info('Initialization complete') + logger.info('------------------------------------------------------------------') +} diff --git a/apps/tests/internal-tests/src/__tests__/basic.spec.ts b/apps/tests/internal-tests/src/__tests__/basic.spec.ts deleted file mode 100644 index 11a83779..00000000 --- a/apps/tests/internal-tests/src/__tests__/basic.spec.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { promisify } from 'util' -import { Expectation, literal } from '@shared/api' -import fsOrg from 'fs' -import type * as fsMockType from '../__mocks__/fs' -import WNDOrg from 'windows-network-drive' -import type * as WNDType from '../__mocks__/windows-network-drive' -import * as QGatewayClientOrg from 'tv-automation-quantel-gateway-client' -import type * as QGatewayClientType from '../__mocks__/tv-automation-quantel-gateway-client' -import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' -import { waitSeconds } from './lib/lib' -import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' -import { - getFileShareSource, - getLocalSource, - getLocalTarget, - getQuantelSource, - getQuantelTarget, -} from './lib/containers' -jest.mock('fs') -jest.mock('child_process') -jest.mock('windows-network-drive') -jest.mock('tv-automation-quantel-gateway-client') - -const fs = (fsOrg as any) as typeof fsMockType -const WND = (WNDOrg as any) as typeof WNDType -const QGatewayClient = (QGatewayClientOrg as any) as typeof QGatewayClientType - -const fsStat = promisify(fs.stat) - -const WAIT_JOB_TIME = 1 // seconds - -describe('Basic', () => { - let env: TestEnviromnent - - beforeAll(async () => { - env = await prepareTestEnviromnent(false) // set to true to enable debug-logging - // Verify that the fs mock works: - expect(fs.lstat).toBeTruthy() - expect(fs.__mockReset).toBeTruthy() - }) - afterAll(() => { - env.terminate() - }) - beforeEach(() => { - fs.__mockReset() - env.reset() - QGatewayClient.resetMock() - }) - test( - 'Be able to copy local file', - async () => { - fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) - fs.__mockSetDirectory('/targets/target0') - // console.log(fs.__printAllFiles()) - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.FILE_COPY, - statusReport: { - label: `Copy file0`, - description: `Copy file0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getLocalSource('source0', 'file0Source.mp4')], - }, - endRequirement: { - targets: [getLocalTarget('target0', 'file0Target.mp4')], - content: { - filePath: 'file0Target.mp4', - }, - version: { type: Expectation.Version.Type.FILE_ON_DISK }, - }, - workOptions: {}, - }), - }) - - await waitSeconds(WAIT_JOB_TIME) - - // Expect the copy to have completed by now: - - expect(env.containerStatuses['target0']).toBeTruthy() - expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) - - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - - expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ - size: 1234, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test( - 'Be able to copy Networked file to local', - async () => { - fs.__mockSetFile('\\\\networkShare/sources/source1/file0Source.mp4', 1234) - fs.__mockSetDirectory('/targets/target1') - // console.log(fs.__printAllFiles()) - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.FILE_COPY, - statusReport: { - label: `Copy file0`, - description: `Copy file0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getFileShareSource('source1', 'file0Source.mp4')], - }, - endRequirement: { - targets: [getLocalTarget('target1', 'subFolder0/file0Target.mp4')], - content: { - filePath: 'subFolder0/file0Target.mp4', - }, - version: { type: Expectation.Version.Type.FILE_ON_DISK }, - }, - workOptions: {}, - }), - }) - - await waitSeconds(WAIT_JOB_TIME) - - // Expect the copy to have completed by now: - - // console.log(fs.__printAllFiles()) - - expect(env.containerStatuses['target1']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) - - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - - expect(await WND.list()).toEqual({ - X: '\\\\networkShare\\sources\\source1\\', - }) - - expect(await fsStat('/targets/target1/subFolder0/file0Target.mp4')).toMatchObject({ - size: 1234, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test( - 'Be able to copy Quantel clips', - async () => { - const orgClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123')[0] - - env.expectationManager.updateExpectations({ - copy0: literal({ - id: 'copy0', - priority: 0, - managerId: 'manager0', - fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], - type: Expectation.Type.QUANTEL_CLIP_COPY, - statusReport: { - label: `Copy quantel clip0`, - description: `Copy clip0 because test`, - requiredForPlayout: true, - displayRank: 0, - sendReport: true, - }, - startRequirement: { - sources: [getQuantelSource('source0')], - }, - endRequirement: { - targets: [getQuantelTarget('target1', 1001)], - content: { - guid: 'abc123', - }, - version: { type: Expectation.Version.Type.QUANTEL_CLIP }, - }, - workOptions: {}, - }), - }) - - await waitSeconds(WAIT_JOB_TIME) - - // Expect the copy to have completed by now: - - expect(env.containerStatuses['target1']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() - expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( - ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - ) - - expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') - - const newClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123' && clip !== orgClip.clip)[0] - expect(newClip).toBeTruthy() - - expect(newClip).toMatchObject({ - server: { - ident: 1001, - }, - clip: { - ClipGUID: 'abc123', - CloneId: orgClip.clip.ClipID, - }, - }) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Be able to copy local file to http', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Be able to handle 1000 expectations', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Media file preview from local to file share', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Media file preview from local to file share', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) -}) - -export {} diff --git a/apps/tests/internal-tests/src/__tests__/issues.spec.ts b/apps/tests/internal-tests/src/__tests__/issues.spec.ts deleted file mode 100644 index 6a2a7fc0..00000000 --- a/apps/tests/internal-tests/src/__tests__/issues.spec.ts +++ /dev/null @@ -1,118 +0,0 @@ -// import { promisify } from 'util' -import * as fsOrg from 'fs' -import type * as fsMock from '../__mocks__/fs' -import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' -jest.mock('fs') -jest.mock('child_process') -jest.mock('windows-network-drive') - -const fs = (fsOrg as any) as typeof fsMock - -// const fsStat = promisify(fs.stat) - -const WAIT_JOB_TIME = 1 // seconds - -describe('Handle unhappy paths', () => { - let env: TestEnviromnent - - beforeAll(async () => { - env = await prepareTestEnviromnent(true) - }) - afterAll(() => { - env.terminate() - }) - - beforeEach(() => { - fs.__mockReset() - env.reset() - }) - test.skip( - 'Wait for non-existing local file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for non-existing network-shared, file', - async () => { - // To be written - - // Handle Drive-letters: Issue when when files aren't found? (Johan knows) - - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for growing file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for non-existing file', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for read access on source', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Wait for write access on target', - async () => { - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Expectation changed', - async () => { - // Expectation is changed while waiting for a file - // Expectation is changed while work-in-progress - // Expectation is changed after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Aborting a job', - async () => { - // Expectation is aborted while waiting for a file - // Expectation is aborted while work-in-progress - // Expectation is aborted after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) - test.skip( - 'Restarting a job', - async () => { - // Expectation is restarted while waiting for a file - // Expectation is restarted while work-in-progress - // Expectation is restarted after fullfilled - - // To be written - expect(1).toEqual(1) - }, - WAIT_JOB_TIME * 1000 + 5000 - ) -}) - -export {} diff --git a/apps/tests/internal-tests/src/__tests__/lib/lib.ts b/apps/tests/internal-tests/src/__tests__/lib/lib.ts deleted file mode 100644 index 8b80ba92..00000000 --- a/apps/tests/internal-tests/src/__tests__/lib/lib.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function waitSeconds(seconds: number) { - return new Promise((resolve) => setTimeout(resolve, seconds * 1000)) -} diff --git a/apps/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/apps/tests/internal-tests/src/__tests__/lib/setupEnv.ts deleted file mode 100644 index 6fd95e9d..00000000 --- a/apps/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ /dev/null @@ -1,231 +0,0 @@ -// import * as HTTPServer from '@http-server/generic' -// import * as PackageManager from '@package-manager/generic' -import * as Workforce from '@shared/workforce' -import * as Worker from '@shared/worker' -import * as Winston from 'winston' -import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, SingleAppConfig } from '@shared/api' -// import deepExtend from 'deep-extend' -import { - ExpectationManager, - ReportExpectationStatus, - ReportPackageContainerPackageStatus, - MessageFromWorker, - ReportPackageContainerExpectationStatus, -} from '@shared/expectation-manager' -import { CoreMockAPI } from './coreMockAPI' -import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' - -const defaultTestConfig: SingleAppConfig = { - singleApp: { - workerCount: 1, - }, - process: { - logPath: '', - unsafeSSL: false, - certificates: [], - }, - workforce: { - port: null, - }, - httpServer: { - port: 0, - basePath: '', - apiKeyRead: '', - apiKeyWrite: '', - }, - packageManager: { - coreHost: '', - corePort: 0, - deviceId: '', - deviceToken: '', - disableWatchdog: true, - port: null, - accessUrl: null, - workforceURL: null, - watchFiles: false, - }, - worker: { - workerId: 'worker', - workforceURL: null, - resourceId: '', - networkIds: [], - windowsDriveLetters: ['X', 'Y', 'Z'], - sourcePackageStabilityThreshold: 0, // Disabling this to speed up the tests - }, - quantelHTTPTransformerProxy: { - port: 0, - transformerURL: '', - }, -} - -export async function setupExpectationManager( - debugLogging: boolean, - workerCount: number = 1, - callbacks: { - reportExpectationStatus: ReportExpectationStatus - reportPackageContainerPackageStatus: ReportPackageContainerPackageStatus - reportPackageContainerExpectationStatus: ReportPackageContainerExpectationStatus - messageFromWorker: MessageFromWorker - } -) { - const logger = new Winston.Logger({}) as LoggerInstance - logger.add(Winston.transports.Console, { - level: debugLogging ? 'verbose' : 'warn', - }) - - const expectationManager = new ExpectationManager( - logger, - 'manager0', - { type: 'internal' }, - undefined, - { type: 'internal' }, - callbacks.reportExpectationStatus, - callbacks.reportPackageContainerPackageStatus, - callbacks.reportPackageContainerExpectationStatus, - callbacks.messageFromWorker - ) - - // Initializing HTTP proxy Server: - // const httpServer = new HTTPServer.PackageProxyServer(logger, config) - // await httpServer.init() - - // Initializing Workforce: - const workforce = new Workforce.Workforce(logger, defaultTestConfig) - await workforce.init() - - // Initializing Expectation Manager: - expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) - await expectationManager.init() - - // Initialize workers: - const workerAgents: Worker.WorkerAgent[] = [] - for (let i = 0; i < workerCount; i++) { - const workerAgent = new Worker.WorkerAgent(logger, { - ...defaultTestConfig, - worker: { - ...defaultTestConfig.worker, - workerId: defaultTestConfig.worker.workerId + '_' + i, - }, - }) - workerAgents.push(workerAgent) - - workerAgent.hookToWorkforce(workforce.getWorkerAgentHook()) - workerAgent.hookToExpectationManager(expectationManager.managerId, expectationManager.getWorkerAgentHook()) - await workerAgent.init() - } - - return { - workforce, - workerAgents, - expectationManager, - } -} - -export async function prepareTestEnviromnent(debugLogging: boolean): Promise { - const expectationStatuses: ExpectationStatuses = {} - const containerStatuses: ContainerStatuses = {} - const coreApi = new CoreMockAPI() - - const em = await setupExpectationManager(debugLogging, 1, { - reportExpectationStatus: ( - expectationId: string, - _expectaction: Expectation.Any | null, - actualVersionHash: string | null, - statusInfo: { - status?: string - progress?: number - statusReason?: string - } - ) => { - if (!expectationStatuses[expectationId]) { - expectationStatuses[expectationId] = { - actualVersionHash: null, - statusInfo: {}, - } - } - const o = expectationStatuses[expectationId] - if (actualVersionHash) o.actualVersionHash = actualVersionHash - if (statusInfo.status) o.statusInfo.status = statusInfo.status - if (statusInfo.progress) o.statusInfo.progress = statusInfo.progress - if (statusInfo.statusReason) o.statusInfo.statusReason = statusInfo.statusReason - }, - reportPackageContainerPackageStatus: ( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - ) => { - if (!containerStatuses[containerId]) { - containerStatuses[containerId] = { - packages: {}, - } - } - const container = containerStatuses[containerId] - container.packages[packageId] = { - packageStatus: packageStatus, - } - }, - reportPackageContainerExpectationStatus: () => { - // todo - }, - messageFromWorker: async (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { - switch (message.type) { - case 'fetchPackageInfoMetadata': - return await coreApi.fetchPackageInfoMetadata(...message.arguments) - case 'updatePackageInfo': - return await coreApi.updatePackageInfo(...message.arguments) - case 'removePackageInfo': - return await coreApi.removePackageInfo(...message.arguments) - case 'reportFromMonitorPackages': - return await coreApi.reportFromMonitorPackages(...message.arguments) - default: - // @ts-expect-error message.type is never - throw new Error(`Unsupported message type "${message.type}"`) - } - }, - }) - - return { - expectationManager: em.expectationManager, - coreApi, - expectationStatuses, - containerStatuses, - reset: () => { - Object.keys(expectationStatuses).forEach((id) => delete expectationStatuses[id]) - Object.keys(containerStatuses).forEach((id) => delete containerStatuses[id]) - coreApi.reset() - }, - terminate: () => { - em.expectationManager.terminate() - em.workforce.terminate() - em.workerAgents.forEach((workerAgent) => workerAgent.terminate()) - }, - } -} -export interface TestEnviromnent { - expectationManager: ExpectationManager - coreApi: CoreMockAPI - expectationStatuses: ExpectationStatuses - containerStatuses: ContainerStatuses - reset: () => void - terminate: () => void -} - -export interface ExpectationStatuses { - [expectationId: string]: { - actualVersionHash: string | null - statusInfo: { - status?: string - progress?: number - statusReason?: string - } - } -} -export interface ContainerStatuses { - [containerId: string]: { - packages: { - [packageId: string]: { - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null - } - } - } -} diff --git a/apps/worker/app/package.json b/apps/worker/app/package.json index 40613280..07467154 100644 --- a/apps/worker/app/package.json +++ b/apps/worker/app/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "oldbuild-win32": "mkdir deploy & rimraf deploy/worker.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/worker.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js worker.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -23,7 +24,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/worker/app/scripts/copy-natives.js b/apps/worker/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/worker/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/worker/app/src/index.ts b/apps/worker/app/src/index.ts index 0cff45b5..d018de04 100644 --- a/apps/worker/app/src/index.ts +++ b/apps/worker/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@worker/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/worker/packages/generic/package.json b/apps/worker/packages/generic/package.json index eed5c165..8e546cd1 100644 --- a/apps/worker/packages/generic/package.json +++ b/apps/worker/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/apps/worker/packages/generic/src/index.ts b/apps/worker/packages/generic/src/index.ts index 55d50ad7..58444c44 100644 --- a/apps/worker/packages/generic/src/index.ts +++ b/apps/worker/packages/generic/src/index.ts @@ -1,4 +1,4 @@ -import { getWorkerConfig, setupLogging } from '@shared/api' +import { getWorkerConfig, ProcessHandler, setupLogging } from '@shared/api' import { WorkerAgent } from '@shared/worker' export async function startProcess(): Promise { @@ -10,6 +10,9 @@ export async function startProcess(): Promise { logger.info('Starting Worker') logger.info('------------------------------------------------------------------') + const process = new ProcessHandler(logger) + process.init(config.process) + const workforce = new WorkerAgent(logger, config) workforce.init().catch(logger.error) diff --git a/apps/workforce/app/package.json b/apps/workforce/app/package.json index defc4eab..2f65bc71 100644 --- a/apps/workforce/app/package.json +++ b/apps/workforce/app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "build-win32": "mkdir deploy & rimraf deploy/workforce.exe && nexe dist/index.js -t windows-x64-12.18.1 -o deploy/workforce.exe && node scripts/copy-natives.js win32-x64", + "build-win32": "mkdir deploy & node ../../../scripts/build-win32.js workforce.exe && node ../../../scripts/copy-natives.js win32-x64", "__test": "jest", "start": "node dist/index.js", "precommit": "lint-staged" @@ -23,7 +23,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/workforce/app/scripts/copy-natives.js b/apps/workforce/app/scripts/copy-natives.js deleted file mode 100644 index abbd1186..00000000 --- a/apps/workforce/app/scripts/copy-natives.js +++ /dev/null @@ -1,33 +0,0 @@ -const find = require('find'); -const os = require('os') -const path = require('path') -const fs = require('fs-extra') - -const arch = os.arch() -const platform = os.platform() -const prebuildType = process.argv[2] || `${platform}-${arch}` - -function isFileForPlatform(filename) { - if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { - return true - } else { - return false - } -} - -const dirName = path.join(__dirname, '../../../..') -console.log('Running in', dirName, 'for', prebuildType) - -console.log(process.argv[2]) - -find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { - files.forEach(fullPath => { - if (fullPath.indexOf(dirName) === 0) { - const file = fullPath.substr(dirName.length + 1) - if (isFileForPlatform(file)) { - console.log('Copy prebuild binary:', file) - fs.copySync(file, path.join('deploy', file)) - } - } - }); -}) diff --git a/apps/workforce/app/src/index.ts b/apps/workforce/app/src/index.ts index a6f38469..a54aa429 100644 --- a/apps/workforce/app/src/index.ts +++ b/apps/workforce/app/src/index.ts @@ -1,3 +1,5 @@ import { startProcess } from '@workforce/generic' +/* eslint-disable no-console */ + console.log('process started') // This is a message all Sofie processes log upon startup startProcess().catch(console.error) diff --git a/apps/workforce/packages/generic/package.json b/apps/workforce/packages/generic/package.json index 18778c26..47f21ab9 100644 --- a/apps/workforce/packages/generic/package.json +++ b/apps/workforce/packages/generic/package.json @@ -22,7 +22,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -32,4 +32,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/commonPackage.json b/commonPackage.json index 558b1485..cab2b52b 100644 --- a/commonPackage.json +++ b/commonPackage.json @@ -4,7 +4,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "devDependencies": { "lint-staged": "^7.2.0" diff --git a/lerna.json b/lerna.json index 0a250d68..18048872 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,9 @@ { - "packages": ["shared/**", "apps/**"], + "packages": [ + "shared/**", + "apps/**", + "tests/**" + ], "version": "independent", "npmClient": "yarn", "useWorkspaces": true diff --git a/package.json b/package.json index b8a8f987..81798675 100644 --- a/package.json +++ b/package.json @@ -5,13 +5,15 @@ "private": true, "workspaces": [ "shared/**", - "apps/**" + "apps/**", + "tests/**" ], "scripts": { "ci": "yarn install && yarn build && yarn lint && yarn test", "setup": "lerna bootstrap", + "reset": "node scripts/reset.js", "build": "lerna run build --stream", - "build:changed": "lerna run --since origin/master --include-dependents build", + "build:changed": "lerna run build --since head --exclude-dependents --stream", "lint": "lerna exec -- eslint . --ext .ts,.tsx", "lintfix": "yarn lint --fix", "lint:changed": "lerna exec --since origin/master --include-dependents -- eslint . --ext .js,.jsx,.ts,.tsx", @@ -21,7 +23,7 @@ "test:update:changed": "lerna run --since origin/master --include-dependents test -- -u", "typecheck": "lerna exec -- tsc --noEmit", "typecheck:changed": "lerna exec --since origin/master --include-dependents -- tsc --noEmit", - "build-win32": "lerna run build-win32", + "build-win32": "node scripts/run-sequencially.js lerna run build-win32 --stream", "gather-built": "node scripts/gather-all-built.js", "start:http-server": "lerna run start --stream --scope @http-server/app", "start:workforce": "lerna run start --stream --scope @workforce/app", @@ -52,11 +54,11 @@ "fs-extra": "^9.1.0" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.11.0" }, "dependencies": { - "@sofie-automation/blueprints-integration": "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0", - "@sofie-automation/server-core-integration": "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" + "@sofie-automation/blueprints-integration": "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0", + "@sofie-automation/server-core-integration": "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "husky": { diff --git a/scripts/build-win32.js b/scripts/build-win32.js new file mode 100644 index 00000000..a6178274 --- /dev/null +++ b/scripts/build-win32.js @@ -0,0 +1,103 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const cp = require('child_process') +const path = require('path') +const nexe = require('nexe') +const exec = promisify(cp.exec) +const glob = promisify(require('glob')) + +const fse = require('fs-extra'); +const mkdirp = require('mkdirp'); +const rimraf = promisify(require('rimraf')) + +const fseCopy = promisify(fse.copy) + +/* + Due to nexe not taking into account the packages in the mono-repo, we're doing a hack, + copying the packages into node_modules, so that nexe will include them. +*/ +const basePath = process.cwd() +const packageJson = require(path.join(basePath, '/package.json')) +const outputDirectory = path.join(basePath, './deploy/') +const executableName = process.argv[2] +if (!executableName) { + throw new Error(`Argument for the output executable file name not provided`) +} + +;(async () => { + + log(`Collecting dependencies for ${packageJson.name}...`) + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') + + const packages = JSON.parse(str) + + await mkdirp(basePath + 'node_modules') + + // Copy the packages into node_modules: + const copiedFolders = [] + let ps = [] + for (const package0 of packages) { + if (package0.name.match(/boilerplate/)) continue + if (package0.name.match(packageJson.name)) continue + + log(` Copying: ${package0.name}`) + const target = path.resolve(path.join(basePath, 'node_modules', package0.name)) + + // log(` ${package0.location} -> ${target}`) + ps.push(fseCopy(package0.location, target)) + + copiedFolders.push(target) + } + + await Promise.all(ps) + ps = [] + + // Remove things that arent used, to reduce file size: + log(`Remove unused files...`) + const copiedFiles = [ + ...(await glob(`${basePath}node_modules/@*/app/*`)), + ...(await glob(`${basePath}node_modules/@*/generic/*`)), + ] + + for (const file of copiedFiles) { + if ( + // Only keep these: + !file.match(/package0.json$/) && + !file.match(/node_modules$/) && + !file.match(/dist$/) + ) { + ps.push(rimraf(file)) + } + } + await Promise.all(ps) + ps = [] + + log(`Compiling using nexe...`) + + const nexeOutputPath = path.join(outputDirectory, executableName) + + console.log('nexeOutputPath', nexeOutputPath) + + await nexe.compile({ + input: path.join(basePath, './dist/index.js'), + output: nexeOutputPath, + // build: true, //required to use patches + targets: ['windows-x64-12.18.1'], + }) + + log(`Cleaning up...`) + // Clean up after ourselves: + for (const copiedFolder of copiedFolders) { + await rimraf(copiedFolder) + } + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/scripts/copy-natives.js b/scripts/copy-natives.js new file mode 100644 index 00000000..3ddad6d8 --- /dev/null +++ b/scripts/copy-natives.js @@ -0,0 +1,44 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const find = require('find') +const os = require('os') +const path = require('path') +const fs = require('fs-extra') + +const arch = os.arch() +const platform = os.platform() +const prebuildType = process.argv[2] || `${platform}-${arch}` + +log('Copying native dependencies...') + +const basePath = process.cwd() + +const dirName = path.join(basePath, '../../..') +log(`Running in directory ${dirName} for patform "${prebuildType}"`) + +// log(process.argv[2]) + +find.file(/\.node$/, path.join(dirName, 'node_modules'), (files) => { + files.forEach((fullPath) => { + if (fullPath.indexOf(dirName) === 0) { + const file = fullPath.substr(dirName.length + 1) + if (isFileForPlatform(file)) { + log('Copy prebuild binary:', file) + fs.copySync(file, path.join('deploy', file)) + } + } + }) +}) + +function isFileForPlatform(filename) { + if (filename.indexOf(path.join('prebuilds', prebuildType)) !== -1) { + return true + } else { + return false + } +} + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/scripts/gather-all-built.js b/scripts/gather-all-built.js index 1cb6383c..1763ecc2 100644 --- a/scripts/gather-all-built.js +++ b/scripts/gather-all-built.js @@ -2,9 +2,12 @@ const promisify = require("util").promisify const glob = promisify(require("glob")) const fse = require('fs-extra'); const mkdirp = require('mkdirp'); -const rimraf = promisify(require('rimraf')) +const path = require('path'); +// const rimraf = promisify(require('rimraf')) const fseCopy = promisify(fse.copy) +const fseReaddir = promisify(fse.readdir) +const fseUnlink = promisify(fse.unlink) /* This script gathers all files in the deploy/ folders of the various apps @@ -15,20 +18,26 @@ const targetFolder = 'deploy/' ;(async function () { - await rimraf(targetFolder) - await mkdirp(targetFolder) + // await rimraf(targetFolder) + await mkdirp(targetFolder) + // clear folder: + const files = await fseReaddir(targetFolder) + for (const file of files) { + if (!file.match(/ffmpeg|ffprobe/)) { + await fseUnlink(path.join(targetFolder, file)) + } + } - const deployfolders = await glob(`apps/*/app/deploy`) + const deployfolders = await glob(`apps/*/app/deploy`) + for (const deployfolder of deployfolders) { - for (const deployfolder of deployfolders) { + if (deployfolder.match(/boilerplate/)) continue - if (deployfolder.match(/boilerplate/)) continue + console.log(`Copying: ${deployfolder}`) + await fseCopy(deployfolder, targetFolder) + } - console.log(`Copying: ${deployfolder}`) - await fseCopy(deployfolder, targetFolder) - } - - console.log(`All files have been copied to: ${targetFolder}`) + console.log(`All files have been copied to: ${targetFolder}`) })().catch(console.error) diff --git a/scripts/reset.js b/scripts/reset.js new file mode 100644 index 00000000..e2565a85 --- /dev/null +++ b/scripts/reset.js @@ -0,0 +1,63 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const glob = promisify(require('glob')) +const rimraf = promisify(require('rimraf')) + +/* + Removing all /node_modules and /dist folders +*/ + +const basePath = process.cwd() + +;(async () => { + + log('Gathering files to remove...') + + // Remove things that arent used, to reduce file size: + const searchForFolder = async (name) => { + return [ + ...(await glob(`${basePath}/${name}`)), + ...(await glob(`${basePath}/*/${name}`)), + ...(await glob(`${basePath}/*/*/${name}`)), + ...(await glob(`${basePath}/*/*/*/${name}`)), + ...(await glob(`${basePath}/*/*/*/*/${name}`)), + ] + } + const allFolders = [ + ...(await searchForFolder('node_modules')), + ...(await searchForFolder('dist')), + ...(await searchForFolder('deploy')), + ] + const resultingFolders = [] + for (const folder of allFolders) { + let found = false + for (const resultingFolder of resultingFolders) { + if (folder.match(resultingFolder)) { + found = true + break + } + } + if (!found) { + resultingFolders.push(folder) + } + } + + const rootNodeModules = await glob(`${basePath}/node_modules`) + if (rootNodeModules.length !== 1) throw new Error(`Wrong length of root node_modules (${rootNodeModule.length})`) + const rootNodeModule = rootNodeModules[0] + + log(`Removing ${resultingFolders.length} files...`) + for (const folder of resultingFolders) { + if (folder === rootNodeModule) continue + log(`Removing ${folder}`) + await rimraf(folder) + } + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/scripts/run-sequencially.js b/scripts/run-sequencially.js new file mode 100644 index 00000000..8e9a5776 --- /dev/null +++ b/scripts/run-sequencially.js @@ -0,0 +1,84 @@ +/* eslint-disable node/no-unpublished-require, node/no-extraneous-require */ + +const promisify = require('util').promisify +const cp = require('child_process') +const path = require('path') +// const nexe = require('nexe') +const exec = promisify(cp.exec) +// const spawn = promisify(cp.spawn) +const glob = promisify(require('glob')) + +const fse = require('fs-extra'); +const mkdirp = require('mkdirp'); +const rimraf = promisify(require('rimraf')) + +const fseCopy = promisify(fse.copy) + +/* + Runs a command in all lerna packages, one at a time +*/ +let commands = process.argv.slice(2) +const lastCommand = commands[commands.length-1] + +;(async () => { + + let scopes = [] + if (lastCommand.match(/scope/)) { + + log(`Running command "${commands.join(' ')}" for ${lastCommand}`) + + scopes = [lastCommand] + commands = commands.slice(0, -1) + } else { + + log(`Running command "${commands.join(' ')}" in all packages...`) + + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout.replace(/^\$.*$/gm, '').replace(/^Done in.*$/gm, '') + + const packages = JSON.parse(str) + + scopes = packages.map((p) => `--scope=${p.name}`) + } + + for (const scope of scopes) { + const cmd = `${commands.join(' ')} ${scope}` + log(cmd) + + await new Promise((resolve, reject) => { + const process = cp.exec(cmd, {}) + // const process = cp.spawn(commands[0], [commands.slice(1), `--scope=${package.name}`] ) + process.stdout.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.stderr.on('data', (data) => { + log((data+'').trimEnd() ) + }) + process.on('error', (error) => { + reject(error) + }) + process.on('close', (code) => { + if (code === 0) { + resolve() + } else { + reject('Process exited with code '+code) + } + }) + + }) + + + + + } + // log(packages) + + + log(`...done!`) +})().catch(log) + +function log(...args) { + // eslint-disable-next-line no-console + console.log(...args) +} diff --git a/shared/packages/api/package.json b/shared/packages/api/package.json index 957f1f68..2f4776a1 100644 --- a/shared/packages/api/package.json +++ b/shared/packages/api/package.json @@ -29,7 +29,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -39,4 +39,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/api/src/adapterClient.ts b/shared/packages/api/src/adapterClient.ts index d95ed6b3..bdd4f86c 100644 --- a/shared/packages/api/src/adapterClient.ts +++ b/shared/packages/api/src/adapterClient.ts @@ -1,3 +1,4 @@ +import { LoggerInstance } from './logger' import { WebsocketClient } from './websocketClient' import { Hook, MessageBase, MessageIdentifyClient } from './websocketConnection' @@ -14,13 +15,15 @@ export abstract class AdapterClient { throw new Error('.init() must be called first!') } - constructor(private clientType: MessageIdentifyClient['clientType']) {} + constructor(protected logger: LoggerInstance, private clientType: MessageIdentifyClient['clientType']) {} private conn?: WebsocketClient + private terminated = false async init(id: string, connectionOptions: ClientConnectionOptions, clientMethods: ME): Promise { if (connectionOptions.type === 'websocket') { const conn = new WebsocketClient( + this.logger, id, connectionOptions.url, this.clientType, @@ -28,7 +31,7 @@ export abstract class AdapterClient { // On message from other party: const fcn = (clientMethods as any)[message.type] if (fcn) { - return fcn(...message.args) + return fcn.call(clientMethods, ...message.args) } else { throw new Error(`Unknown method "${message.type}"`) } @@ -37,21 +40,22 @@ export abstract class AdapterClient { this.conn = conn conn.on('connected', () => { - console.log('Websocket client connected') + this.logger.debug('Websocket client connected') }) conn.on('disconnected', () => { - console.log('Websocket client disconnected') + this.logger.debug('Websocket client disconnected') }) this._sendMessage = ((type: string, ...args: any[]) => conn.send(type, ...args)) as any await conn.connect() } else { - // TODO if (!this.serverHook) throw new Error(`AdapterClient: can't init() an internal connection, call hook() first!`) const serverHook: OTHER = this.serverHook(id, clientMethods) this._sendMessage = (type: keyof OTHER, ...args: any[]) => { + if (this.terminated) throw new Error(`Can't send message due to being terminated`) + const fcn = serverHook[type] as any if (fcn) { return fcn(...args) @@ -66,7 +70,9 @@ export abstract class AdapterClient { this.serverHook = serverHook } terminate(): void { + this.terminated = true this.conn?.close() + delete this.serverHook } } /** Options for an AdepterClient */ diff --git a/shared/packages/api/src/adapterServer.ts b/shared/packages/api/src/adapterServer.ts index 53824c5d..f2bc2de6 100644 --- a/shared/packages/api/src/adapterServer.ts +++ b/shared/packages/api/src/adapterServer.ts @@ -1,4 +1,5 @@ -import { MessageBase } from './websocketConnection' +import { promiseTimeout } from './lib' +import { MessageBase, MESSAGE_TIMEOUT } from './websocketConnection' import { ClientConnection } from './websocketServer' /** @@ -9,7 +10,11 @@ import { ClientConnection } from './websocketServer' export abstract class AdapterServer { protected _sendMessage: (type: keyof OTHER, ...args: any[]) => Promise + public readonly type: string + constructor(serverMethods: ME, options: AdapterServerOptions) { + this.type = options.type + if (options.type === 'websocket') { this._sendMessage = ((type: string, ...args: any[]) => options.clientConnection.send(type, ...args)) as any @@ -24,10 +29,14 @@ export abstract class AdapterServer { }) } else { const clientHook: OTHER = options.hookMethods - this._sendMessage = (type: keyof OTHER, ...args: any[]) => { + this._sendMessage = async (type: keyof OTHER, ...args: any[]) => { const fcn = (clientHook[type] as unknown) as (...args: any[]) => any if (fcn) { - return fcn(...args) + try { + return await promiseTimeout(fcn(...args), MESSAGE_TIMEOUT) + } catch (err) { + throw new Error(`Error in message "${type}": ${err.toString()}`) + } } else { throw new Error(`Unknown method "${type}"`) } diff --git a/shared/packages/api/src/appContainer.ts b/shared/packages/api/src/appContainer.ts new file mode 100644 index 00000000..ad335ae1 --- /dev/null +++ b/shared/packages/api/src/appContainer.ts @@ -0,0 +1,57 @@ +import { WorkerAgentConfig } from './worker' + +/** The AppContainer is a host application responsible for spawning other applications */ + +/** How often the appContainer expect to be pinged by its child apps */ +export const APPCONTAINER_PING_TIME = 5000 // ms + +export interface AppContainerConfig { + workforceURL: string | null + port: number | null + appContainerId: string + minRunningApps: number + maxRunningApps: number + spinDownTime: number + + resourceId: string + networkIds: string[] + + windowsDriveLetters: WorkerAgentConfig['windowsDriveLetters'] +} + +// eslint-disable-next-line @typescript-eslint/no-namespace +export namespace AppContainer { + export type AppType = 'worker' // | other + + export enum Type { + NODEJS = 'nodejs', + // DOCKER = 'docker', + // KUBERNETES = 'kubernetes', + } + + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Generic { + /** Information on how to access the AppContainer */ + export interface AppContainer { + type: Type + } + + /** Information about an App running in an AppContainer */ + export interface App { + /** Uniquely identifies a running instance of an app. */ + appId: string + } + } + + /** NodeJS app container */ + // eslint-disable-next-line @typescript-eslint/no-namespace + export namespace NodeJS { + export interface AppContainer extends Generic.AppContainer { + /** URL to the REST interface */ + url: string + } + export interface App extends Generic.App { + type: string // to be better defined later? + } + } +} diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 01af456b..2a2cfca0 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -2,6 +2,7 @@ import { Options } from 'yargs' import yargs = require('yargs/yargs') import _ from 'underscore' import { WorkerAgentConfig } from './worker' +import { AppContainerConfig } from './appContainer' /* * This file contains various CLI argument definitions, used by the various processes that together constitutes the Package Manager @@ -108,6 +109,11 @@ const workerArguments = defineArguments({ default: process.env.WORKFORCE_URL || 'ws://localhost:8070', describe: 'The URL to the Workforce', }, + appContainerURL: { + type: 'string', + default: process.env.APP_CONTAINER_URL || '', // 'ws://localhost:8090', + describe: 'The URL to the AppContainer', + }, windowsDriveLetters: { type: 'string', default: process.env.WORKER_WINDOWS_DRIVE_LETTERS || 'X;Y;Z', @@ -124,6 +130,55 @@ const workerArguments = defineArguments({ describe: 'Identifier of the local networks this worker has access to ("networkA;networkB")', }, }) +/** CLI-argument-definitions for the AppContainer process */ +const appContainerArguments = defineArguments({ + appContainerId: { + type: 'string', + default: process.env.APP_CONTAINER_ID || 'appContainer0', + describe: 'Unique id of the appContainer', + }, + workforceURL: { + type: 'string', + default: process.env.WORKFORCE_URL || 'ws://localhost:8070', + describe: 'The URL to the Workforce', + }, + port: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_PORT || '', 10) || 8090, + describe: 'The port number to start the App Container websocket server on', + }, + maxRunningApps: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_MAX_RUNNING_APPS || '', 10) || 3, + describe: 'How many apps the appContainer can run at the same time', + }, + minRunningApps: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_MIN_RUNNING_APPS || '', 10) || 0, + describe: 'Minimum amount of apps (of a certain appType) to be running', + }, + spinDownTime: { + type: 'number', + default: parseInt(process.env.APP_CONTAINER_SPIN_DOWN_TIME || '', 10) || 60 * 1000, + describe: 'How long a Worker should stay idle before attempting to be spun down', + }, + + resourceId: { + type: 'string', + default: process.env.WORKER_NETWORK_ID || 'default', + describe: 'Identifier of the local resource/computer this worker runs on', + }, + networkIds: { + type: 'string', + default: process.env.WORKER_NETWORK_ID || 'default', + describe: 'Identifier of the local networks this worker has access to ("networkA;networkB")', + }, + windowsDriveLetters: { + type: 'string', + default: process.env.WORKER_WINDOWS_DRIVE_LETTERS || 'X;Y;Z', + describe: 'Which Windows Drive letters can be used to map shares. ("X;Y;Z") ', + }, +}) /** CLI-argument-definitions for the "Single" process */ const singleAppArguments = defineArguments({ workerCount: { @@ -199,7 +254,7 @@ export function getHTTPServerConfig(): HTTPServerConfig { }).argv if (!argv.apiKeyWrite && argv.apiKeyRead) { - throw `Error: When apiKeyRead is given, apiKeyWrite is required!` + throw new Error(`Error: When apiKeyRead is given, apiKeyWrite is required!`) } return { @@ -257,6 +312,7 @@ export interface WorkerConfig { process: ProcessConfig worker: { workforceURL: string | null + appContainerURL: string | null resourceId: string networkIds: string[] } & WorkerAgentConfig @@ -272,12 +328,40 @@ export function getWorkerConfig(): WorkerConfig { worker: { workerId: argv.workerId, workforceURL: argv.workforceURL, + appContainerURL: argv.appContainerURL, windowsDriveLetters: argv.windowsDriveLetters ? argv.windowsDriveLetters.split(';') : [], resourceId: argv.resourceId, networkIds: argv.networkIds ? argv.networkIds.split(';') : [], }, } } +// Configuration for the AppContainer Application: ------------------------------ +export interface AppContainerProcessConfig { + process: ProcessConfig + appContainer: AppContainerConfig +} +export function getAppContainerConfig(): AppContainerProcessConfig { + const argv = yargs(process.argv.slice(2)).options({ + ...appContainerArguments, + ...processOptions, + }).argv + + return { + process: getProcessConfig(argv), + appContainer: { + workforceURL: argv.workforceURL, + port: argv.port, + appContainerId: argv.appContainerId, + maxRunningApps: argv.maxRunningApps, + minRunningApps: argv.minRunningApps, + spinDownTime: argv.spinDownTime, + + resourceId: argv.resourceId, + networkIds: argv.networkIds ? argv.networkIds.split(';') : [], + windowsDriveLetters: argv.windowsDriveLetters ? argv.windowsDriveLetters.split(';') : [], + }, + } +} // Configuration for the Single-app Application: ------------------------------ export interface SingleAppConfig @@ -285,6 +369,7 @@ export interface SingleAppConfig HTTPServerConfig, PackageManagerConfig, WorkerConfig, + AppContainerProcessConfig, QuantelHTTPTransformerProxyConfig { singleApp: { workerCount: number @@ -299,6 +384,7 @@ export function getSingleAppConfig(): SingleAppConfig { ...workerArguments, ...processOptions, ...singleAppArguments, + ...appContainerArguments, ...quantelHTTPTransformerProxyConfigArguments, } // Remove some that are not used in the Single-App, so that they won't show up when running '--help': @@ -323,6 +409,7 @@ export function getSingleAppConfig(): SingleAppConfig { singleApp: { workerCount: argv.workerCount || 1, }, + appContainer: getAppContainerConfig().appContainer, quantelHTTPTransformerProxy: getQuantelHTTPTransformerProxyConfig().quantelHTTPTransformerProxy, } } diff --git a/shared/packages/api/src/expectationApi.ts b/shared/packages/api/src/expectationApi.ts index 1fa0b08b..d4ca2a9d 100644 --- a/shared/packages/api/src/expectationApi.ts +++ b/shared/packages/api/src/expectationApi.ts @@ -23,6 +23,7 @@ export namespace Expectation { // | QuantelClipDeepScan | QuantelClipThumbnail | QuantelClipPreview + | JsonDataCopy /** Defines the Expectation type, used to separate the different Expectations */ export enum Type { @@ -38,6 +39,8 @@ export namespace Expectation { // QUANTEL_CLIP_DEEP_SCAN = 'quantel_clip_deep_scan', QUANTEL_CLIP_THUMBNAIL = 'quantel_clip_thumbnail', QUANTEL_CLIP_PREVIEW = 'quantel_clip_preview', + + JSON_DATA_COPY = 'json_data_copy', } /** Common attributes of all Expectations */ @@ -48,7 +51,7 @@ export namespace Expectation { /** Id of the ExpectationManager the expectation was created from */ managerId: string - /** Expectation priority. Lower will be handled first */ + /** Expectation priority. Lower will be handled first. Note: This is not absolute, the actual execution order might vary. */ priority: number /** A list of which expectedPackages that resultet in this expectation */ @@ -101,7 +104,7 @@ export namespace Expectation { } version: Version.ExpectedFileOnDisk } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Scan of a Media file. A Scan is to be performed on (one of) the sources and the scan result is to be stored on the target. */ export interface PackageScan extends Base { @@ -177,7 +180,7 @@ export namespace Expectation { } version: Version.ExpectedMediaFileThumbnail } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Preview of a Media file. A Preview is to be created from one of the the sources and the resulting file is to be stored on the target. */ export interface MediaFilePreview extends Base { @@ -195,7 +198,7 @@ export namespace Expectation { } version: Version.ExpectedMediaFilePreview } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Quantel clip. A Quantel clip is to be copied from one of the Sources, to the Target. */ @@ -231,7 +234,7 @@ export namespace Expectation { } version: Version.ExpectedQuantelClipThumbnail } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Defines a Preview of a Quantel Clip. A Preview is to be created from one of the the sources and the resulting file is to be stored on the target. */ export interface QuantelClipPreview extends Base { @@ -249,7 +252,23 @@ export namespace Expectation { } version: Version.ExpectedQuantelClipPreview } - workOptions: WorkOptions.RemoveDelay + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath + } + /** Defines a File Copy. A File is to be copied from one of the Sources, to the Target. */ + export interface JsonDataCopy extends Base { + type: Type.JSON_DATA_COPY + + startRequirement: { + sources: SpecificPackageContainerOnPackage.File[] + } + endRequirement: { + targets: [SpecificPackageContainerOnPackage.File] + content: { + path: string + } + version: Version.ExpectedFileOnDisk // maybe something else? + } + workOptions: WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath } /** Contains definitions of specific PackageContainer types, used in the Expectation-definitions */ @@ -262,6 +281,7 @@ export namespace Expectation { | AccessorOnPackage.LocalFolder | AccessorOnPackage.FileShare | AccessorOnPackage.HTTP + | AccessorOnPackage.HTTPProxy } } /** Defines a PackageContainer for CorePackage (A collection in Sofie-Core accessible through an API). */ @@ -284,6 +304,10 @@ export namespace Expectation { /** When removing, wait a duration of time before actually removing it (milliseconds). If not set, package is removed right away. */ removeDelay?: number } + export interface UseTemporaryFilePath { + /** When set, will work on a temporary package first, then move the package to the right place */ + useTemporaryFilePath?: boolean + } } /** Version defines properties to use for determining the version of a Package */ diff --git a/shared/packages/api/src/index.ts b/shared/packages/api/src/index.ts index 130f7462..549b136d 100644 --- a/shared/packages/api/src/index.ts +++ b/shared/packages/api/src/index.ts @@ -1,11 +1,14 @@ export * from './adapterClient' export * from './adapterServer' +export * from './appContainer' export * from './config' export * from './expectationApi' export * from './lib' export * from './logger' export * from './methods' export * from './packageContainerApi' +export * from './process' +export * from './status' export * from './websocketClient' export { MessageBase, MessageReply, MessageIdentifyClient, Hook } from './websocketConnection' export * from './websocketServer' diff --git a/shared/packages/api/src/lib.ts b/shared/packages/api/src/lib.ts index fdbcb953..5b2d4029 100644 --- a/shared/packages/api/src/lib.ts +++ b/shared/packages/api/src/lib.ts @@ -40,3 +40,25 @@ export function hash(str: string): string { const hash0 = crypto.createHash('sha1') return hash0.update(str).digest('hex') } +/** Helper function to simply assert that the value is of the type never */ +export function assertNever(_value: never): void { + // does nothing +} +export function waitTime(duration: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, duration) + }) +} +export function promiseTimeout(p: Promise, timeoutTime: number): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject('Timeout') + }, timeoutTime) + + p.then(resolve) + .catch(reject) + .finally(() => { + clearTimeout(timeout) + }) + }) +} diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts index 083e131f..51995eb5 100644 --- a/shared/packages/api/src/logger.ts +++ b/shared/packages/api/src/logger.ts @@ -43,20 +43,21 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance json: true, stringify: (obj) => { obj.localTimestamp = getCurrentTime() - obj.randomId = Math.round(Math.random() * 10000) + // obj.randomId = Math.round(Math.random() * 10000) return JSON.stringify(obj) // make single line }, }) logger.info('Logging to Console') // Hijack console.log: - console.log = function (...args: any[]) { - // orgConsoleLog('a') - if (args.length >= 1) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore one or more arguments - logger.debug(...args) - } - } + // eslint-disable-next-line no-console + // console.log = function (...args: any[]) { + // // orgConsoleLog('a') + // if (args.length >= 1) { + // // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // // @ts-ignore one or more arguments + // logger.debug(...args) + // } + // } } function getCurrentTime() { const v = Date.now() @@ -78,3 +79,7 @@ export function setupLogging(config: { process: ProcessConfig }): LoggerInstance }) return logger } +export enum LogLevel { + INFO = 'info', + DEBUG = 'debug', +} diff --git a/shared/packages/api/src/methods.ts b/shared/packages/api/src/methods.ts index 5069a347..7f90484f 100644 --- a/shared/packages/api/src/methods.ts +++ b/shared/packages/api/src/methods.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-namespace */ -import { ExpectedPackage } from '@sofie-automation/blueprints-integration' +import { ExpectedPackage, ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { Expectation } from './expectationApi' import { PackageContainerExpectation } from './packageContainerApi' import { @@ -12,7 +12,11 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, } from './worker' +import { WorkforceStatus } from './status' +import { LogLevel } from './logger' +/** Contains textual descriptions for statuses. */ +export type Reason = ExpectedPackageStatusAPI.Reason /* * This file contains API definitions for the methods used to communicate between the Workforce, Worker and Expectation-Manager. */ @@ -21,17 +25,30 @@ import { export namespace WorkForceExpectationManager { /** Methods on WorkForce, called by ExpectationManager */ export interface WorkForce { + setLogLevel: (logLevel: LogLevel) => Promise + setLogLevelOfApp: (appId: string, logLevel: LogLevel) => Promise + _debugKillApp(appId: string): Promise + getStatus: () => Promise + + requestResources: (exp: Expectation.Any) => Promise + registerExpectationManager: (managerId: string, url: string) => Promise } /** Methods on ExpectationManager, called by WorkForce */ // eslint-disable-next-line @typescript-eslint/no-empty-interface - export interface ExpectationManager {} + export interface ExpectationManager { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + } } /** Methods used by WorkForce and WorkerAgent */ export namespace WorkForceWorkerAgent { /** Methods on WorkerAgent, called by WorkForce */ export interface WorkerAgent { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + expectationManagerAvailable: (id: string, url: string) => Promise expectationManagerGone: (id: string) => Promise } @@ -79,8 +96,8 @@ export namespace ExpectationManagerWorkerAgent { // Events emitted from a workInProgress: wipEventProgress: (wipId: number, actualVersionHash: string | null, progress: number) => Promise - wipEventDone: (wipId: number, actualVersionHash: string, reason: string, result: any) => Promise - wipEventError: (wipId: number, error: string) => Promise + wipEventDone: (wipId: number, actualVersionHash: string, reason: Reason, result: any) => Promise + wipEventError: (wipId: number, reason: Reason) => Promise } export interface WorkInProgressInfo { wipId: number @@ -147,3 +164,39 @@ export namespace ExpectationManagerWorkerAgent { result?: any } } +/** Methods used by WorkForce and AppContainer */ +export namespace WorkForceAppContainer { + /** Methods on AppContainer, called by WorkForce */ + export interface AppContainer { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + + requestAppTypeForExpectation: (exp: Expectation.Any) => Promise<{ appType: string; cost: number } | null> + spinUp: ( + appType: 'worker' // | other + ) => Promise + spinDown: (appId: string) => Promise + getRunningApps: () => Promise<{ appId: string; appType: string }[]> + } + /** Methods on WorkForce, called by AppContainer */ + export interface WorkForce { + registerAvailableApps: (availableApps: { appType: string }[]) => Promise + } +} + +/** Methods used by AppContainer and WorkerAgent */ +export namespace AppContainerWorkerAgent { + /** Methods on WorkerAgent, called by AppContainer */ + export interface WorkerAgent { + setLogLevel: (logLevel: LogLevel) => Promise + _debugKill: () => Promise + + doYouSupportExpectation: (exp: Expectation.Any) => Promise + setSpinDownTime: (spinDownTime: number) => Promise + } + /** Methods on AppContainer, called by WorkerAgent */ + export interface AppContainer { + ping: () => Promise + requestSpinDown: () => Promise + } +} diff --git a/shared/packages/api/src/packageContainerApi.ts b/shared/packages/api/src/packageContainerApi.ts index d3194c48..31f25a87 100644 --- a/shared/packages/api/src/packageContainerApi.ts +++ b/shared/packages/api/src/packageContainerApi.ts @@ -23,6 +23,11 @@ export interface PackageContainerExpectation extends PackageContainer { /** If set, ignore any files matching this. (Regular expression). */ ignore?: string + /** If set, the monitoring will be using polling */ + usePolling?: number | null + /** If set, will set the awaitWriteFinish.StabilityThreshold of chokidar */ + awaitWriteFinishStabilityThreshold?: number | null + /** What layers to set on the resulting ExpectedPackage */ targetLayers: string[] diff --git a/apps/package-manager/packages/generic/src/process.ts b/shared/packages/api/src/process.ts similarity index 85% rename from apps/package-manager/packages/generic/src/process.ts rename to shared/packages/api/src/process.ts index c9dc53b4..9df051b2 100644 --- a/apps/package-manager/packages/generic/src/process.ts +++ b/shared/packages/api/src/process.ts @@ -1,6 +1,8 @@ -import { LoggerInstance } from '@shared/api' +import { ProcessConfig } from './config' import fs from 'fs' -import { ProcessConfig } from './connector' +import { LoggerInstance } from './logger' + +// export function setupProcess(config: ProcessConfig): void {} export class ProcessHandler { logger: LoggerInstance diff --git a/shared/packages/api/src/status.ts b/shared/packages/api/src/status.ts new file mode 100644 index 00000000..9fdcd86d --- /dev/null +++ b/shared/packages/api/src/status.ts @@ -0,0 +1,37 @@ +export interface WorkforceStatus { + workerAgents: { + id: string + }[] + expectationManagers: { + id: string + url?: string + }[] + appContainers: { + id: string + initialized: boolean + + availableApps: { + appType: string + }[] + }[] +} +export interface ExpectationManagerStatus { + id: string + expectationStatistics: { + countTotal: number + + countNew: number + countWaiting: number + countReady: number + countWorking: number + countFulfilled: number + countRemoved: number + countRestarted: number + countAborted: number + countNoAvailableWorkers: number + countError: number + } + workerAgents: { + workerId: string + }[] +} diff --git a/shared/packages/api/src/websocketClient.ts b/shared/packages/api/src/websocketClient.ts index 90aa9bcc..907e66aa 100644 --- a/shared/packages/api/src/websocketClient.ts +++ b/shared/packages/api/src/websocketClient.ts @@ -1,4 +1,5 @@ import WebSocket from 'ws' +import { LoggerInstance } from './logger' import { MessageBase, MessageIdentifyClient, PING_TIME, WebsocketConnection } from './websocketConnection' /** A Class which */ @@ -8,6 +9,7 @@ export class WebsocketClient extends WebsocketConnection { private closed = false constructor( + private logger: LoggerInstance, private readonly id: string, private readonly url: string, private readonly clientType: MessageIdentifyClient['clientType'], @@ -71,7 +73,7 @@ export class WebsocketClient extends WebsocketConnection { } private reconnect() { this.connect().catch((err) => { - console.error(err) + this.logger.error(err) this.onLostConnection() }) } diff --git a/shared/packages/api/src/websocketConnection.ts b/shared/packages/api/src/websocketConnection.ts index aac1141c..b40b3d3c 100644 --- a/shared/packages/api/src/websocketConnection.ts +++ b/shared/packages/api/src/websocketConnection.ts @@ -103,7 +103,7 @@ export interface MessageReply { } export interface MessageIdentifyClient { internalType: 'identify_client' - clientType: 'N/A' | 'workerAgent' | 'expectationManager' + clientType: 'N/A' | 'workerAgent' | 'expectationManager' | 'appContainer' id: string } diff --git a/shared/packages/api/src/websocketServer.ts b/shared/packages/api/src/websocketServer.ts index 37fdb505..f5483924 100644 --- a/shared/packages/api/src/websocketServer.ts +++ b/shared/packages/api/src/websocketServer.ts @@ -39,6 +39,13 @@ export class WebsocketServer { this.clients.forEach((client) => client.close()) this.wss.close() } + get port(): number { + const address = this.wss.address() + if (typeof address === 'string') + throw new Error(`Internal error: to be implemented: wss.address() as string "${address}"`) + + return address.port + } } export class ClientConnection extends WebsocketConnection { @@ -96,4 +103,4 @@ export class ClientConnection extends WebsocketConnection { this.ws?.close() } } -type ClientType = MessageIdentifyClient['clientType'] +export type ClientType = MessageIdentifyClient['clientType'] diff --git a/shared/packages/api/src/worker.ts b/shared/packages/api/src/worker.ts index ff941c22..5c5ab357 100644 --- a/shared/packages/api/src/worker.ts +++ b/shared/packages/api/src/worker.ts @@ -2,24 +2,43 @@ * This file contains API definitions for the Worker methods */ -export interface ReturnTypeDoYouSupportExpectation { - support: boolean - reason: string -} +import { Reason } from './methods' + +export type ReturnTypeDoYouSupportExpectation = + | { + support: true + } + | { + support: false + reason: Reason + } export type ReturnTypeGetCostFortExpectation = number -export interface ReturnTypeIsExpectationReadyToStartWorkingOn { - ready: boolean - sourceExists?: boolean - reason?: string -} -export interface ReturnTypeIsExpectationFullfilled { - fulfilled: boolean - reason?: string -} -export interface ReturnTypeRemoveExpectation { - removed: boolean - reason?: string -} +export type ReturnTypeIsExpectationReadyToStartWorkingOn = + | { + ready: true + sourceExists?: boolean + } + | { + ready: false + sourceExists?: boolean + reason: Reason + } +export type ReturnTypeIsExpectationFullfilled = + | { + fulfilled: true + } + | { + fulfilled: false + reason: Reason + } +export type ReturnTypeRemoveExpectation = + | { + removed: true + } + | { + removed: false + reason: Reason + } /** Configurations for any of the workers */ export interface WorkerAgentConfig { @@ -38,23 +57,39 @@ export interface WorkerAgentConfig { */ windowsDriveLetters?: string[] } -export interface ReturnTypeDoYouSupportPackageContainer { - support: boolean - reason: string -} -export interface ReturnTypeRunPackageContainerCronJob { - completed: boolean - reason?: string -} -export interface ReturnTypeDisposePackageContainerMonitors { - disposed: boolean - reason?: string -} -export interface ReturnTypeSetupPackageContainerMonitors { - setupOk: boolean - reason?: string - monitors?: { [monitorId: string]: MonitorProperties } -} +export type ReturnTypeDoYouSupportPackageContainer = + | { + support: true + } + | { + support: false + reason: Reason + } +export type ReturnTypeRunPackageContainerCronJob = + | { + success: true + } + | { + success: false + reason: Reason + } +export type ReturnTypeDisposePackageContainerMonitors = + | { + success: true + } + | { + success: false + reason: Reason + } +export type ReturnTypeSetupPackageContainerMonitors = + | { + success: true + monitors: { [monitorId: string]: MonitorProperties } + } + | { + success: false + reason: Reason + } export interface MonitorProperties { label: string } diff --git a/shared/packages/expectationManager/package.json b/shared/packages/expectationManager/package.json index 99ba85b7..bfbe93b7 100644 --- a/shared/packages/expectationManager/package.json +++ b/shared/packages/expectationManager/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "devDependencies": { "lint-staged": "^7.2.0" @@ -35,4 +35,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/expectationManager/src/expectationManager.ts b/shared/packages/expectationManager/src/expectationManager.ts index edc3cf22..960e159b 100644 --- a/shared/packages/expectationManager/src/expectationManager.ts +++ b/shared/packages/expectationManager/src/expectationManager.ts @@ -10,6 +10,10 @@ import { Hook, LoggerInstance, PackageContainerExpectation, + Reason, + assertNever, + ExpectationManagerStatus, + LogLevel, } from '@shared/api' import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' import { WorkforceAPI } from './workforceApi' @@ -19,20 +23,13 @@ import PromisePool from '@supercharge/promise-pool' /** * The Expectation Manager is responsible for tracking the state of the Expectations, * and communicate with the Workers to progress them. + * @see FOR_DEVELOPERS.md */ export class ExpectationManager { - /** Time between iterations of the expectation queue */ - private readonly EVALUATE_INTERVAL = 10 * 1000 // ms - /** Minimum time between re-evaluating fulfilled expectations */ - private readonly FULLFILLED_MONITOR_TIME = 10 * 1000 // ms - /** - * If the iteration of the queue has been going for this time - * allow skipping the rest of the queue in order to reiterate the high-prio expectations - */ - private readonly ALLOW_SKIPPING_QUEUE_TIME = 30 * 1000 // ms + private constants: ExpectationManagerConstants - private workforceAPI = new WorkforceAPI() + private workforceAPI: WorkforceAPI /** Store for various incoming data, to be processed on next iteration round */ private receivedUpdates: { @@ -86,42 +83,70 @@ export class ExpectationManager { properties: ExpectationManagerWorkerAgent.WorkInProgressProperties trackedExp: TrackedExpectation worker: WorkerAgentAPI + lastUpdated: number } } = {} - private terminating = false + private terminated = false + + private status: ExpectationManagerStatus constructor( private logger: LoggerInstance, public readonly managerId: string, private serverOptions: ExpectationManagerServerOptions, /** At what url the ExpectationManager can be reached on */ - private serverAccessUrl: string | undefined, + private serverAccessBaseUrl: string | undefined, private workForceConnectionOptions: ClientConnectionOptions, - private reportExpectationStatus: ReportExpectationStatus, - private reportPackageContainerPackageStatus: ReportPackageContainerPackageStatus, - private reportPackageContainerExpectationStatus: ReportPackageContainerExpectationStatus, - private onMessageFromWorker: MessageFromWorker + private callbacks: ExpectationManagerCallbacks, + options?: ExpectationManagerOptions ) { + this.constants = { + // Default values: + EVALUATE_INTERVAL: 5 * 1000, + FULLFILLED_MONITOR_TIME: 10 * 1000, + WORK_TIMEOUT_TIME: 10 * 1000, + ALLOW_SKIPPING_QUEUE_TIME: 30 * 1000, + SCALE_UP_TIME: 5 * 1000, + SCALE_UP_COUNT: 1, + WORKER_SUPPORT_TIME: 60 * 1000, + + ...options?.constants, + } + this.workforceAPI = new WorkforceAPI(this.logger) + this.status = this.updateStatus() if (this.serverOptions.type === 'websocket') { - this.logger.info(`Expectation Manager on port ${this.serverOptions.port}`) this.websocketServer = new WebsocketServer(this.serverOptions.port, (client: ClientConnection) => { // A new client has connected - this.logger.info(`New ${client.clientType} connected, id "${client.clientId}"`) + this.logger.info(`ExpectationManager: New ${client.clientType} connected, id "${client.clientId}"`) - if (client.clientType === 'workerAgent') { - const expectationManagerMethods = this.getWorkerAgentAPI(client.clientId) + switch (client.clientType) { + case 'workerAgent': { + const expectationManagerMethods = this.getWorkerAgentAPI(client.clientId) - const api = new WorkerAgentAPI(expectationManagerMethods, { - type: 'websocket', - clientConnection: client, - }) + const api = new WorkerAgentAPI(expectationManagerMethods, { + type: 'websocket', + clientConnection: client, + }) + this.workerAgents[client.clientId] = { api } + client.on('close', () => { + delete this.workerAgents[client.clientId] + }) - this.workerAgents[client.clientId] = { api } - } else { - throw new Error(`Unknown clientType "${client.clientType}"`) + this._triggerEvaluateExpectations(true) + break + } + case 'N/A': + case 'expectationManager': + case 'appContainer': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: { + assertNever(client.clientType) + throw new Error(`ExpectationManager: Unknown clientType "${client.clientType}"`) + } } }) + this.logger.info(`Expectation Manager running on port ${this.websocketServer.port}`) } else { // todo: handle direct connections } @@ -131,11 +156,17 @@ export class ExpectationManager { async init(): Promise { await this.workforceAPI.init(this.managerId, this.workForceConnectionOptions, this) - const serverAccessUrl = - this.workForceConnectionOptions.type === 'internal' ? '__internal' : this.serverAccessUrl - + let serverAccessUrl: string + if (this.workForceConnectionOptions.type === 'internal') { + serverAccessUrl = '__internal' + } else { + serverAccessUrl = this.serverAccessBaseUrl || 'ws://127.0.0.1' + if (this.serverOptions.type === 'websocket' && this.serverOptions.port === 0) { + // When the configured port i 0, the next free port is picked + serverAccessUrl += `:${this.websocketServer?.port}` + } + } if (!serverAccessUrl) throw new Error(`ExpectationManager.serverAccessUrl not set!`) - await this.workforceAPI.registerExpectationManager(this.managerId, serverAccessUrl) this._triggerEvaluateExpectations(true) @@ -147,10 +178,32 @@ export class ExpectationManager { this.workforceAPI.hook(hook) } terminate(): void { - this.terminating = true + this.terminated = true if (this.websocketServer) { this.websocketServer.terminate() } + if (this._evaluateExpectationsTimeout) { + clearTimeout(this._evaluateExpectationsTimeout) + this._evaluateExpectationsTimeout = undefined + } + } + /** USED IN TESTS ONLY. Quickly reset the tracked work of the expectationManager. */ + resetWork(): void { + this.receivedUpdates = { + expectations: {}, + expectationsHasBeenUpdated: false, + packageContainers: {}, + packageContainersHasBeenUpdated: false, + restartExpectations: {}, + abortExpectations: {}, + restartAllExpectations: false, + } + this.trackedExpectations = {} + this.trackedExpectationsCount = 0 + this.trackedPackageContainers = {} + // this.worksInProgress + + this._triggerEvaluateExpectations(true) } /** Returns a Hook used to hook up a WorkerAgent to our API-methods. */ getWorkerAgentHook(): Hook< @@ -170,6 +223,15 @@ export class ExpectationManager { return workerAgentMethods } } + removeWorkerAgentHook(clientId: string): void { + const workerAgent = this.workerAgents[clientId] + if (!workerAgent) throw new Error(`WorkerAgent "${clientId}" not found!`) + + if (workerAgent.api.type !== 'internal') + throw new Error(`Cannot remove WorkerAgent "${clientId}", due to the type being "${workerAgent.api.type}"`) + + delete this.workerAgents[clientId] + } /** Called when there is an updated set of PackageContainerExpectations. */ updatePackageContainerExpectations(packageContainers: { [id: string]: PackageContainerExpectation }): void { // We store the incoming expectations here, so that we don't modify anything in the middle of the _evaluateExpectations() iteration loop. @@ -208,11 +270,35 @@ export class ExpectationManager { this.receivedUpdates.expectationsHasBeenUpdated = true this._triggerEvaluateExpectations(true) } + async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } + async _debugKill(): Promise { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } + async getStatus(): Promise { + return { + workforce: await this.workforceAPI.getStatus(), + expectationManager: this.status, + } + } + async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + return this.workforceAPI.setLogLevelOfApp(appId, logLevel) + } + async debugKillApp(appId: string): Promise { + return this.workforceAPI._debugKillApp(appId) + } /** * Schedule the evaluateExpectations() to run * @param asap If true, will re-schedule evaluateExpectations() to run as soon as possible */ private _triggerEvaluateExpectations(asap?: boolean): void { + if (this.terminated) return + if (asap) this._evaluateExpectationsRunAsap = true if (this._evaluateExpectationsIsBusy) return @@ -221,11 +307,9 @@ export class ExpectationManager { this._evaluateExpectationsTimeout = undefined } - if (this.terminating) return - this._evaluateExpectationsTimeout = setTimeout( () => { - if (this.terminating) return + if (this.terminated) return this._evaluateExpectationsRunAsap = false this._evaluateExpectationsIsBusy = true @@ -241,7 +325,7 @@ export class ExpectationManager { this._triggerEvaluateExpectations() }) }, - this._evaluateExpectationsRunAsap ? 1 : this.EVALUATE_INTERVAL + this._evaluateExpectationsRunAsap ? 1 : this.constants.EVALUATE_INTERVAL ) } /** Return the API-methods that the ExpectationManager exposes to the WorkerAgent */ @@ -250,7 +334,7 @@ export class ExpectationManager { messageFromWorker: async ( message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any ): Promise => { - return this.onMessageFromWorker(message) + return this.callbacks.messageFromWorker(message) }, wipEventProgress: async ( @@ -260,19 +344,25 @@ export class ExpectationManager { ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + wip.lastUpdated = Date.now() + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash wip.trackedExp.status.workProgress = progress - this.logger.info( + this.logger.debug( `Expectation "${JSON.stringify( wip.trackedExp.exp.statusReport.label )}" progress: ${progress}` ) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, actualVersionHash, { - progress: progress, - }) + this.callbacks.reportExpectationStatus( + wip.trackedExp.id, + wip.trackedExp.exp, + actualVersionHash, + { + progress: progress, + } + ) } else { // ignore } @@ -281,19 +371,29 @@ export class ExpectationManager { wipEventDone: async ( wipId: number, actualVersionHash: string, - reason: string, + reason: Reason, _result: any ): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { + wip.lastUpdated = Date.now() + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { wip.trackedExp.status.actualVersionHash = actualVersionHash - this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.FULFILLED, reason) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, actualVersionHash, { - status: wip.trackedExp.state, - statusReason: wip.trackedExp.reason, - progress: 1, - }) + this.updateTrackedExpStatus( + wip.trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, + reason + ) + this.callbacks.reportExpectationStatus( + wip.trackedExp.id, + wip.trackedExp.exp, + actualVersionHash, + { + status: wip.trackedExp.state, + statusReason: wip.trackedExp.reason, + progress: 1, + } + ) if (this.handleTriggerByFullfilledIds(wip.trackedExp)) { // Something was triggered, run again asap. @@ -306,12 +406,18 @@ export class ExpectationManager { delete this.worksInProgress[`${clientId}_${wipId}`] } }, - wipEventError: async (wipId: number, error: string): Promise => { + wipEventError: async (wipId: number, reason: Reason): Promise => { const wip = this.worksInProgress[`${clientId}_${wipId}`] if (wip) { - if (wip.trackedExp.state === TrackedExpectationState.WORKING) { - this.updateTrackedExpStatus(wip.trackedExp, TrackedExpectationState.WAITING, error) - this.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { + wip.lastUpdated = Date.now() + if (wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { + wip.trackedExp.errorCount++ + this.updateTrackedExpStatus( + wip.trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + reason + ) + this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { status: wip.trackedExp.state, statusReason: wip.trackedExp.reason, }) @@ -328,7 +434,7 @@ export class ExpectationManager { * Evaluates the Expectations and PackageContainerExpectations */ private async _evaluateExpectations(): Promise { - this.logger.info(Date.now() / 1000 + ' _evaluateExpectations ----------') + this.logger.debug(Date.now() / 1000 + ' _evaluateExpectations ----------') // First we're going to see if there is any new incoming data which needs to be pulled in. if (this.receivedUpdates.expectationsHasBeenUpdated) { @@ -343,9 +449,15 @@ export class ExpectationManager { // Iterate through the PackageContainerExpectations: await this._evaluateAllTrackedPackageContainers() + this.monitorWorksInProgress() + // Iterate through all Expectations: const runAgainASAP = await this._evaluateAllExpectations() + this.updateStatus() + + this.checkIfNeedToScaleUp() + if (runAgainASAP) { this._triggerEvaluateExpectations(true) } @@ -367,9 +479,9 @@ export class ExpectationManager { } else if (!_.isEqual(this.trackedExpectations[id].exp, exp)) { const trackedExp = this.trackedExpectations[id] - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to update`) + this.logger.debug(`Cancelling ${trackedExp.id} due to update`) await trackedExp.status.workInProgressCancel() } } @@ -379,38 +491,56 @@ export class ExpectationManager { const trackedExp: TrackedExpectation = { id: id, exp: exp, - state: TrackedExpectationState.NEW, - availableWorkers: [], + state: ExpectedPackageStatusAPI.WorkStatusState.NEW, + queriedWorkers: {}, + availableWorkers: {}, + noAvailableWorkersReason: { + user: 'Unknown reason', + tech: 'N/A (init)', + }, lastEvaluationTime: 0, - reason: '', + waitingForWorkerTime: null, + errorCount: 0, + reason: { + user: '', + tech: '', + }, status: {}, session: null, } this.trackedExpectations[id] = trackedExp if (isNew) { - this.updateTrackedExpStatus(trackedExp, undefined, 'Added just now') + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Added just now`, + tech: `Added ${Date.now()}`, + }) } else { - this.updateTrackedExpStatus(trackedExp, undefined, 'Updated just now') + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Updated just now`, + tech: `Updated ${Date.now()}`, + }) } } } // Removed: for (const id of Object.keys(this.trackedExpectations)) { + this.trackedExpectations[id].errorCount = 0 // Also reset the errorCount, to start fresh. + if (!this.receivedUpdates.expectations[id]) { // This expectation has been removed // TODO: handled removed expectations! const trackedExp = this.trackedExpectations[id] - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to removed`) + this.logger.debug(`Cancelling ${trackedExp.id} due to removed`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.REMOVED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.REMOVED trackedExp.lastEvaluationTime = 0 // To rerun ASAP } } @@ -426,14 +556,14 @@ export class ExpectationManager { for (const id of Object.keys(this.receivedUpdates.restartExpectations)) { const trackedExp = this.trackedExpectations[id] if (trackedExp) { - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to restart`) + this.logger.debug(`Cancelling ${trackedExp.id} due to restart`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.RESTARTED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.RESTARTED trackedExp.lastEvaluationTime = 0 // To rerun ASAP } } @@ -443,14 +573,14 @@ export class ExpectationManager { for (const id of Object.keys(this.receivedUpdates.abortExpectations)) { const trackedExp = this.trackedExpectations[id] if (trackedExp) { - if (trackedExp.state == TrackedExpectationState.WORKING) { + if (trackedExp.state == ExpectedPackageStatusAPI.WorkStatusState.WORKING) { if (trackedExp.status.workInProgressCancel) { - this.logger.info(`Cancelling ${trackedExp.id} due to abort`) + this.logger.debug(`Cancelling ${trackedExp.id} due to abort`) await trackedExp.status.workInProgressCancel() } } - trackedExp.state = TrackedExpectationState.ABORTED + trackedExp.state = ExpectedPackageStatusAPI.WorkStatusState.ABORTED } } this.receivedUpdates.abortExpectations = {} @@ -475,6 +605,10 @@ export class ExpectationManager { private getTrackedExpectations(): TrackedExpectation[] { const tracked: TrackedExpectation[] = Object.values(this.trackedExpectations) tracked.sort((a, b) => { + // Lowest errorCount first, this is to make it so that if one expectation fails, it'll not block all the others + if (a.errorCount > b.errorCount) return 1 + if (a.errorCount < b.errorCount) return -1 + // Lowest priority first if (a.exp.priority > b.exp.priority) return 1 if (a.exp.priority < b.exp.priority) return -1 @@ -505,16 +639,16 @@ export class ExpectationManager { // Step 1: Evaluate the Expectations which are in the states that can be handled in parallel: for (const handleState of [ // Note: The order of these is important, as the states normally progress in this order: - TrackedExpectationState.ABORTED, - TrackedExpectationState.NEW, - TrackedExpectationState.WAITING, - TrackedExpectationState.FULFILLED, + ExpectedPackageStatusAPI.WorkStatusState.ABORTED, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, ]) { // Filter out the ones that are in the state we're about to handle: const trackedWithState = tracked.filter((trackedExp) => trackedExp.state === handleState) if (trackedWithState.length) { - this.logger.info(`Handle state ${handleState}, ${trackedWithState.length} expectations..`) + this.logger.debug(`Handle state ${handleState}, ${trackedWithState.length} expectations..`) } if (trackedWithState.length) { @@ -536,7 +670,7 @@ export class ExpectationManager { } } - this.logger.info(`Handle other states..`) + this.logger.debug(`Handle other states..`) // Step 1.5: Reset the session: // Because during the next iteration, the worker-assignment need to be done in series @@ -565,7 +699,7 @@ export class ExpectationManager { removeIds.push(trackedExp.id) } } - if (runAgainASAP && Date.now() - startTime > this.ALLOW_SKIPPING_QUEUE_TIME) { + if (runAgainASAP && Date.now() - startTime > this.constants.ALLOW_SKIPPING_QUEUE_TIME) { // Skip the rest of the queue, so that we don't get stuck on evaluating low-prio expectations. break } @@ -588,154 +722,240 @@ export class ExpectationManager { if (trackedExp.session.hadError) return // do nothing try { - if (trackedExp.state === TrackedExpectationState.NEW) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW) { // Check which workers might want to handle it: - // Reset properties: - trackedExp.availableWorkers = [] + // trackedExp.availableWorkers = [] trackedExp.status = {} - let notSupportReason = 'No workers registered' + let hasQueriedAnyone = false await Promise.all( - Object.entries(this.workerAgents).map(async ([id, workerAgent]) => { - const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) - - if (support.support) { - trackedExp.availableWorkers.push(id) - } else { - notSupportReason = support.reason + Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { + // Only ask each worker once: + if ( + !trackedExp.queriedWorkers[workerId] || + Date.now() - trackedExp.queriedWorkers[workerId] > this.constants.WORKER_SUPPORT_TIME + ) { + trackedExp.queriedWorkers[workerId] = Date.now() + hasQueriedAnyone = true + try { + const support = await workerAgent.api.doYouSupportExpectation(trackedExp.exp) + + if (support.support) { + trackedExp.availableWorkers[workerId] = true + } else { + delete trackedExp.availableWorkers[workerId] + trackedExp.noAvailableWorkersReason = support.reason + } + } catch (err) { + delete trackedExp.availableWorkers[workerId] + + if ((err + '').match(/timeout/i)) { + trackedExp.noAvailableWorkersReason = { + user: 'Worker timed out', + tech: `Worker "${workerId} timeout"`, + } + } else throw err + } } }) ) - if (trackedExp.availableWorkers.length) { - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.WAITING, - `Found ${trackedExp.availableWorkers.length} workers who supports this Expectation` - ) - trackedExp.session.triggerExpectationAgain = true + const availableWorkersCount = Object.keys(trackedExp.availableWorkers).length + if (availableWorkersCount) { + if (hasQueriedAnyone) { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.WAITING, { + user: `${availableWorkersCount} workers available, about to start...`, + tech: `Found ${availableWorkersCount} workers who supports this Expectation`, + }) + trackedExp.session.triggerExpectationAgain = true + } else { + // If we didn't query anyone, just skip ahead to next state without being too verbose: + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WAITING, + undefined + ) + } } else { - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.NEW, - `Found no workers who supports this Expectation: "${notSupportReason}"` - ) + if (!Object.keys(trackedExp.queriedWorkers).length) { + if (!Object.keys(this.workerAgents).length) { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `No Workers available (this is likely a configuration issue)`, + tech: `No Workers available`, + }) + } else { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `No Workers available (this is likely a configuration issue)`, + tech: `No Workers queried, ${Object.keys(this.workerAgents).length} available`, + }) + } + } else { + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: `Found no workers who supports this Expectation, due to: ${trackedExp.noAvailableWorkersReason.user}`, + tech: `Found no workers who supports this Expectation: "${ + trackedExp.noAvailableWorkersReason.tech + }", have asked workers: [${Object.keys(trackedExp.queriedWorkers).join(',')}]`, + }) + } } - } else if (trackedExp.state === TrackedExpectationState.WAITING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { // Check if the expectation is ready to start: - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { - // First, check if it is already fulfilled: - const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( - trackedExp.exp, - false - ) - if (fulfilled.fulfilled) { - // The expectation is already fulfilled: - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.FULFILLED, fulfilled.reason) - if (this.handleTriggerByFullfilledIds(trackedExp)) { - // Something was triggered, run again ASAP: - trackedExp.session.triggerOtherExpectationsAgain = true - } - } else { - const readyToStart = await this.isExpectationReadyToStartWorkingOn( - trackedExp.session.assignedWorker.worker, - trackedExp + try { + // First, check if it is already fulfilled: + const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( + trackedExp.exp, + false ) - - const newStatus: Partial = {} - if (readyToStart.sourceExists !== undefined) newStatus.sourceExists = readyToStart.sourceExists - - if (readyToStart.ready) { + if (fulfilled.fulfilled) { + // The expectation is already fulfilled: this.updateTrackedExpStatus( trackedExp, - TrackedExpectationState.READY, - 'Ready to start', - newStatus + ExpectedPackageStatusAPI.WorkStatusState.FULFILLED, + undefined ) - trackedExp.session.triggerExpectationAgain = true + if (this.handleTriggerByFullfilledIds(trackedExp)) { + // Something was triggered, run again ASAP: + trackedExp.session.triggerOtherExpectationsAgain = true + } } else { - // Not ready to start - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.WAITING, - readyToStart.reason, - newStatus + const readyToStart = await this.isExpectationReadyToStartWorkingOn( + trackedExp.session.assignedWorker.worker, + trackedExp ) + + const newStatus: Partial = {} + if (readyToStart.sourceExists !== undefined) + newStatus.sourceExists = readyToStart.sourceExists + + if (readyToStart.ready) { + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.READY, + { + user: 'About to start working..', + tech: 'About to start working..', + }, + newStatus + ) + trackedExp.session.triggerExpectationAgain = true + } else { + // Not ready to start + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + readyToStart.reason, + newStatus + ) + } } + } catch (error) { + // There was an error, clearly it's not ready to start + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: 'Restarting due to error', + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) } } else { // No worker is available at the moment. // Do nothing, hopefully some will be available at a later iteration this.updateTrackedExpStatus( trackedExp, - undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + ExpectedPackageStatusAPI.WorkStatusState.NEW, + this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.READY) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) { // Start working on it: - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { const assignedWorker = trackedExp.session.assignedWorker - // Start working on the Expectation: - const wipInfo = await assignedWorker.worker.workOnExpectation(trackedExp.exp, assignedWorker.cost) + try { + this.logger.debug(`workOnExpectation: "${trackedExp.exp.id}" (${trackedExp.exp.type})`) - trackedExp.status.workInProgressCancel = async () => { - await assignedWorker.worker.cancelWorkInProgress(wipInfo.wipId) - delete trackedExp.status.workInProgressCancel - } + // Start working on the Expectation: + const wipInfo = await assignedWorker.worker.workOnExpectation( + trackedExp.exp, + assignedWorker.cost + ) - // trackedExp.status.workInProgress = new WorkInProgressReceiver(wipInfo.properties) - this.worksInProgress[`${assignedWorker.id}_${wipInfo.wipId}`] = { - properties: wipInfo.properties, - trackedExp: trackedExp, - worker: assignedWorker.worker, - } + trackedExp.status.workInProgressCancel = async () => { + await assignedWorker.worker.cancelWorkInProgress(wipInfo.wipId) + delete trackedExp.status.workInProgressCancel + } - this.updateTrackedExpStatus( - trackedExp, - TrackedExpectationState.WORKING, - undefined, - wipInfo.properties - ) + // trackedExp.status.workInProgress = new WorkInProgressReceiver(wipInfo.properties) + this.worksInProgress[`${assignedWorker.id}_${wipInfo.wipId}`] = { + properties: wipInfo.properties, + trackedExp: trackedExp, + worker: assignedWorker.worker, + lastUpdated: Date.now(), + } + + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.WORKING, + undefined, + wipInfo.properties + ) + } catch (error) { + // There was an error + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: 'Restarting due to an error', + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) + } } else { // No worker is available at the moment. // Do nothing, hopefully some will be available at a later iteration this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.WORKING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { // It is already working, don't do anything // TODO: work-timeout? - } else if (trackedExp.state === TrackedExpectationState.FULFILLED) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { // TODO: Some monitor that is able to invalidate if it isn't fullfilled anymore? if (timeSinceLastEvaluation > this.getFullfilledWaitTime()) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { - // Check if it is still fulfilled: - const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( - trackedExp.exp, - true - ) - if (fulfilled.fulfilled) { - // Yes it is still fullfiled - // No need to update the tracked state, since it's already fullfilled: - // this.updateTrackedExp(trackedExp, TrackedExpectationState.FULFILLED, fulfilled.reason) - } else { - // It appears like it's not fullfilled anymore - trackedExp.status.actualVersionHash = undefined - trackedExp.status.workProgress = undefined - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.WAITING, fulfilled.reason) - trackedExp.session.triggerExpectationAgain = true + try { + // Check if it is still fulfilled: + const fulfilled = await trackedExp.session.assignedWorker.worker.isExpectationFullfilled( + trackedExp.exp, + true + ) + if (fulfilled.fulfilled) { + // Yes it is still fullfiled + // No need to update the tracked state, since it's already fullfilled: + // this.updateTrackedExp(trackedExp, WorkStatusState.FULFILLED, fulfilled.reason) + } else { + // It appears like it's not fullfilled anymore + trackedExp.status.actualVersionHash = undefined + trackedExp.status.workProgress = undefined + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.NEW, + fulfilled.reason + ) + trackedExp.session.triggerExpectationAgain = true + } + } catch (error) { + // Do nothing, hopefully some will be available at a later iteration + // todo: Is this the right thing to do? + this.updateTrackedExpStatus(trackedExp, undefined, { + user: `Can't check if fulfilled, due to an error`, + tech: `Error from worker ${trackedExp.session.assignedWorker.id}: "${error}"`, + }) } } else { // No worker is available at the moment. @@ -743,22 +963,26 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else { // Do nothing } - } else if (trackedExp.state === TrackedExpectationState.REMOVED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.REMOVED) { + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { trackedExp.session.expectationCanBeRemoved = true - this.reportExpectationStatus(trackedExp.id, null, null, {}) + this.callbacks.reportExpectationStatus(trackedExp.id, null, null, {}) } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.REMOVED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.REMOVED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -766,19 +990,26 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.RESTARTED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.RESTARTED) { + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.NEW, 'Ready to start') + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, { + user: 'Ready to start (after restart)', + tech: 'Ready to start (after restart)', + }) trackedExp.session.triggerExpectationAgain = true } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.RESTARTED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.RESTARTED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -786,19 +1017,26 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } - } else if (trackedExp.state === TrackedExpectationState.ABORTED) { - await this.assignWorkerToSession(trackedExp.session, trackedExp) + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.ABORTED) { + await this.assignWorkerToSession(trackedExp) if (trackedExp.session.assignedWorker) { // Start by removing the expectation const removed = await trackedExp.session.assignedWorker.worker.removeExpectation(trackedExp.exp) if (removed.removed) { // This will cause the expectation to be intentionally stuck in the ABORTED state. - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, 'Aborted') + this.updateTrackedExpStatus(trackedExp, ExpectedPackageStatusAPI.WorkStatusState.ABORTED, { + user: 'Aborted', + tech: 'Aborted', + }) } else { - this.updateTrackedExpStatus(trackedExp, TrackedExpectationState.ABORTED, removed.reason) + this.updateTrackedExpStatus( + trackedExp, + ExpectedPackageStatusAPI.WorkStatusState.ABORTED, + removed.reason + ) } } else { // No worker is available at the moment. @@ -806,22 +1044,26 @@ export class ExpectationManager { this.updateTrackedExpStatus( trackedExp, undefined, - trackedExp.session.noAssignedWorkerReason || 'Unknown reason' + this.getNoAssignedWorkerReason(trackedExp.session) ) } } else { assertNever(trackedExp.state) } } catch (err) { + this.logger.error(`Error thrown in evaluateExpectationState for expectation "${trackedExp.id}"`) this.logger.error(err) - this.updateTrackedExpStatus(trackedExp, undefined, err.toString()) + this.updateTrackedExpStatus(trackedExp, undefined, { + user: 'Internal error in Package Manager', + tech: err.toString(), + }) } } /** Returns the appropriate time to wait before checking a fulfilled expectation again */ private getFullfilledWaitTime(): number { return ( // Default minimum time to wait: - this.FULLFILLED_MONITOR_TIME + + this.constants.FULLFILLED_MONITOR_TIME + // Also add some more time, so that we don't check too often when we have a lot of expectations: this.trackedExpectationsCount * 0.02 ) @@ -829,8 +1071,8 @@ export class ExpectationManager { /** Update the state and status of a trackedExpectation */ private updateTrackedExpStatus( trackedExp: TrackedExpectation, - state: TrackedExpectationState | undefined, - reason: string | undefined, + state: ExpectedPackageStatusAPI.WorkStatusState | undefined, + reason: Reason | undefined, newStatus?: Partial ) { trackedExp.lastEvaluationTime = Date.now() @@ -847,8 +1089,8 @@ export class ExpectationManager { updatedState = true } - if (trackedExp.reason !== reason) { - trackedExp.reason = reason || '' + if (!_.isEqual(trackedExp.reason, reason)) { + trackedExp.reason = reason || { user: '', tech: '' } updatedReason = true } const status = Object.assign({}, trackedExp.status, newStatus) // extend with new values @@ -858,17 +1100,17 @@ export class ExpectationManager { } // Log and report new states an reasons: if (updatedState) { - this.logger.info( - `${trackedExp.exp.statusReport.label}: New state: "${prevState}"->"${trackedExp.state}", reason: "${trackedExp.reason}"` + this.logger.debug( + `${trackedExp.exp.statusReport.label}: New state: "${prevState}"->"${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } else if (updatedReason) { - this.logger.info( - `${trackedExp.exp.statusReport.label}: State: "${trackedExp.state}", reason: "${trackedExp.reason}"` + this.logger.debug( + `${trackedExp.exp.statusReport.label}: State: "${trackedExp.state}", reason: "${trackedExp.reason.tech}"` ) } if (updatedState || updatedReason) { - this.reportExpectationStatus(trackedExp.id, trackedExp.exp, null, { + this.callbacks.reportExpectationStatus(trackedExp.id, trackedExp.exp, null, { status: trackedExp.state, statusReason: trackedExp.reason, }) @@ -878,11 +1120,11 @@ export class ExpectationManager { } } private updatePackageContainerPackageStatus(trackedExp: TrackedExpectation) { - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { for (const fromPackage of trackedExp.exp.fromPackages) { // TODO: this is probably not eh right thing to do: for (const packageContainer of trackedExp.exp.endRequirement.targets) { - this.reportPackageContainerPackageStatus(packageContainer.containerId, fromPackage.id, { + this.callbacks.reportPackageContainerPackageStatus(packageContainer.containerId, fromPackage.id, { contentVersionHash: trackedExp.status.actualVersionHash || '', progress: trackedExp.status.workProgress || 0, status: this.getPackageStatus(trackedExp), @@ -898,9 +1140,9 @@ export class ExpectationManager { private getPackageStatus( trackedExp: TrackedExpectation ): ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus { - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { return ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY - } else if (trackedExp.state === TrackedExpectationState.WORKING) { + } else if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { return trackedExp.status.targetCanBeUsedWhileTransferring ? ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.TRANSFERRING_READY : ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.TRANSFERRING_NOT_READY @@ -911,10 +1153,9 @@ export class ExpectationManager { } } /** Do a bidding between the available Workers and assign the cheapest one to use for the evaulation-session. */ - private async assignWorkerToSession( - session: ExpectationStateHandlerSession, - trackedExp: TrackedExpectation - ): Promise { + private async assignWorkerToSession(trackedExp: TrackedExpectation): Promise { + const session: ExpectationStateHandlerSession | null = trackedExp.session + if (!session) throw new Error('ExpectationManager: INternal error: Session not set') if (session.assignedWorker) return // A worker has already been assigned /** How many requests to send out simultaneously */ @@ -922,19 +1163,20 @@ export class ExpectationManager { /** How many answers we want to have to be content */ const minWorkerCount = batchSize / 2 - if (!trackedExp.availableWorkers.length) { - session.noAssignedWorkerReason = 'No workers available' + if (!Object.keys(trackedExp.availableWorkers).length) { + session.noAssignedWorkerReason = { user: `No workers available`, tech: `No workers available` } } const workerCosts: WorkerAgentAssignment[] = [] - - for (let i = 0; i < trackedExp.availableWorkers.length; i += batchSize) { - const batchOfWorkers = trackedExp.availableWorkers.slice(i, i + batchSize) - - await Promise.all( - batchOfWorkers.map(async (workerId) => { - const workerAgent = this.workerAgents[workerId] - if (workerAgent) { + let noCostReason: string | undefined = undefined + + // Send a number of requests simultaneously: + await runInBatches( + Object.keys(trackedExp.availableWorkers), + async (workerId: string, abort: () => void) => { + const workerAgent = this.workerAgents[workerId] + if (workerAgent) { + try { const cost = await workerAgent.api.getCostForExpectation(trackedExp.exp) if (cost.cost < Number.POSITIVE_INFINITY) { @@ -945,11 +1187,14 @@ export class ExpectationManager { randomCost: Math.random(), // To randomize if there are several with the same best cost }) } + } catch (error) { + noCostReason = error.toString() } - }) - ) - if (workerCosts.length >= minWorkerCount) break - } + } + if (workerCosts.length >= minWorkerCount) abort() + }, + batchSize + ) workerCosts.sort((a, b) => { // Lowest cost first: @@ -971,7 +1216,14 @@ export class ExpectationManager { // Only allow starting if the job can start in a short while session.assignedWorker = bestWorker } else { - session.noAssignedWorkerReason = `Waiting for a free worker (${trackedExp.availableWorkers.length} busy)` + session.noAssignedWorkerReason = { + user: `Waiting for a free worker (${ + Object.keys(trackedExp.availableWorkers).length + } workers are currently busy)`, + tech: `Waiting for a free worker (${ + Object.keys(trackedExp.availableWorkers).length + } busy) ${noCostReason}`, + } } } /** @@ -980,7 +1232,7 @@ export class ExpectationManager { */ private handleTriggerByFullfilledIds(trackedExp: TrackedExpectation): boolean { let hasTriggeredSomething = false - if (trackedExp.state === TrackedExpectationState.FULFILLED) { + if (trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { const toTriggerIds = this._triggerByFullfilledIds[trackedExp.id] || [] for (const id of toTriggerIds) { @@ -999,24 +1251,36 @@ export class ExpectationManager { trackedExp: TrackedExpectation ): Promise { // First check if the Expectation depends on the fullfilled-status of another Expectation: + const waitingFor = this.isExpectationWaitingForOther(trackedExp) + + if (waitingFor) { + return { + ready: false, + reason: { + user: `Waiting for "${waitingFor.exp.statusReport.label}"`, + tech: `Waiting for "${waitingFor.exp.statusReport.label}"`, + }, + } + } + + return workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) + } + /** Checks if the expectation is waiting for another expectation, and returns the awaited Expectation, otherwise null */ + private isExpectationWaitingForOther(trackedExp: TrackedExpectation): TrackedExpectation | null { if (trackedExp.exp.dependsOnFullfilled?.length) { // Check if those are fullfilled: let waitingFor: TrackedExpectation | undefined = undefined for (const id of trackedExp.exp.dependsOnFullfilled) { - if (this.trackedExpectations[id].state !== TrackedExpectationState.FULFILLED) { + if (this.trackedExpectations[id].state !== ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { waitingFor = this.trackedExpectations[id] break } } if (waitingFor) { - return { - ready: false, - reason: `Waiting for "${waitingFor.exp.statusReport.label}"`, - } + return waitingFor } } - - return await workerAgent.isExpectationReadyToStartWorkingOn(trackedExp.exp) + return null } private async _updateReceivedPackageContainerExpectations() { this.receivedUpdates.packageContainersHasBeenUpdated = false @@ -1041,6 +1305,7 @@ export class ExpectationManager { currentWorker: null, isUpdated: true, lastEvaluationTime: 0, + monitorIsSetup: false, status: { monitors: {}, }, @@ -1070,85 +1335,118 @@ export class ExpectationManager { } private async _evaluateAllTrackedPackageContainers(): Promise { for (const trackedPackageContainer of Object.values(this.trackedPackageContainers)) { - if (trackedPackageContainer.isUpdated) { - // If the packageContainer was newly updated, reset and set up again: - if (trackedPackageContainer.currentWorker) { - const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - const dispose = await workerAgent.api.disposePackageContainerMonitors( - trackedPackageContainer.packageContainer - ) - if (!dispose.disposed) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, dispose.reason) - continue // Break further execution for this PackageContainer + try { + if (trackedPackageContainer.isUpdated) { + // If the packageContainer was newly updated, reset and set up again: + if (trackedPackageContainer.currentWorker) { + const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] + const disposeMonitorResult = await workerAgent.api.disposePackageContainerMonitors( + trackedPackageContainer.packageContainer + ) + if (!disposeMonitorResult.success) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to remove monitor, due to ${disposeMonitorResult.reason.user}`, + tech: `Unable to dispose monitor: ${disposeMonitorResult.reason.tech}`, + }) + continue // Break further execution for this PackageContainer + } + trackedPackageContainer.currentWorker = null } - trackedPackageContainer.currentWorker = null + trackedPackageContainer.isUpdated = false } - trackedPackageContainer.isUpdated = false - } - if (trackedPackageContainer.currentWorker) { - // Check that the worker still exists: - if (!this.workerAgents[trackedPackageContainer.currentWorker]) { - trackedPackageContainer.currentWorker = null + if (trackedPackageContainer.currentWorker) { + // Check that the worker still exists: + if (!this.workerAgents[trackedPackageContainer.currentWorker]) { + trackedPackageContainer.currentWorker = null + } } - } - let currentWorkerIsNew = false - if (!trackedPackageContainer.currentWorker) { - // Find a worker - let notSupportReason: string | null = null - await Promise.all( - Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { - const support = await workerAgent.api.doYouSupportPackageContainer( - trackedPackageContainer.packageContainer - ) - if (!trackedPackageContainer.currentWorker) { - if (support.support) { - trackedPackageContainer.currentWorker = workerId - currentWorkerIsNew = true - } else { - notSupportReason = support.reason + if (!trackedPackageContainer.currentWorker) { + // Find a worker that supports this PackageContainer + + let notSupportReason: Reason | null = null + await Promise.all( + Object.entries(this.workerAgents).map(async ([workerId, workerAgent]) => { + const support = await workerAgent.api.doYouSupportPackageContainer( + trackedPackageContainer.packageContainer + ) + if (!trackedPackageContainer.currentWorker) { + if (support.support) { + trackedPackageContainer.currentWorker = workerId + } else { + notSupportReason = support.reason + } } + }) + ) + if (!trackedPackageContainer.currentWorker) { + notSupportReason = { + user: 'Found no worker that supports this packageContainer', + tech: 'Found no worker that supports this packageContainer', } - }) - ) - if (!trackedPackageContainer.currentWorker) { - notSupportReason = 'Found no worker that supports this packageContainer' - } - if (notSupportReason) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, notSupportReason) - continue // Break further execution for this PackageContainer + } + if (notSupportReason) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to handle PackageContainer, due to: ${notSupportReason.user}`, + tech: `Unable to handle PackageContainer, due to: ${notSupportReason.tech}`, + }) + continue // Break further execution for this PackageContainer + } } - } - if (trackedPackageContainer.currentWorker) { - const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] + if (trackedPackageContainer.currentWorker) { + const workerAgent = this.workerAgents[trackedPackageContainer.currentWorker] - if (currentWorkerIsNew) { - const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( - trackedPackageContainer.packageContainer - ) + if (Object.keys(trackedPackageContainer.packageContainer.monitors).length !== 0) { + if (!trackedPackageContainer.monitorIsSetup) { + const monitorSetup = await workerAgent.api.setupPackageContainerMonitors( + trackedPackageContainer.packageContainer + ) - trackedPackageContainer.status.monitors = {} - for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors ?? {})) { - trackedPackageContainer.status.monitors[monitorId] = { - label: monitor.label, - reason: 'Starting up', + trackedPackageContainer.status.monitors = {} + if (monitorSetup.success) { + trackedPackageContainer.monitorIsSetup = true + for (const [monitorId, monitor] of Object.entries(monitorSetup.monitors)) { + trackedPackageContainer.status.monitors[monitorId] = { + label: monitor.label, + reason: { + user: 'Starting up', + tech: 'Starting up', + }, + } + } + } else { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.user}`, + tech: `Unable to set up monitor for PackageContainer, due to: ${monitorSetup.reason.tech}`, + }) + } + } + } + if (Object.keys(trackedPackageContainer.packageContainer.cronjobs).length !== 0) { + const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( + trackedPackageContainer.packageContainer + ) + if (!cronJobStatus.success) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: 'Cron job not completed: ' + cronJobStatus.reason.user, + tech: 'Cron job not completed: ' + cronJobStatus.reason.tech, + }) + continue } } } - const cronJobStatus = await workerAgent.api.runPackageContainerCronJob( - trackedPackageContainer.packageContainer - ) - if (!cronJobStatus.completed) { - this.updateTrackedPackageContainerStatus(trackedPackageContainer, cronJobStatus.reason) - continue - } + } catch (err) { + this.updateTrackedPackageContainerStatus(trackedPackageContainer, { + user: 'Internal Error', + tech: `Unhandled Error: ${err}`, + }) } } } private updateTrackedPackageContainerStatus( trackedPackageContainer: TrackedPackageContainerExpectation, - reason: string | undefined + reason: Reason ) { trackedPackageContainer.lastEvaluationTime = Date.now() @@ -1160,13 +1458,13 @@ export class ExpectationManager { } if (updatedReason) { - this.logger.info( - `${trackedPackageContainer.packageContainer.label}: Reason: "${trackedPackageContainer.status.reason}"` + this.logger.debug( + `PackageContainerStatus "${trackedPackageContainer.packageContainer.label}": Reason: "${trackedPackageContainer.status.reason.tech}"` ) } if (updatedReason) { - this.reportPackageContainerExpectationStatus( + this.callbacks.reportPackageContainerExpectationStatus( trackedPackageContainer.id, trackedPackageContainer.packageContainer, { @@ -1175,6 +1473,157 @@ export class ExpectationManager { ) } } + private getNoAssignedWorkerReason(session: ExpectationStateHandlerSession): ExpectedPackageStatusAPI.Reason { + if (!session.noAssignedWorkerReason) { + this.logger.error( + `trackedExp.session.noAssignedWorkerReason is undefined, although assignedWorker was set..` + ) + return { + user: 'Unknown reason (internal error)', + tech: 'Unknown reason', + } + } + return session.noAssignedWorkerReason + } + private updateStatus(): ExpectationManagerStatus { + this.status = { + id: this.managerId, + expectationStatistics: { + countTotal: 0, + + countNew: 0, + countWaiting: 0, + countReady: 0, + countWorking: 0, + countFulfilled: 0, + countRemoved: 0, + countRestarted: 0, + countAborted: 0, + + countNoAvailableWorkers: 0, + countError: 0, + }, + workerAgents: Object.entries(this.workerAgents).map(([id, _workerAgent]) => { + return { + workerId: id, + } + }), + } + const expectationStatistics = this.status.expectationStatistics + for (const exp of Object.values(this.trackedExpectations)) { + expectationStatistics.countTotal++ + + if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW) { + expectationStatistics.countNew++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING) { + expectationStatistics.countWaiting++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) { + expectationStatistics.countReady++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING) { + expectationStatistics.countWorking++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.FULFILLED) { + expectationStatistics.countFulfilled++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.REMOVED) { + expectationStatistics.countRemoved++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.RESTARTED) { + expectationStatistics.countRestarted++ + } else if (exp.state === ExpectedPackageStatusAPI.WorkStatusState.ABORTED) { + expectationStatistics.countAborted++ + } else assertNever(exp.state) + + if (!exp.availableWorkers.length) { + expectationStatistics.countNoAvailableWorkers++ + } + if ( + exp.errorCount > 0 && + exp.state !== ExpectedPackageStatusAPI.WorkStatusState.WORKING && + exp.state !== ExpectedPackageStatusAPI.WorkStatusState.FULFILLED + ) { + expectationStatistics.countError++ + } + } + return this.status + } + private async checkIfNeedToScaleUp(): Promise { + const waitingExpectations: TrackedExpectation[] = [] + + for (const exp of Object.values(this.trackedExpectations)) { + if ( + (exp.state === ExpectedPackageStatusAPI.WorkStatusState.NEW || + exp.state === ExpectedPackageStatusAPI.WorkStatusState.WAITING || + exp.state === ExpectedPackageStatusAPI.WorkStatusState.READY) && + (!exp.availableWorkers.length || // No workers supports it + !exp.session?.assignedWorker) && // No worker has time to work on it + !this.isExpectationWaitingForOther(exp) // Filter out expectations that aren't ready to begin working on anyway + ) { + if (!exp.waitingForWorkerTime) { + exp.waitingForWorkerTime = Date.now() + } + } else { + exp.waitingForWorkerTime = null + } + if (exp.waitingForWorkerTime) + if (exp.waitingForWorkerTime && Date.now() - exp.waitingForWorkerTime > this.constants.SCALE_UP_TIME) { + if (waitingExpectations.length < this.constants.SCALE_UP_COUNT) { + waitingExpectations.push(exp) + } + } + } + + for (const exp of waitingExpectations) { + this.logger.debug(`Requesting more resources to handle expectation "${exp.id}"`) + await this.workforceAPI.requestResources(exp.exp) + } + } + /** */ + private monitorWorksInProgress() { + for (const [wipId, wip] of Object.entries(this.worksInProgress)) { + if ( + wip.trackedExp.state === ExpectedPackageStatusAPI.WorkStatusState.WORKING && + Date.now() - wip.lastUpdated > this.constants.WORK_TIMEOUT_TIME + ) { + // It seems that the work has stalled.. + // Restart the job: + const reason: Reason = { + tech: 'WorkInProgress timeout', + user: 'The job timed out', + } + + wip.trackedExp.errorCount++ + this.updateTrackedExpStatus(wip.trackedExp, ExpectedPackageStatusAPI.WorkStatusState.NEW, reason) + this.callbacks.reportExpectationStatus(wip.trackedExp.id, wip.trackedExp.exp, null, { + status: wip.trackedExp.state, + statusReason: wip.trackedExp.reason, + }) + delete this.worksInProgress[wipId] + } + } + } +} +export interface ExpectationManagerOptions { + constants: Partial +} +export interface ExpectationManagerConstants { + /** Time between iterations of the expectation queue [ms] */ + EVALUATE_INTERVAL: number + /** Minimum time between re-evaluating fulfilled expectations [ms] */ + FULLFILLED_MONITOR_TIME: number + /** + * If the iteration of the queue has been going for this time + * allow skipping the rest of the queue in order to reiterate the high-prio expectations [ms] + */ + ALLOW_SKIPPING_QUEUE_TIME: number + + /** If there has been no updated on a work-in-progress, time it out after this time */ + WORK_TIMEOUT_TIME: number + + /** How long to wait before requesting more resources (workers) [ms] */ + SCALE_UP_TIME: number + /** How many resources to request at a time */ + SCALE_UP_COUNT: number + + /** How often to re-query a worker if it supports an expectation [ms] */ + WORKER_SUPPORT_TIME: number } export type ExpectationManagerServerOptions = | { @@ -1185,19 +1634,7 @@ export type ExpectationManagerServerOptions = | { type: 'internal' } -/** Denotes the various states of a Tracked Expectation */ -export enum TrackedExpectationState { - NEW = 'new', - WAITING = 'waiting', - READY = 'ready', - WORKING = 'working', - FULFILLED = 'fulfilled', - REMOVED = 'removed', - - // Triggered from Core: - RESTARTED = 'restarted', - ABORTED = 'aborted', -} + interface TrackedExpectation { /** Unique ID of the tracked expectation */ id: string @@ -1205,14 +1642,21 @@ interface TrackedExpectation { exp: Expectation.Any /** The current State of the expectation. */ - state: TrackedExpectationState - /** Human-readable reason for the current state. (To be used in GUIs) */ - reason: string + state: ExpectedPackageStatusAPI.WorkStatusState + /** Reason for the current state. */ + reason: Reason + /** List of worker ids that have gotten the question wether they support this expectation */ + queriedWorkers: { [workerId: string]: number } /** List of worker ids that supports this Expectation */ - availableWorkers: string[] + availableWorkers: { [workerId: string]: true } + noAvailableWorkersReason: Reason /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number + /** Timestamp to be track how long the expectation has been awiting for a worker (can't start working) */ + waitingForWorkerTime: number | null + /** The number of times the expectation has failed */ + errorCount: number /** These statuses are sent from the workers */ status: { @@ -1242,7 +1686,7 @@ interface ExpectationStateHandlerSession { /** The Worker assigned to the Expectation during this evaluation-session */ assignedWorker?: WorkerAgentAssignment - noAssignedWorkerReason?: string + noAssignedWorkerReason?: Reason } interface WorkerAgentAssignment { worker: WorkerAgentAPI @@ -1252,28 +1696,32 @@ interface WorkerAgentAssignment { } export type MessageFromWorker = (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => Promise -export type ReportExpectationStatus = ( - expectationId: string, - expectaction: Expectation.Any | null, - actualVersionHash: string | null, - statusInfo: { - status?: string - progress?: number - statusReason?: string - } -) => void -export type ReportPackageContainerPackageStatus = ( - containerId: string, - packageId: string, - packageStatus: ExpectedPackageStatusAPI.PackageContainerPackageStatus | null -) => void -export type ReportPackageContainerExpectationStatus = ( - containerId: string, - packageContainer: PackageContainerExpectation | null, - statusInfo: { - statusReason?: string - } -) => void +export interface ExpectationManagerCallbacks { + reportExpectationStatus: ( + expectationId: string, + expectaction: Expectation.Any | null, + actualVersionHash: string | null, + statusInfo: { + status?: ExpectedPackageStatusAPI.WorkStatusState + progress?: number + statusReason?: Reason + } + ) => void + reportPackageContainerPackageStatus: ( + containerId: string, + packageId: string, + packageStatus: Omit | null + ) => void + reportPackageContainerExpectationStatus: ( + containerId: string, + packageContainer: PackageContainerExpectation | null, + statusInfo: { + statusReason?: Reason + } + ) => void + messageFromWorker: MessageFromWorker +} + interface TrackedPackageContainerExpectation { /** Unique ID of the tracked packageContainer */ id: string @@ -1287,17 +1735,52 @@ interface TrackedPackageContainerExpectation { /** Timestamp of the last time the expectation was evaluated. */ lastEvaluationTime: number + /** If the monitor is set up okay */ + monitorIsSetup: boolean + /** These statuses are sent from the workers */ status: { - reason?: string + /** Reason for the status (used in GUIs) */ + reason?: Reason monitors: { [monitorId: string]: { label: string - reason: string + reason: Reason } } } } -function assertNever(_shouldBeNever: never) { - // Nothing +/** Execute callback in batches */ +async function runInBatches( + values: T[], + cb: (value: T, abortAndReturn: (returnValue?: ReturnValue) => void) => Promise, + batchSize: number +): Promise { + const batches: T[][] = [] + let batch: T[] = [] + for (const value of values) { + batch.push(value) + if (batch.length >= batchSize) { + batches.push(batch) + batch = [] + } + } + batches.push(batch) + + let aborted = false + let returnValue: ReturnValue | undefined = undefined + const abortAndReturn = (value?: ReturnValue) => { + aborted = true + returnValue = value + } + + for (const batch of batches) { + if (aborted) break + await Promise.all( + batch.map(async (value) => { + return cb(value, abortAndReturn) + }) + ) + } + return returnValue } diff --git a/shared/packages/expectationManager/src/workerAgentApi.ts b/shared/packages/expectationManager/src/workerAgentApi.ts index 36faf87f..3e2266c6 100644 --- a/shared/packages/expectationManager/src/workerAgentApi.ts +++ b/shared/packages/expectationManager/src/workerAgentApi.ts @@ -31,40 +31,40 @@ export class WorkerAgentAPI async doYouSupportExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('doYouSupportExpectation', exp) + return this._sendMessage('doYouSupportExpectation', exp) } async getCostForExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('getCostForExpectation', exp) + return this._sendMessage('getCostForExpectation', exp) } async isExpectationReadyToStartWorkingOn( exp: Expectation.Any ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('isExpectationReadyToStartWorkingOn', exp) + return this._sendMessage('isExpectationReadyToStartWorkingOn', exp) } async isExpectationFullfilled( exp: Expectation.Any, wasFullfilled: boolean ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('isExpectationFullfilled', exp, wasFullfilled) + return this._sendMessage('isExpectationFullfilled', exp, wasFullfilled) } async workOnExpectation( exp: Expectation.Any, cost: ExpectationManagerWorkerAgent.ExpectationCost ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('workOnExpectation', exp, cost) + return this._sendMessage('workOnExpectation', exp, cost) } async removeExpectation(exp: Expectation.Any): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('removeExpectation', exp) + return this._sendMessage('removeExpectation', exp) } async cancelWorkInProgress(wipId: number): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('cancelWorkInProgress', wipId) + return this._sendMessage('cancelWorkInProgress', wipId) } // PackageContainer-related methods: ---------------------------------------------------------------------------------------- @@ -72,24 +72,24 @@ export class WorkerAgentAPI packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('doYouSupportPackageContainer', packageContainer) + return this._sendMessage('doYouSupportPackageContainer', packageContainer) } async runPackageContainerCronJob( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('runPackageContainerCronJob', packageContainer) + return this._sendMessage('runPackageContainerCronJob', packageContainer) } async setupPackageContainerMonitors( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('setupPackageContainerMonitors', packageContainer) + return this._sendMessage('setupPackageContainerMonitors', packageContainer) } async disposePackageContainerMonitors( packageContainer: PackageContainerExpectation ): Promise { // Note: This call is ultimately received in shared/packages/worker/src/workerAgent.ts - return await this._sendMessage('disposePackageContainerMonitors', packageContainer) + return this._sendMessage('disposePackageContainerMonitors', packageContainer) } } diff --git a/shared/packages/expectationManager/src/workforceApi.ts b/shared/packages/expectationManager/src/workforceApi.ts index 9cdd2510..aa8a473b 100644 --- a/shared/packages/expectationManager/src/workforceApi.ts +++ b/shared/packages/expectationManager/src/workforceApi.ts @@ -1,4 +1,11 @@ -import { AdapterClient, WorkForceExpectationManager } from '@shared/api' +import { + WorkForceExpectationManager, + AdapterClient, + LoggerInstance, + LogLevel, + WorkforceStatus, + Expectation, +} from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the ExpectationManager @@ -8,11 +15,32 @@ import { AdapterClient, WorkForceExpectationManager } from '@shared/api' export class WorkforceAPI extends AdapterClient implements WorkForceExpectationManager.WorkForce { - constructor() { - super('expectationManager') + constructor(logger: LoggerInstance) { + super(logger, 'expectationManager') } + async registerExpectationManager(managerId: string, url: string): Promise { // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts - return await this._sendMessage('registerExpectationManager', managerId, url) + return this._sendMessage('registerExpectationManager', managerId, url) + } + async getStatus(): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('getStatus') + } + async setLogLevel(logLevel: LogLevel): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('setLogLevel', logLevel) + } + async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('setLogLevelOfApp', appId, logLevel) + } + async _debugKillApp(appId: string): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('_debugKillApp', appId) + } + async requestResources(exp: Expectation.Any): Promise { + // Note: This call is ultimately received in shared/packages/workforce/src/workforce.ts + return this._sendMessage('requestResources', exp) } } diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json index 0b6ff040..f83e2a97 100644 --- a/shared/packages/worker/package.json +++ b/shared/packages/worker/package.json @@ -11,7 +11,7 @@ "precommit": "lint-staged" }, "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "devDependencies": { "@types/deep-diff": "^1.0.0", @@ -40,4 +40,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/worker/src/appContainerApi.ts b/shared/packages/worker/src/appContainerApi.ts new file mode 100644 index 00000000..ad18abd4 --- /dev/null +++ b/shared/packages/worker/src/appContainerApi.ts @@ -0,0 +1,20 @@ +import { AdapterClient, LoggerInstance, AppContainerWorkerAgent } from '@shared/api' + +/** + * Exposes the API-methods of a Workforce, to be called from the WorkerAgent + * Note: The WorkerAgent connects to the Workforce, therefore the WorkerAgent is the AdapterClient here. + * The corresponding other side is implemented at shared/packages/workforce/src/workerAgentApi.ts + */ +export class AppContainerAPI + extends AdapterClient + implements AppContainerWorkerAgent.AppContainer { + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') + } + async ping(): Promise { + return this._sendMessage('ping') + } + async requestSpinDown(): Promise { + return this._sendMessage('requestSpinDown') + } +} diff --git a/shared/packages/worker/src/expectationManagerApi.ts b/shared/packages/worker/src/expectationManagerApi.ts index fba70133..5f2e62d2 100644 --- a/shared/packages/worker/src/expectationManagerApi.ts +++ b/shared/packages/worker/src/expectationManagerApi.ts @@ -1,4 +1,4 @@ -import { ExpectationManagerWorkerAgent, AdapterClient } from '@shared/api' +import { ExpectationManagerWorkerAgent, AdapterClient, LoggerInstance, Reason } from '@shared/api' /** * Exposes the API-methods of a ExpectationManager, to be called from the WorkerAgent @@ -8,23 +8,24 @@ import { ExpectationManagerWorkerAgent, AdapterClient } from '@shared/api' export class ExpectationManagerAPI extends AdapterClient implements ExpectationManagerWorkerAgent.ExpectationManager { - constructor() { - super('workerAgent') + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') } + async messageFromWorker(message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('messageFromWorker', message) + return this._sendMessage('messageFromWorker', message) } async wipEventProgress(wipId: number, actualVersionHash: string | null, progress: number): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventProgress', wipId, actualVersionHash, progress) + return this._sendMessage('wipEventProgress', wipId, actualVersionHash, progress) } - async wipEventDone(wipId: number, actualVersionHash: string, reason: string, result: unknown): Promise { + async wipEventDone(wipId: number, actualVersionHash: string, reason: Reason, result: unknown): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventDone', wipId, actualVersionHash, reason, result) + return this._sendMessage('wipEventDone', wipId, actualVersionHash, reason, result) } - async wipEventError(wipId: number, error: string): Promise { + async wipEventError(wipId: number, reason: Reason): Promise { // This call is ultimately received at shared/packages/expectationManager/src/workerAgentApi.ts - return await this._sendMessage('wipEventError', wipId, error) + return this._sendMessage('wipEventError', wipId, reason) } } diff --git a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts index f7121ff5..1f1347c0 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts @@ -1,9 +1,11 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' +import { assertNever } from '@shared/api' import { GenericWorker } from '../worker' import { CorePackageInfoAccessorHandle } from './corePackageInfo' import { FileShareAccessorHandle } from './fileShare' import { GenericAccessorHandle } from './genericHandle' import { HTTPAccessorHandle } from './http' +import { HTTPProxyAccessorHandle } from './httpProxy' import { LocalFolderAccessorHandle } from './localFolder' import { QuantelAccessorHandle } from './quantel' @@ -30,6 +32,8 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { return CorePackageInfoAccessorHandle } else if (accessor.type === Accessor.AccessType.HTTP) { return HTTPAccessorHandle + } else if (accessor.type === Accessor.AccessType.HTTP_PROXY) { + return HTTPProxyAccessorHandle } else if (accessor.type === Accessor.AccessType.FILE_SHARE) { return FileShareAccessorHandle } else if (accessor.type === Accessor.AccessType.QUANTEL) { @@ -39,9 +43,6 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { throw new Error(`Unsupported Accessor type "${accessor.type}"`) } } -function assertNever(_shouldBeNever: never) { - // Nothing -} export function isLocalFolderAccessorHandle( accessorHandler: GenericAccessorHandle @@ -53,10 +54,10 @@ export function isCorePackageInfoAccessorHandle( ): accessorHandler is CorePackageInfoAccessorHandle { return accessorHandler.type === CorePackageInfoAccessorHandle.type } -export function isHTTPAccessorHandle( +export function isHTTPProxyAccessorHandle( accessorHandler: GenericAccessorHandle -): accessorHandler is HTTPAccessorHandle { - return accessorHandler.type === HTTPAccessorHandle.type +): accessorHandler is HTTPProxyAccessorHandle { + return accessorHandler.type === HTTPProxyAccessorHandle.type } export function isFileShareAccessorHandle( accessorHandler: GenericAccessorHandle @@ -82,6 +83,7 @@ export function getAccessorCost(accessorType: Accessor.AccessType | undefined): // -------------------------------------------------------- case Accessor.AccessType.FILE_SHARE: return 2 + case Accessor.AccessType.HTTP_PROXY: case Accessor.AccessType.HTTP: return 3 diff --git a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts index 51861c91..916fdb13 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/corePackageInfo.ts @@ -1,6 +1,6 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { GenericAccessorHandle, PackageReadInfo, PutPackageHandler } from './genericHandle' -import { hashObj, Expectation } from '@shared/api' +import { GenericAccessorHandle, PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' +import { hashObj, Expectation, Reason } from '@shared/api' import { GenericWorker } from '../worker' /** Accessor handle for accessing data store in Core */ @@ -28,31 +28,37 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand static doYouSupportAccess(): boolean { return true // always has access } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { // Note: We assume that we always have write access here, no need to check this.accessor.allowRead return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { // Note: We assume that we always have write access here, no need to check this.accessor.allowWrite return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.CORE_PACKAGE_INFO) { - return `CorePackageInfo Accessor type is not CORE_PACKAGE_INFO ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `CorePackageInfo Accessor type is not CORE_PACKAGE_INFO ("${this.accessor.type}")!`, + }, + } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { // todo: add a check here? - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { // not needed - return undefined + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { // todo: add a check here? - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { throw new Error('getPackageActualVersion not applicable for CorePackageInfo') @@ -81,6 +87,9 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand async putPackageInfo(_readInfo: PackageReadInfo): Promise { throw new Error('CorePackageInfo.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + // do nothing + } async fetchMetadata(): Promise { throw new Error('fetchMetadata not applicable for CorePackageInfo') @@ -91,14 +100,29 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand async removeMetadata(): Promise { // Not applicable } - async runCronJob(): Promise { - return undefined // not applicable - } - async setupPackageContainerMonitors(): Promise { - return undefined // not applicable + async runCronJob(): Promise { + return { + success: false, + reason: { user: `There is an internal issue in Package Manager`, tech: 'runCronJob not supported' }, + } // not applicable } - async disposePackageContainerMonitors(): Promise { - return undefined // not applicable + async setupPackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'setupPackageContainerMonitors, not supported', + }, + } // not applicable + } + async disposePackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'disposePackageContainerMonitors, not supported', + }, + } // not applicable } public async findUnUpdatedPackageInfo( @@ -107,7 +131,7 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand content: unknown, actualSourceVersion: Expectation.Version.Any, expectTargetVersion: unknown - ): Promise<{ needsUpdate: boolean; reason: string }> { + ): Promise<{ needsUpdate: false } | { needsUpdate: true; reason: Reason }> { const actualContentVersionHash = this.getActualContentVersionHash( content, actualSourceVersion, @@ -128,24 +152,32 @@ export class CorePackageInfoAccessorHandle extends GenericAccessorHand if (!packageInfo) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" not found in PackageInfo store`, + reason: { + user: 'Package info needs to be stored', + tech: `Package "${fromPackage.id}" not found in PackageInfo store`, + }, } } else if (packageInfo.expectedContentVersionHash !== fromPackage.expectedContentVersionHash) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" expected version differs in PackageInfo store`, + reason: { + user: 'Package info needs to be updated', + tech: `Package "${fromPackage.id}" expected version differs in PackageInfo store`, + }, } } else if (packageInfo.actualContentVersionHash !== actualContentVersionHash) { return { needsUpdate: true, - reason: `Package "${fromPackage.id}" actual version differs in PackageInfo store`, + reason: { + user: 'Package info needs to be re-synced', + tech: `Package "${fromPackage.id}" actual version differs in PackageInfo store`, + }, } } } return { needsUpdate: false, - reason: `All packages in PackageInfo store are in sync`, } } public async updatePackageInfo( diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index 8710adc2..b280713f 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -1,13 +1,12 @@ import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { PackageReadInfo, PutPackageHandler } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import { WindowsWorker } from '../workers/windowsWorker/windowsWorker' import networkDrive from 'windows-network-drive' import { exec } from 'child_process' -import { assertNever } from '../lib/lib' import { FileShareAccessorHandleType, GenericFileAccessorHandle } from './lib/FileHandler' const fsStat = promisify(fs.stat) @@ -16,6 +15,8 @@ const fsOpen = promisify(fs.open) const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) +const fsRename = promisify(fs.rename) +const fsUnlink = promisify(fs.unlink) const pExec = promisify(exec) /** Accessor handle for accessing files on a network share */ @@ -28,10 +29,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } = {} private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean filePath?: string } - private workOptions: Expectation.WorkOptions.RemoveDelay + private workOptions: Expectation.WorkOptions.RemoveDelay & Expectation.WorkOptions.UseTemporaryFilePath constructor( worker: GenericWorker, @@ -48,8 +50,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') } this.content = content + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') throw new Error('Bad input data: workOptions.removeDelay is not a number!') + if (workOptions.useTemporaryFilePath && typeof workOptions.useTemporaryFilePath !== 'boolean') + throw new Error('Bad input data: workOptions.useTemporaryFilePath is not a boolean!') this.workOptions = workOptions } /** Path to the PackageContainer, ie the folder on the share */ @@ -65,32 +70,52 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle const accessor = accessor0 as AccessorOnPackage.FileShare return !accessor.networkId || worker.location.localNetworkIds.includes(accessor.networkId) } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { if (!this.accessor.allowRead) { - return `Not allowed to read` + return { + success: false, + reason: { + user: `Not allowed to read`, + tech: `Not allowed to read`, + }, + } } return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { if (!this.accessor.allowWrite) { - return `Not allowed to write` + return { + success: false, + reason: { + user: `Not allowed to write`, + tech: `Not allowed to write`, + }, + } } return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.FILE_SHARE) { - return `FileShare Accessor type is not FILE_SHARE ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `FileShare Accessor type is not FILE_SHARE ("${this.accessor.type}")!`, + }, + } } - if (!this.accessor.folderPath) return `Folder path not set` + if (!this.accessor.folderPath) + return { success: false, reason: { user: `Folder path not set`, tech: `Folder path not set` } } if (!this.content.onlyContainerAccess) { - if (!this.filePath) return `File path not set` + if (!this.filePath) + return { success: false, reason: { user: `File path not set`, tech: `File path not set` } } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { const readIssue = await this._checkPackageReadAccess() - if (readIssue) { - if (readIssue.match(/EPERM/)) { + if (!readIssue.success) { + if (readIssue.reason.tech.match(/EPERM/)) { // "EPERM: operation not permitted" if (this.accessor.userName) { // Try resetting the access permissions: @@ -103,9 +128,9 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return readIssue } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { try { // Check if we can open the file for reading: const fd = await fsOpen(this.fullPath, 'r+') @@ -114,16 +139,22 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle await fsClose(fd) } catch (err) { if (err && err.code === 'EBUSY') { - return `Not able to read file (busy)` + return { + success: false, + reason: { user: `Not able to read file (file is busy)`, tech: err.toString() }, + } } else if (err && err.code === 'ENOENT') { - return `File does not exist (ENOENT)` + return { success: false, reason: { user: `File does not exist`, tech: err.toString() } } } else { - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { user: `Not able to read file`, tech: err.toString() }, + } } } - return undefined // all good + return { success: true } } - private async _checkPackageReadAccess(): Promise { + private async _checkPackageReadAccess(): Promise { await this.prepareFileAccess() try { @@ -131,20 +162,32 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle // The file exists } catch (err) { // File is not readable - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { + user: `File doesn't exist`, + tech: `Not able to read file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { await this.prepareFileAccess() try { await fsAccess(this.folderPath, fs.constants.W_OK) // The file exists } catch (err) { - // File is not readable - return `Not able to write to file: ${err.toString()}` + // File is not writeable + return { + success: false, + reason: { + user: `Not able to write to file`, + tech: `Not able to write to file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { await this.prepareFileAccess() @@ -181,6 +224,19 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle await this.prepareFileAccess() await this.clearPackageRemoval(this.filePath) + const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath + + // Remove the file if it exists: + let exists = false + try { + await fsAccess(fullPath, fs.constants.R_OK) + // The file exists + exists = true + } catch (err) { + // Ignore + } + if (exists) await fsUnlink(fullPath) + const writeStream = sourceStream.pipe(fs.createWriteStream(this.fullPath)) const streamWrapper: PutPackageHandler = new PutPackageHandler(() => { @@ -201,6 +257,12 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle throw new Error('FileShare.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + if (this.workOptions.useTemporaryFilePath) { + await fsRename(this.temporaryFilePath, this.fullPath) + } + } + // Note: We handle metadata by storing a metadata json-file to the side of the file. async fetchMetadata(): Promise { @@ -223,22 +285,26 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle async removeMetadata(): Promise { await this.unlinkIfExists(this.metadataPath) } - async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] + let badReason: Reason | null = null for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return undefined + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } - async setupPackageContainerMonitors(packageContainerExp: PackageContainerExpectation): Promise { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -250,11 +316,11 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle } } - return undefined // all good + return { success: true } } async disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise { + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -265,17 +331,24 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle assertNever(monitor) } } - return undefined // all good + return { success: true } + } + /** Called when the package is supposed to be in place */ + async packageIsInPlace(): Promise { + await this.clearPackageRemoval(this.filePath) } /** Local path to the Package, ie the File */ - private get filePath(): string { + get filePath(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') const filePath = this.accessor.filePath || this.content.filePath if (!filePath) throw new Error(`FileShareAccessor: filePath not set!`) return filePath } - + /** Full path to a temporary file */ + get temporaryFilePath(): string { + return this.fullPath + '.pmtemp' + } private get metadataPath() { return this.getMetadataPath(this.filePath) } diff --git a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts index 8593093c..259a1034 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/genericHandle.ts @@ -1,6 +1,6 @@ import { AccessorOnPackage } from '@sofie-automation/blueprints-integration' import { EventEmitter } from 'events' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { Expectation, PackageContainerExpectation, Reason } from '@shared/api' import { GenericWorker } from '../worker' /** @@ -24,30 +24,30 @@ export abstract class GenericAccessorHandle { * Checks if there are any issues with the properties in the accessor or content for being able to read * @returns undefined if all is OK / string with error message */ - abstract checkHandleRead(): string | undefined + abstract checkHandleRead(): AccessorHandlerResult /** * Checks if there are any issues with the properties in the accessor or content for being able to write * @returns undefined if all is OK / string with error message */ - abstract checkHandleWrite(): string | undefined + abstract checkHandleWrite(): AccessorHandlerResult /** * Checks if Accesor has access to the Package, for reading. * Errors from this method are related to access/permission issues, or that the package doesn't exist. * @returns undefined if all is OK / string with error message */ - abstract checkPackageReadAccess(): Promise + abstract checkPackageReadAccess(): Promise /** * Do a check if it actually is possible to access the package. * Errors from this method are related to the actual access of the package (such as resource is busy). * @returns undefined if all is OK / string with error message */ - abstract tryPackageRead(): Promise + abstract tryPackageRead(): Promise /** * Checks if the PackageContainer can be written to * @returns undefined if all is OK / string with error message */ - abstract checkPackageContainerWriteAccess(): Promise + abstract checkPackageContainerWriteAccess(): Promise /** * Extracts and returns the version from the package * @returns the vesion of the package @@ -84,25 +84,28 @@ export abstract class GenericAccessorHandle { /** For accessors that supports readInfo: Pipe info about a package source (obtained from getPackageReadInfo()) */ abstract putPackageInfo(readInfo: PackageReadInfo): Promise + /** Finalize the package. To be called after a .putPackageStream() or putPackageInfo() has completed. */ + abstract finalizePackage(): Promise + /** * Performs a cronjob on the Package container * @returns undefined if all is OK / string with error message */ - abstract runCronJob(packageContainerExp: PackageContainerExpectation): Promise + abstract runCronJob(packageContainerExp: PackageContainerExpectation): Promise /** * Setup monitors on the Package container * @returns undefined if all is OK / string with error message */ abstract setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise + ): Promise /** * Tear down monitors on the Package container * @returns undefined if all is OK / string with error message */ abstract disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise + ): Promise protected setCache(key: string, value: T): T { if (!this.worker.accessorCache[this.type]) { @@ -129,6 +132,19 @@ export abstract class GenericAccessorHandle { } } +/** Default result returned from most accessorHandler-methods */ +export type AccessorHandlerResult = + | { + /** Whether the action was successful or not */ + success: true + } + | { + /** Whether the action was successful or not */ + success: false + /** If the action isn't successful, the reason why */ + reason: Reason + } + /** * A class emitted from putPackageStream() and putPackageInfo(), used to signal the progression of an ongoing write operation. * Users of this class are required to emit the events 'error' on error and 'close' upon completion diff --git a/shared/packages/worker/src/worker/accessorHandlers/http.ts b/shared/packages/worker/src/worker/accessorHandlers/http.ts index d11a93e7..5fd464b9 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/http.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/http.ts @@ -1,18 +1,24 @@ import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { GenericAccessorHandle, PackageReadInfo, PackageReadStream, PutPackageHandler } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { + GenericAccessorHandle, + PackageReadInfo, + PackageReadStream, + PutPackageHandler, + AccessorHandlerResult, +} from './genericHandle' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' import fetch from 'node-fetch' import FormData from 'form-data' import AbortController from 'abort-controller' -import { assertNever } from '../lib/lib' /** Accessor handle for accessing files in a local folder */ export class HTTPAccessorHandle extends GenericAccessorHandle { static readonly type = 'http' private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean - filePath?: string + path?: string } private workOptions: Expectation.WorkOptions.RemoveDelay constructor( @@ -29,6 +35,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { + async checkPackageReadAccess(): Promise { const header = await this.fetchHeader() if (header.status >= 400) { - return `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}` + return { + success: false, + reason: { + user: `Got error code ${header.status} when trying to fetch package`, + tech: `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}`, + }, + } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { - // TODO: how to do this? - return undefined + async tryPackageRead(): Promise { + // TODO: Do a OPTIONS request? + // 204 or 404 is "not found" + // Access-Control-Allow-Methods should contain GET + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { // todo: how to check this? - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { const header = await this.fetchHeader() @@ -89,38 +116,8 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { - await this.clearPackageRemoval() - - const formData = new FormData() - formData.append('file', sourceStream) - - const controller = new AbortController() - - const streamHandler: PutPackageHandler = new PutPackageHandler(() => { - controller.abort() - }) - - fetch(this.fullUrl, { - method: 'POST', - body: formData, // sourceStream.readStream, - signal: controller.signal, - }) - .then((result) => { - if (result.status >= 400) { - throw new Error( - `Upload file: Bad response: [${result.status}]: ${result.statusText} POST "${this.fullUrl}"` - ) - } - }) - .then(() => { - streamHandler.emit('close') - }) - .catch((error) => { - streamHandler.emit('error', error) - }) - - return streamHandler + async putPackageStream(_sourceStream: NodeJS.ReadableStream): Promise { + throw new Error('HTTP.putPackageStream: Not supported') } async getPackageReadInfo(): Promise<{ readInfo: PackageReadInfo; cancel: () => void }> { throw new Error('HTTP.getPackageReadInfo: Not supported') @@ -128,33 +125,40 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { throw new Error('HTTP.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + // do nothing + } async fetchMetadata(): Promise { - return this.fetchJSON(this.fullUrl + '_metadata.json') + return undefined } - async updateMetadata(metadata: Metadata): Promise { - await this.storeJSON(this.fullUrl + '_metadata.json', metadata) + async updateMetadata(_metadata: Metadata): Promise { + // Not supported } async removeMetadata(): Promise { - await this.deletePackageIfExists(this.fullUrl + '_metadata.json') + // Not supported } - async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + let badReason: Reason | null = null const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return undefined + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } - async setupPackageContainerMonitors(packageContainerExp: PackageContainerExpectation): Promise { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -166,39 +170,59 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + ): Promise { // todo: implement monitors - return undefined + return { success: true } } get fullUrl(): string { return [ this.baseUrl.replace(/\/$/, ''), // trim trailing slash - this.filePath.replace(/^\//, ''), // trim leading slash + this.path.replace(/^\//, ''), // trim leading slash ].join('/') } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.HTTP) { - return `HTTP Accessor type is not HTTP ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `HTTP Accessor type is not HTTP ("${this.accessor.type}")!`, + }, + } } - if (!this.accessor.baseUrl) return `Accessor baseUrl not set` + if (!this.accessor.baseUrl) + return { + success: false, + reason: { + user: `Accessor baseUrl not set`, + tech: `Accessor baseUrl not set`, + }, + } if (!this.content.onlyContainerAccess) { - if (!this.filePath) return `filePath not set` + if (!this.path) + return { + success: false, + reason: { + user: `filePath not set`, + tech: `filePath not set`, + }, + } } - return undefined // all good + return { success: true } } private get baseUrl(): string { if (!this.accessor.baseUrl) throw new Error(`HTTPAccessorHandle: accessor.baseUrl not set!`) return this.accessor.baseUrl } - private get filePath(): string { + get path(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') - const filePath = this.accessor.url || this.content.filePath - if (!filePath) throw new Error(`HTTPAccessorHandle: filePath not set!`) + const filePath = this.accessor.url || this.content.path + if (!filePath) throw new Error(`HTTPAccessorHandle: path not set!`) return filePath } private convertHeadersToVersion(headers: HTTPHeaders): Expectation.Version.HTTPFile { @@ -215,6 +239,10 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + // Swallow the error. Since we're aborting the request, we're not interested in the body anyway. + }) + const headers: HTTPHeaders = { contentType: res.headers.get('content-type'), contentLength: res.headers.get('content-length'), @@ -234,7 +262,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { const packagesToRemove = await this.getPackagesToRemove() - const filePath = this.filePath + const filePath = this.path // Search for a pre-existing entry: let found = false @@ -260,7 +288,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { const packagesToRemove = await this.getPackagesToRemove() - const filePath = this.filePath + const filePath = this.path let found = false for (let i = 0; i < packagesToRemove.length; i++) { @@ -276,7 +304,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle { + async removeDuePackages(): Promise { let packagesToRemove = await this.getPackagesToRemove() const removedFilePaths: string[] = [] @@ -289,7 +317,7 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { const result = await fetch(url, { @@ -360,6 +389,10 @@ export class HTTPAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle { + static readonly type = 'http-proxy' + private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ + onlyContainerAccess?: boolean + filePath?: string + } + private workOptions: Expectation.WorkOptions.RemoveDelay + constructor( + worker: GenericWorker, + public readonly accessorId: string, + private accessor: AccessorOnPackage.HTTPProxy, + content: any, // eslint-disable-line @typescript-eslint/explicit-module-boundary-types + workOptions: any // eslint-disable-line @typescript-eslint/explicit-module-boundary-types + ) { + super(worker, accessorId, accessor, content, HTTPProxyAccessorHandle.type) + + // Verify content data: + if (!content.onlyContainerAccess) { + if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') + } + this.content = content + + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') + throw new Error('Bad input data: workOptions.removeDelay is not a number!') + this.workOptions = workOptions + } + static doYouSupportAccess(worker: GenericWorker, accessor0: AccessorOnPackage.Any): boolean { + const accessor = accessor0 as AccessorOnPackage.HTTP + return !accessor.networkId || worker.location.localNetworkIds.includes(accessor.networkId) + } + checkHandleRead(): AccessorHandlerResult { + if (!this.accessor.allowRead) { + return { + success: false, + reason: { + user: `Not allowed to read`, + tech: `Not allowed to read`, + }, + } + } + return this.checkAccessor() + } + checkHandleWrite(): AccessorHandlerResult { + if (!this.accessor.allowWrite) { + return { + success: false, + reason: { + user: `Not allowed to write`, + tech: `Not allowed to write`, + }, + } + } + return this.checkAccessor() + } + async checkPackageReadAccess(): Promise { + const header = await this.fetchHeader() + + if (header.status >= 400) { + return { + success: false, + reason: { + user: `Got error code ${header.status} when trying to fetch package`, + tech: `Error when requesting url "${this.fullUrl}": [${header.status}]: ${header.statusText}`, + }, + } + } + return { success: true } + } + async tryPackageRead(): Promise { + // TODO: how to do this? + return { success: true } + } + async checkPackageContainerWriteAccess(): Promise { + // todo: how to check this? + return { success: true } + } + async getPackageActualVersion(): Promise { + const header = await this.fetchHeader() + + return this.convertHeadersToVersion(header.headers) + } + async removePackage(): Promise { + if (this.workOptions.removeDelay) { + await this.delayPackageRemoval(this.workOptions.removeDelay) + } else { + await this.removeMetadata() + await this.deletePackageIfExists(this.fullUrl) + } + } + async getPackageReadStream(): Promise { + const controller = new AbortController() + const res = await fetch(this.fullUrl, { signal: controller.signal }) + + return { + readStream: res.body, + cancel: () => { + controller.abort() + }, + } + } + async putPackageStream(sourceStream: NodeJS.ReadableStream): Promise { + await this.clearPackageRemoval() + + const formData = new FormData() + formData.append('file', sourceStream) + + const controller = new AbortController() + + const streamHandler: PutPackageHandler = new PutPackageHandler(() => { + controller.abort() + }) + + fetch(this.fullUrl, { + method: 'POST', + body: formData, // sourceStream.readStream, + signal: controller.signal, + }) + .then((result) => { + if (result.status >= 400) { + throw new Error( + `Upload file: Bad response: [${result.status}]: ${result.statusText} POST "${this.fullUrl}"` + ) + } + }) + .then(() => { + streamHandler.emit('close') + }) + .catch((error) => { + streamHandler.emit('error', error) + }) + + return streamHandler + } + async getPackageReadInfo(): Promise<{ readInfo: PackageReadInfo; cancel: () => void }> { + throw new Error('HTTP.getPackageReadInfo: Not supported') + } + async putPackageInfo(_readInfo: PackageReadInfo): Promise { + throw new Error('HTTP.putPackageInfo: Not supported') + } + async finalizePackage(): Promise { + // do nothing + } + + async fetchMetadata(): Promise { + return this.fetchJSON(this.getMetadataPath(this.fullUrl)) + } + async updateMetadata(metadata: Metadata): Promise { + await this.storeJSON(this.getMetadataPath(this.fullUrl), metadata) + } + async removeMetadata(): Promise { + await this.deletePackageIfExists(this.getMetadataPath(this.fullUrl)) + } + + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + let badReason: Reason | null = null + const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] + for (const cronjob of cronjobs) { + if (cronjob === 'interval') { + // ignore + } else if (cronjob === 'cleanup') { + badReason = await this.removeDuePackages() + } else { + // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: + assertNever(cronjob) + } + } + + if (!badReason) return { success: true } + else return { success: false, reason: badReason } + } + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { + const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] + for (const monitor of monitors) { + if (monitor === 'packages') { + // todo: implement monitors + throw new Error('Not implemented yet') + } else { + // Assert that cronjob is of type "never", to ensure that all types of monitors are handled: + assertNever(monitor) + } + } + + return { success: true } + } + async disposePackageContainerMonitors( + _packageContainerExp: PackageContainerExpectation + ): Promise { + // todo: implement monitors + return { success: true } + } + get fullUrl(): string { + return [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + this.filePath.replace(/^\//, ''), // trim leading slash + ].join('/') + } + + private checkAccessor(): AccessorHandlerResult { + if (this.accessor.type !== Accessor.AccessType.HTTP_PROXY) { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `HTTPProxy Accessor type is not HTTP_PROXY ("${this.accessor.type}")!`, + }, + } + } + if (!this.accessor.baseUrl) + return { + success: false, + reason: { + user: `Accessor baseUrl not set`, + tech: `Accessor baseUrl not set`, + }, + } + if (!this.content.onlyContainerAccess) { + if (!this.filePath) + return { + success: false, + reason: { + user: `filePath not set`, + tech: `filePath not set`, + }, + } + } + return { success: true } + } + private get baseUrl(): string { + if (!this.accessor.baseUrl) throw new Error(`HTTPAccessorHandle: accessor.baseUrl not set!`) + return this.accessor.baseUrl + } + get filePath(): string { + if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') + const filePath = this.accessor.url || this.content.filePath + if (!filePath) throw new Error(`HTTPAccessorHandle: filePath not set!`) + return filePath + } + private convertHeadersToVersion(headers: HTTPHeaders): Expectation.Version.HTTPFile { + return { + type: Expectation.Version.Type.HTTP_FILE, + + contentType: headers.contentType || '', + contentLength: parseInt(headers.contentLength || '0', 10) || 0, + modified: headers.lastModified ? new Date(headers.lastModified).getTime() : 0, + etags: [], // headers.etags, // todo! + } + } + private async fetchHeader() { + const controller = new AbortController() + const res = await fetch(this.fullUrl, { signal: controller.signal }) + + res.body.on('error', () => { + // Swallow the error. Since we're aborting the request, we're not interested in the body anyway. + }) + + const headers: HTTPHeaders = { + contentType: res.headers.get('content-type'), + contentLength: res.headers.get('content-length'), + lastModified: res.headers.get('last-modified'), + etags: res.headers.get('etag'), + } + // We've got the headers, abort the call so we don't have to download the whole file: + controller.abort() + + return { + status: res.status, + statusText: res.statusText, + headers: headers, + } + } + + async delayPackageRemoval(ttl: number): Promise { + const packagesToRemove = await this.getPackagesToRemove() + + const filePath = this.filePath + + // Search for a pre-existing entry: + let found = false + for (const entry of packagesToRemove) { + if (entry.filePath === filePath) { + // extend the TTL if it was found: + entry.removeTime = Date.now() + ttl + + found = true + break + } + } + if (!found) { + packagesToRemove.push({ + filePath: filePath, + removeTime: Date.now() + ttl, + }) + } + + await this.storePackagesToRemove(packagesToRemove) + } + /** Clear a scheduled later removal of a package */ + async clearPackageRemoval(): Promise { + const packagesToRemove = await this.getPackagesToRemove() + + const filePath = this.filePath + + let found = false + for (let i = 0; i < packagesToRemove.length; i++) { + const entry = packagesToRemove[i] + if (entry.filePath === filePath) { + packagesToRemove.splice(i, 1) + found = true + break + } + } + if (found) { + await this.storePackagesToRemove(packagesToRemove) + } + } + /** Remove any packages that are due for removal */ + async removeDuePackages(): Promise { + let packagesToRemove = await this.getPackagesToRemove() + + const removedFilePaths: string[] = [] + for (const entry of packagesToRemove) { + // Check if it is time to remove the package: + if (entry.removeTime < Date.now()) { + // it is time to remove the package: + const fullUrl: string = [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + entry.filePath, + ].join('/') + + await this.deletePackageIfExists(this.getMetadataPath(fullUrl)) + await this.deletePackageIfExists(fullUrl) + removedFilePaths.push(entry.filePath) + } + } + + // Fetch again, to decrease the risk of race-conditions: + packagesToRemove = await this.getPackagesToRemove() + let changed = false + // Remove paths from array: + for (let i = 0; i < packagesToRemove.length; i++) { + const entry = packagesToRemove[i] + if (removedFilePaths.includes(entry.filePath)) { + packagesToRemove.splice(i, 1) + changed = true + break + } + } + if (changed) { + await this.storePackagesToRemove(packagesToRemove) + } + return null + } + private async deletePackageIfExists(url: string): Promise { + const result = await fetch(url, { + method: 'DELETE', + }) + if (result.status === 404) return undefined // that's ok + if (result.status >= 400) { + const text = await result.text() + throw new Error( + `deletePackageIfExists: Bad response: [${result.status}]: ${result.statusText}, DELETE ${this.fullUrl}, ${text}` + ) + } + } + /** Full path to the file containing deferred removals */ + private get deferRemovePackagesPath(): string { + return [ + this.baseUrl.replace(/\/$/, ''), // trim trailing slash + '__removePackages.json', + ].join('/') + } + /** */ + private async getPackagesToRemove(): Promise { + return (await this.fetchJSON(this.deferRemovePackagesPath)) ?? [] + } + private async storePackagesToRemove(packagesToRemove: DelayPackageRemovalEntry[]): Promise { + await this.storeJSON(this.deferRemovePackagesPath, packagesToRemove) + } + private async fetchJSON(url: string): Promise { + const result = await fetch(url) + if (result.status === 404) return undefined + if (result.status >= 400) { + const text = await result.text() + throw new Error( + `getPackagesToRemove: Bad response: [${result.status}]: ${result.statusText}, GET ${url}, ${text}` + ) + } + return result.json() + } + private async storeJSON(url: string, data: any): Promise { + const formData = new FormData() + formData.append('text', JSON.stringify(data)) + const result = await fetch(url, { + method: 'POST', + body: formData, + }) + if (result.status >= 400) { + const text = await result.text() + throw new Error(`storeJSON: Bad response: [${result.status}]: ${result.statusText}, POST ${url}, ${text}`) + } + } + /** Full path to the metadata file */ + private getMetadataPath(fullUrl: string) { + return fullUrl + '_metadata.json' + } +} +interface HTTPHeaders { + contentType: string | null + contentLength: string | null + lastModified: string | null + etags: string | null +} + +interface DelayPackageRemovalEntry { + /** Local file path */ + filePath: string + /** Unix timestamp for when it's clear to remove the file */ + removeTime: number +} diff --git a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts index 712aa47e..a184f9a9 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts @@ -1,12 +1,11 @@ import path from 'path' import { promisify } from 'util' import fs from 'fs' -import { Expectation, hashObj, literal, PackageContainerExpectation } from '@shared/api' +import { Expectation, hashObj, literal, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import chokidar from 'chokidar' import { GenericWorker } from '../../worker' import { Accessor, AccessorOnPackage, ExpectedPackage } from '@sofie-automation/blueprints-integration' import { GenericAccessorHandle } from '../genericHandle' -import { assertNever } from '../../lib/lib' export const LocalFolderAccessorHandleType = 'localFolder' export const FileShareAccessorHandleType = 'fileShare' @@ -75,7 +74,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso } } /** Remove any packages that are due for removal */ - async removeDuePackages(): Promise { + async removeDuePackages(): Promise { let packagesToRemove = await this.getPackagesToRemove() const removedFilePaths: string[] = [] @@ -108,6 +107,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso if (changed) { await this.storePackagesToRemove(packagesToRemove) } + return null } /** Unlink (remove) a file, if it exists. */ async unlinkIfExists(filePath: string): Promise { @@ -132,11 +132,22 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso const options = packageContainerExp.monitors.packages if (!options) throw new Error('Options not set (this should never happen)') - const watcher = chokidar.watch(this.folderPath, { + const chokidarOptions: chokidar.WatchOptions = { ignored: options.ignore ? new RegExp(options.ignore) : undefined, - persistent: true, - }) + } + if (options.usePolling) { + chokidarOptions.usePolling = true + chokidarOptions.interval = 2000 + chokidarOptions.binaryInterval = 2000 + } + if (options.awaitWriteFinishStabilityThreshold) { + chokidarOptions.awaitWriteFinish = { + stabilityThreshold: options.awaitWriteFinishStabilityThreshold, + pollInterval: options.awaitWriteFinishStabilityThreshold, + } + } + const watcher = chokidar.watch(this.folderPath, chokidarOptions) const monitorId = `${this.worker.genericConfig.workerId}_${this.worker.uniqueId}_${Date.now()}` const seenFiles = new Map() @@ -170,7 +181,8 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso seenFiles.set(filePath, version) } catch (err) { - console.log('error', err) + version = null + this.worker.logger.error(err) } } @@ -195,6 +207,9 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso ], sideEffect: options.sideEffect, } + if (!expPackage.sources[0].accessors) { + expPackage.sources[0].accessors = {} + } if (this._type === LocalFolderAccessorHandleType) { expPackage.sources[0].accessors[ this.accessorId @@ -225,7 +240,7 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso triggerSendUpdateIsRunning = false })().catch((err) => { triggerSendUpdateIsRunning = false - console.log('error', err) + this.worker.logger.error(err) }) }, 1000) // Wait just a little bit, to avoid doing multiple updates } @@ -242,15 +257,33 @@ export abstract class GenericFileAccessorHandle extends GenericAccesso triggerSendUpdate() } }) - .on('unlink', (fullPath) => { + .on('change', (fullPath) => { const localPath = getFilePath(fullPath) if (localPath) { - seenFiles.delete(localPath) + seenFiles.set(localPath, null) // This will cause triggerSendUpdate() to update the version triggerSendUpdate() } }) + .on('unlink', (fullPath) => { + // We don't trust chokidar, so we'll check it ourselves first.. + // (We've seen an issue where removing a single file from a folder causes chokidar to emit unlink for ALL the files) + fsAccess(fullPath, fs.constants.R_OK) + .then(() => { + // The file seems to exist, even though chokidar says it doesn't. + // Ignore the event, then + }) + .catch(() => { + // The file truly doesn't exist + + const localPath = getFilePath(fullPath) + if (localPath) { + seenFiles.delete(localPath) + triggerSendUpdate() + } + }) + }) .on('error', (error) => { - console.log('error', error) + this.worker.logger.error(error.toString()) }) /** Persistant store for Monitors */ diff --git a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts index f69f3f2a..edec1184 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/localFolder.ts @@ -2,10 +2,9 @@ import path from 'path' import { promisify } from 'util' import fs from 'fs' import { Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration' -import { PackageReadInfo, PutPackageHandler } from './genericHandle' -import { Expectation, PackageContainerExpectation } from '@shared/api' +import { PackageReadInfo, PutPackageHandler, AccessorHandlerResult } from './genericHandle' +import { Expectation, PackageContainerExpectation, assertNever, Reason } from '@shared/api' import { GenericWorker } from '../worker' -import { assertNever } from '../lib/lib' import { GenericFileAccessorHandle, LocalFolderAccessorHandleType } from './lib/FileHandler' const fsStat = promisify(fs.stat) @@ -14,16 +13,19 @@ const fsOpen = promisify(fs.open) const fsClose = promisify(fs.close) const fsReadFile = promisify(fs.readFile) const fsWriteFile = promisify(fs.writeFile) +const fsRename = promisify(fs.rename) +const fsUnlink = promisify(fs.unlink) /** Accessor handle for accessing files in a local folder */ export class LocalFolderAccessorHandle extends GenericFileAccessorHandle { static readonly type = LocalFolderAccessorHandleType private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean filePath?: string } - private workOptions: Expectation.WorkOptions.RemoveDelay + private workOptions: Expectation.WorkOptions.RemoveDelay & Expectation.WorkOptions.UseTemporaryFilePath constructor( worker: GenericWorker, @@ -39,8 +41,11 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand if (!content.filePath) throw new Error('Bad input data: content.filePath not set!') } this.content = content + if (workOptions.removeDelay && typeof workOptions.removeDelay !== 'number') throw new Error('Bad input data: workOptions.removeDelay is not a number!') + if (workOptions.useTemporaryFilePath && typeof workOptions.useTemporaryFilePath !== 'boolean') + throw new Error('Bad input data: workOptions.useTemporaryFilePath is not a boolean!') this.workOptions = workOptions } static doYouSupportAccess(worker: GenericWorker, accessor0: AccessorOnPackage.Any): boolean { @@ -52,39 +57,53 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand return path.join(this.folderPath, this.filePath) } - checkHandleRead(): string | undefined { + checkHandleRead(): AccessorHandlerResult { if (!this.accessor.allowRead) { - return `Not allowed to read` + return { success: false, reason: { user: `Not allowed to read`, tech: `Not allowed to read` } } } return this.checkAccessor() } - checkHandleWrite(): string | undefined { + checkHandleWrite(): AccessorHandlerResult { if (!this.accessor.allowWrite) { - return `Not allowed to write` + return { success: false, reason: { user: `Not allowed to write`, tech: `Not allowed to write` } } } return this.checkAccessor() } - private checkAccessor(): string | undefined { + private checkAccessor(): AccessorHandlerResult { if (this.accessor.type !== Accessor.AccessType.LOCAL_FOLDER) { - return `LocalFolder Accessor type is not LOCAL_FOLDER ("${this.accessor.type}")!` + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: `LocalFolder Accessor type is not LOCAL_FOLDER ("${this.accessor.type}")!`, + }, + } } - if (!this.accessor.folderPath) return `Folder path not set` + if (!this.accessor.folderPath) + return { success: false, reason: { user: `Folder path not set`, tech: `Folder path not set` } } if (!this.content.onlyContainerAccess) { - if (!this.filePath) return `File path not set` + if (!this.filePath) + return { success: false, reason: { user: `File path not set`, tech: `File path not set` } } } - return undefined // all good + return { success: true } } - async checkPackageReadAccess(): Promise { + async checkPackageReadAccess(): Promise { try { await fsAccess(this.fullPath, fs.constants.R_OK) - // The file exists + // The file exists and can be read } catch (err) { // File is not readable - return `Not able to access file: ${err.toString()}` + return { + success: false, + reason: { + user: `File doesn't exist`, + tech: `Not able to access file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } - async tryPackageRead(): Promise { + async tryPackageRead(): Promise { try { // Check if we can open the file for reading: const fd = await fsOpen(this.fullPath, 'r+') @@ -93,24 +112,36 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand await fsClose(fd) } catch (err) { if (err && err.code === 'EBUSY') { - return `Not able to read file (busy)` + return { + success: false, + reason: { user: `Not able to read file (file is busy)`, tech: err.toString() }, + } } else if (err && err.code === 'ENOENT') { - return `File does not exist (ENOENT)` + return { success: false, reason: { user: `File does not exist`, tech: err.toString() } } } else { - return `Not able to read file: ${err.toString()}` + return { + success: false, + reason: { user: `Not able to read file`, tech: err.toString() }, + } } } - return undefined // all good + return { success: true } } - async checkPackageContainerWriteAccess(): Promise { + async checkPackageContainerWriteAccess(): Promise { try { await fsAccess(this.folderPath, fs.constants.W_OK) // The file exists } catch (err) { - // File is not readable - return `Not able to write to file: ${err.toString()}` + // File is not writeable + return { + success: false, + reason: { + user: `Not able to write to file`, + tech: `Not able to write to file: ${err.toString()}`, + }, + } } - return undefined // all good + return { success: true } } async getPackageActualVersion(): Promise { const stat = await fsStat(this.fullPath) @@ -142,7 +173,20 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand async putPackageStream(sourceStream: NodeJS.ReadableStream): Promise { await this.clearPackageRemoval(this.filePath) - const writeStream = sourceStream.pipe(fs.createWriteStream(this.fullPath)) + const fullPath = this.workOptions.useTemporaryFilePath ? this.temporaryFilePath : this.fullPath + + // Remove the file if it exists: + let exists = false + try { + await fsAccess(fullPath, fs.constants.R_OK) + // The file exists + exists = true + } catch (err) { + // Ignore + } + if (exists) await fsUnlink(fullPath) + + const writeStream = sourceStream.pipe(fs.createWriteStream(fullPath)) const streamWrapper: PutPackageHandler = new PutPackageHandler(() => { writeStream.destroy() @@ -162,6 +206,12 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand throw new Error('LocalFolder.putPackageInfo: Not supported') } + async finalizePackage(): Promise { + if (this.workOptions.useTemporaryFilePath) { + await fsRename(this.temporaryFilePath, this.fullPath) + } + } + // Note: We handle metadata by storing a metadata json-file to the side of the file. async fetchMetadata(): Promise { @@ -184,22 +234,26 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand async removeMetadata(): Promise { await this.unlinkIfExists(this.metadataPath) } - async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + let badReason: Reason | null = null const cronjobs = Object.keys(packageContainerExp.cronjobs) as (keyof PackageContainerExpectation['cronjobs'])[] for (const cronjob of cronjobs) { if (cronjob === 'interval') { // ignore } else if (cronjob === 'cleanup') { - await this.removeDuePackages() + badReason = await this.removeDuePackages() } else { // Assert that cronjob is of type "never", to ensure that all types of cronjobs are handled: assertNever(cronjob) } } - return undefined + if (!badReason) return { success: true } + else return { success: false, reason: badReason } } - async setupPackageContainerMonitors(packageContainerExp: PackageContainerExpectation): Promise { + async setupPackageContainerMonitors( + packageContainerExp: PackageContainerExpectation + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -211,11 +265,11 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand } } - return undefined // all good + return { success: true } } async disposePackageContainerMonitors( packageContainerExp: PackageContainerExpectation - ): Promise { + ): Promise { const monitors = Object.keys(packageContainerExp.monitors) as (keyof PackageContainerExpectation['monitors'])[] for (const monitor of monitors) { if (monitor === 'packages') { @@ -226,7 +280,7 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand assertNever(monitor) } } - return undefined // all good + return { success: true } } /** Called when the package is supposed to be in place */ @@ -240,12 +294,16 @@ export class LocalFolderAccessorHandle extends GenericFileAccessorHand return this.accessor.folderPath } /** Local path to the Package, ie the File */ - private get filePath(): string { + get filePath(): string { if (this.content.onlyContainerAccess) throw new Error('onlyContainerAccess is set!') const filePath = this.accessor.filePath || this.content.filePath if (!filePath) throw new Error(`LocalFolderAccessor: filePath not set!`) return filePath } + /** Full path to a temporary file */ + get temporaryFilePath(): string { + return this.fullPath + '.pmtemp' + } /** Full path to the metadata file */ private get metadataPath() { return this.fullPath + '_metadata.json' diff --git a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts index 30a47824..498797a1 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/quantel.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/quantel.ts @@ -6,8 +6,9 @@ import { PackageReadInfoBaseType, PackageReadInfoQuantelClip, PutPackageHandler, + AccessorHandlerResult, } from './genericHandle' -import { Expectation, literal } from '@shared/api' +import { Expectation, literal, Reason } from '@shared/api' import { GenericWorker } from '../worker' import { ClipData, ClipDataSummary, ServerInfo } from 'tv-automation-quantel-gateway-client/dist/quantelTypes' @@ -18,12 +19,13 @@ const MINIMUM_FRAMES = 10 export class QuantelAccessorHandle extends GenericAccessorHandle { static readonly type = 'quantel' private content: { + /** This is set when the class-instance is only going to be used for PackageContainer access.*/ onlyContainerAccess?: boolean guid?: string title?: string } // @ts-expect-error unused variable - private workOptions: any // {} + private workOptions: any constructor( worker: GenericWorker, accessorId: string, @@ -41,44 +43,70 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async checkPackageReadAccess(): Promise { const quantel = await this.getQuantelGateway() // Search for a clip that match: @@ -86,24 +114,42 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async tryPackageRead(): Promise { const quantel = await this.getQuantelGateway() const clipSummary = await this.searchForLatestClip(quantel) - if (!clipSummary) return `No clip found` + if (!clipSummary) return { success: false, reason: { user: `No clip found`, tech: `No clip found` } } if (!parseInt(clipSummary.Frames, 10)) { - return `Clip "${clipSummary.ClipGUID}" has no frames` + return { + success: false, + reason: { + user: `The clip has no frames`, + tech: `Clip "${clipSummary.ClipGUID}" has no frames`, + }, + } } if (parseInt(clipSummary.Frames, 10) < MINIMUM_FRAMES) { // Check that it is meaningfully playable - return `Clip "${clipSummary.ClipGUID}" hasn't received enough frames` + return { + success: false, + reason: { + user: `The clip hasn't received enough frames`, + tech: `Clip "${clipSummary.ClipGUID}" hasn't received enough frames (${clipSummary.Frames})`, + }, + } } // 5/5/21: Removed check for completed - OA tests shoes it does nothing for placeholders / Richard // if (!clipSummary.Completed || !clipSummary.Completed.length) { @@ -111,9 +157,9 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + async checkPackageContainerWriteAccess(): Promise { const quantel = await this.getQuantelGateway() const server = await quantel.getServer() @@ -124,7 +170,7 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { const quantel = await this.getQuantelGateway() @@ -234,6 +280,9 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { + // do nothing + } async fetchMetadata(): Promise { throw new Error('Quantel.fetchMetadata: Not supported') @@ -244,33 +293,61 @@ export class QuantelAccessorHandle extends GenericAccessorHandle { // Not supported, do nothing } - async runCronJob(): Promise { - return undefined // not applicable + async runCronJob(): Promise { + return { + success: false, + reason: { user: `There is an internal issue in Package Manager`, tech: 'runCronJob not supported' }, + } // not applicable } - async setupPackageContainerMonitors(): Promise { - return undefined // not applicable + async setupPackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'setupPackageContainerMonitors, not supported', + }, + } // not applicable } - async disposePackageContainerMonitors(): Promise { - return undefined // not applicable + async disposePackageContainerMonitors(): Promise { + return { + success: false, + reason: { + user: `There is an internal issue in Package Manager`, + tech: 'disposePackageContainerMonitors, not supported', + }, + } // not applicable } async getClip(): Promise { const quantel = await this.getQuantelGateway() - return await this.searchForLatestClip(quantel) + return this.searchForLatestClip(quantel) } async getClipDetails(clipId: number): Promise { const quantel = await this.getQuantelGateway() - return await quantel.getClip(clipId) + return quantel.getClip(clipId) } - async getTransformerStreamURL(): Promise<{ baseURL: string; url: string; fullURL: string } | undefined> { - if (!this.accessor.transformerURL) return undefined + get transformerURL(): string | undefined { + return this.accessor.transformerURL + } + async getTransformerStreamURL(): Promise< + { success: true; baseURL: string; url: string; fullURL: string } | { success: false; reason: Reason } + > { + if (!this.accessor.transformerURL) + return { + success: false, + reason: { + user: `transformerURL is not set in settings`, + tech: `transformerURL not set on accessor ${this.accessorId}`, + }, + } const clip = await this.getClip() if (clip) { const baseURL = this.accessor.transformerURL const url = `/quantel/homezone/clips/streams/${clip.ClipID}/stream.mpd` return { + success: true, baseURL, url, fullURL: [ @@ -278,8 +355,15 @@ export class QuantelAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle extends GenericAccessorHandle console.log(`Quantel.QuantelGateway`, e)) - // @todo: We should be able to emit statuses somehow: - // gateway.monitorServerStatus(() => {}) + gateway.on('error', (e) => this.worker.logger.error(`Quantel.QuantelGateway`, e)) cacheGateways[id] = gateway } diff --git a/shared/packages/worker/src/worker/lib/expectationHandler.ts b/shared/packages/worker/src/worker/lib/expectationHandler.ts index 22746146..55e5e9a3 100644 --- a/shared/packages/worker/src/worker/lib/expectationHandler.ts +++ b/shared/packages/worker/src/worker/lib/expectationHandler.ts @@ -45,6 +45,7 @@ export interface ExpectationHandler { */ isExpectationFullfilled: ( exp: Expectation.Any, + /** If the caller believes that the expectation was fullfilled before */ wasFullfilled: boolean, genericWorker: GenericWorker, specificWorker: any diff --git a/shared/packages/worker/src/worker/lib/lib.ts b/shared/packages/worker/src/worker/lib/lib.ts index d5531233..b2d6dd31 100644 --- a/shared/packages/worker/src/worker/lib/lib.ts +++ b/shared/packages/worker/src/worker/lib/lib.ts @@ -4,7 +4,8 @@ const accessorTypePriority: { [key: string]: number } = { [Accessor.AccessType.LOCAL_FOLDER]: 0, [Accessor.AccessType.QUANTEL]: 1, [Accessor.AccessType.FILE_SHARE]: 2, - [Accessor.AccessType.HTTP]: 3, + [Accessor.AccessType.HTTP_PROXY]: 3, + [Accessor.AccessType.HTTP]: 4, [Accessor.AccessType.CORE_PACKAGE_INFO]: 99999, } @@ -37,8 +38,3 @@ export interface AccessorWithPackageContainer void - done: (actualVersionHash: string, reason: string, result: any) => void - error: (error: string) => void + done: (actualVersionHash: string, reason: Reason, result: any) => void + error: (reason: string) => void } export declare interface IWorkInProgress { properties: ExpectationManagerWorkerAgent.WorkInProgressProperties @@ -49,7 +49,7 @@ export class WorkInProgress extends EventEmitter implements IWorkInProgress { } } // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - _reportComplete(actualVersionHash: string, reason: string, result: any): void { + _reportComplete(actualVersionHash: string, reason: Reason, result: any): void { this.emit('done', actualVersionHash, reason, result) } _reportError(err: Error): void { diff --git a/shared/packages/worker/src/worker/worker.ts b/shared/packages/worker/src/worker/worker.ts index c2aa14d6..249848a5 100644 --- a/shared/packages/worker/src/worker/worker.ts +++ b/shared/packages/worker/src/worker/worker.ts @@ -1,6 +1,7 @@ import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -24,6 +25,7 @@ export abstract class GenericWorker { private _uniqueId = 0 constructor( + public logger: LoggerInstance, public readonly genericConfig: WorkerAgentConfig, public readonly location: WorkerLocation, public sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, @@ -36,6 +38,9 @@ export abstract class GenericWorker { } /** Called upon startup */ abstract init(): Promise + + /** Called upon termination */ + abstract terminate(): void /** * Does the worker support this expectation? * This includes things like: diff --git a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts index 200701d5..d98d44c7 100644 --- a/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts +++ b/shared/packages/worker/src/worker/workers/linuxWorker/linuxWorker.ts @@ -2,6 +2,7 @@ import { IWorkInProgress } from '../../lib/workInProgress' import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -21,21 +22,25 @@ import { GenericWorker, WorkerLocation } from '../../worker' export class LinuxWorker extends GenericWorker { static readonly type = 'linuxWorker' constructor( + logger: LoggerInstance, public readonly config: WorkerAgentConfig, sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, location: WorkerLocation ) { - super(config, location, sendMessageToManager, LinuxWorker.type) + super(logger, config, location, sendMessageToManager, LinuxWorker.type) } async doYouSupportExpectation(_exp: Expectation.Any): Promise { return { support: false, - reason: `Not implemented yet`, + reason: { user: `Not implemented yet`, tech: `Not implemented yet` }, } } async init(): Promise { throw new Error(`Not implemented yet`) } + terminate(): void { + throw new Error(`Not implemented yet`) + } getCostFortExpectation(_exp: Expectation.Any): Promise { throw new Error(`Not implemented yet`) } @@ -60,7 +65,7 @@ export class LinuxWorker extends GenericWorker { ): Promise { return { support: false, - reason: `Not implemented yet`, + reason: { user: `Not implemented yet`, tech: `Not implemented yet` }, } } async runPackageContainerCronJob( diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts index 7ed57e22..5ce06e22 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/fileCopy.ts @@ -6,6 +6,7 @@ import { UniversalVersion, compareUniversalVersions, makeUniversalVersion, getSt import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { hashObj, + waitTime, Expectation, ReturnTypeDoYouSupportExpectation, ReturnTypeGetCostFortExpectation, @@ -15,7 +16,7 @@ import { } from '@shared/api' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../accessorHandlers/accessor' import { ByteCounter } from '../../../lib/streamByteCounter' @@ -25,7 +26,6 @@ import { lookupAccessorHandles, LookupPackageContainer, userReadableDiff, - waitTime, } from './lib' import { CancelablePromise } from '../../../lib/cancelablePromise' import { PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' @@ -93,20 +93,26 @@ export const FileCopy: ExpectationWindowsHandler = { return { ready: false, sourceExists: true, - reason: `Source is not stable (${userReadableDiff(versionDiff)})`, + reason: { + user: `Waiting for source file to stop growing`, + tech: `Source is not stable (${userReadableDiff(versionDiff)})`, + }, } } } // Also check if we actually can read from the package, // this might help in some cases if the file is currently transferring - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: { + // user: 'Ready to start copying', + // tech: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, + // }, } }, isExpectationFullfilled: async ( @@ -118,30 +124,43 @@ export const FileCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issuePackage = await lookupTarget.handle.checkPackageReadAccess() - if (issuePackage) { - return { fulfilled: false, reason: `File does not exist: ${issuePackage.toString()}` } + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } } // check that the file is of the right version: const actualTargetVersion = await lookupTarget.handle.fetchMetadata() - if (!actualTargetVersion) return { fulfilled: false, reason: `Metadata missing` } + if (!actualTargetVersion) + return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const issueVersions = compareUniversalVersions(makeUniversalVersion(actualSourceVersion), actualTargetVersion) - if (issueVersions) { - return { fulfilled: false, reason: issueVersions } + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } } return { fulfilled: true, - reason: `File "${exp.endRequirement.content.filePath}" already exists on target`, + // reason: `File "${exp.endRequirement.content.filePath}" already exists on target`, } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -151,10 +170,10 @@ export const FileCopy: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupCopyTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -172,10 +191,11 @@ export const FileCopy: ExpectationWindowsHandler = { // We can do RoboCopy if (!isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) - if (!isLocalFolderAccessorHandle(targetHandle)) throw new Error(`Source AccessHandler type is wrong`) + if (!isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle)) + throw new Error(`Source AccessHandler type is wrong`) if (sourceHandle.fullPath === targetHandle.fullPath) { - throw new Error('Unable to copy: source and Target file paths are the same!') + throw new Error('Unable to copy: Source and Target file paths are the same!') } let wasCancelled = false @@ -194,24 +214,30 @@ export const FileCopy: ExpectationWindowsHandler = { await targetHandle.packageIsInPlace() const sourcePath = sourceHandle.fullPath - const targetPath = targetHandle.fullPath + const targetPath = exp.workOptions.useTemporaryFilePath + ? targetHandle.temporaryFilePath + : targetHandle.fullPath copying = roboCopyFile(sourcePath, targetPath, (progress: number) => { workInProgress._reportProgress(actualSourceVersionHash, progress / 100) }) await copying - // The copy is done + // The copy is done at this point + copying = undefined if (wasCancelled) return // ignore - const duration = Date.now() - startTime - + await targetHandle.finalizePackage() await targetHandle.updateMetadata(actualSourceUVersion) + const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Copy completed at ${Date.now()}`, + }, undefined ) }) @@ -220,22 +246,22 @@ export const FileCopy: ExpectationWindowsHandler = { } else if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupSource.accessor.type === Accessor.AccessType.HTTP) && + lookupSource.accessor.type === Accessor.AccessType.HTTP_PROXY) && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { // We can copy by using file streams if ( !isLocalFolderAccessorHandle(lookupSource.handle) && !isFileShareAccessorHandle(lookupSource.handle) && - !isHTTPAccessorHandle(lookupSource.handle) + !isHTTPProxyAccessorHandle(lookupSource.handle) ) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Source AccessHandler type is wrong`) @@ -282,20 +308,22 @@ export const FileCopy: ExpectationWindowsHandler = { if (wasCancelled) return // ignore setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - - targetHandle - .updateMetadata(actualSourceUVersion) - .then(() => { - workInProgress._reportComplete( - actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await targetHandle.finalizePackage() + await targetHandle.updateMetadata(actualSourceUVersion) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Copy completed at ${Date.now()}`, + }, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) }) @@ -313,16 +341,31 @@ export const FileCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed file "${exp.endRequirement.content.filePath}" from target` } + return { + removed: true, + // reason: `Removed file "${exp.endRequirement.content.filePath}" from target` + } }, } function isFileCopy(exp: Expectation.Any): exp is Expectation.FileCopy { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts new file mode 100644 index 00000000..5f0c2a42 --- /dev/null +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/jsonDataCopy.ts @@ -0,0 +1,271 @@ +import { Accessor } from '@sofie-automation/blueprints-integration' +import { GenericWorker } from '../../../worker' +import { UniversalVersion, compareUniversalVersions, makeUniversalVersion, getStandardCost } from '../lib/lib' +import { ExpectationWindowsHandler } from './expectationWindowsHandler' +import { + hashObj, + Expectation, + ReturnTypeDoYouSupportExpectation, + ReturnTypeGetCostFortExpectation, + ReturnTypeIsExpectationFullfilled, + ReturnTypeIsExpectationReadyToStartWorkingOn, + ReturnTypeRemoveExpectation, +} from '@shared/api' +import { + isCorePackageInfoAccessorHandle, + isFileShareAccessorHandle, + isHTTPProxyAccessorHandle, + isLocalFolderAccessorHandle, +} from '../../../accessorHandlers/accessor' +import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' +import { checkWorkerHasAccessToPackageContainersOnPackage, lookupAccessorHandles, LookupPackageContainer } from './lib' +import { PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' + +/** + * Copies a file from one of the sources and into the target PackageContainer + */ +export const JsonDataCopy: ExpectationWindowsHandler = { + doYouSupportExpectation(exp: Expectation.Any, genericWorker: GenericWorker): ReturnTypeDoYouSupportExpectation { + return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { + sources: exp.startRequirement.sources, + targets: exp.endRequirement.targets, + }) + }, + getCostForExpectation: async ( + exp: Expectation.Any, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + return getStandardCost(exp, worker) + }, + isExpectationReadyToStartWorkingOn: async ( + exp: Expectation.Any, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) return { ready: lookupSource.ready, sourceExists: false, reason: lookupSource.reason } + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } + + // Also check if we actually can read from the package, + // this might help in some cases if the file is currently transferring + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } + + return { + ready: true, + sourceExists: true, + } + }, + isExpectationFullfilled: async ( + exp: Expectation.Any, + _wasFullfilled: boolean, + worker: GenericWorker + ): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } + + const issuePackage = await lookupTarget.handle.checkPackageReadAccess() + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } + } + + // check that the file is of the right version: + const actualTargetVersion = await lookupTarget.handle.fetchMetadata() + if (!actualTargetVersion) + return { fulfilled: false, reason: { user: `Target version is wrong`, tech: `Metadata missing` } } + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } + + const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() + + const issueVersions = compareUniversalVersions(makeUniversalVersion(actualSourceVersion), actualTargetVersion) + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } + } + + return { + fulfilled: true, + } + }, + workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + // Copies the file from Source to Target + + const startTime = Date.now() + + const lookupSource = await lookupCopySources(worker, exp) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) + + const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() + const actualSourceVersionHash = hashObj(actualSourceVersion) + // const actualSourceUVersion = makeUniversalVersion(actualSourceVersion) + + // const sourceHandle = lookupSource.handle + const targetHandle = lookupTarget.handle + if ( + (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || + lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || + lookupSource.accessor.type === Accessor.AccessType.HTTP) && + lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO + ) { + // We can copy by using streams: + if ( + !isLocalFolderAccessorHandle(lookupSource.handle) && + !isFileShareAccessorHandle(lookupSource.handle) && + !isHTTPProxyAccessorHandle(lookupSource.handle) + ) + throw new Error(`Source AccessHandler type is wrong`) + if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Source AccessHandler type is wrong`) + + let wasCancelled = false + let sourceStream: PackageReadStream | undefined = undefined + let writeStream: PutPackageHandler | undefined = undefined + const workInProgress = new WorkInProgress({ workLabel: 'Copying, using streams' }, async () => { + // on cancel work + wasCancelled = true + await new Promise((resolve, reject) => { + writeStream?.once('close', () => { + targetHandle + .removePackage() + .then(() => resolve()) + .catch((err) => reject(err)) + }) + sourceStream?.cancel() + writeStream?.abort() + }) + }).do(async () => { + workInProgress._reportProgress(actualSourceVersionHash, 0.1) + + if (wasCancelled) return + sourceStream = await lookupSource.handle.getPackageReadStream() + writeStream = await targetHandle.putPackageStream(sourceStream.readStream) + + workInProgress._reportProgress(actualSourceVersionHash, 0.5) + + sourceStream.readStream.on('error', (err) => { + workInProgress._reportError(err) + }) + writeStream.on('error', (err) => { + workInProgress._reportError(err) + }) + writeStream.once('close', () => { + if (wasCancelled) return // ignore + setImmediate(() => { + // Copying is done + ;(async () => { + await targetHandle.finalizePackage() + // await targetHandle.updateMetadata() + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + { + user: `Completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) + }) + }) + }) + + return workInProgress + } else { + throw new Error( + `JsonDataCopy.workOnExpectation: Unsupported accessor source-target pair "${lookupSource.accessor.type}"-"${lookupTarget.accessor.type}"` + ) + } + }, + removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { + if (!isJsonDataCopy(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) + // Remove the file on the location + + const lookupTarget = await lookupCopyTargets(worker, exp) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } + + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove json-data due to an internal error`, + tech: `Cannot remove json-data: ${err.toString()}`, + }, + } + } + + return { + removed: true, + // reason: `Removed file "${exp.endRequirement.content.filePath}" from target` + } + }, +} +function isJsonDataCopy(exp: Expectation.Any): exp is Expectation.JsonDataCopy { + return exp.type === Expectation.Type.JSON_DATA_COPY +} + +function lookupCopySources( + worker: GenericWorker, + exp: Expectation.JsonDataCopy +): Promise> { + return lookupAccessorHandles( + worker, + exp.startRequirement.sources, + exp.endRequirement.content, + exp.workOptions, + { + read: true, + readPackage: true, + packageVersion: exp.endRequirement.version, + } + ) +} +function lookupCopyTargets( + worker: GenericWorker, + exp: Expectation.JsonDataCopy +): Promise> { + return lookupAccessorHandles( + worker, + exp.endRequirement.targets, + exp.endRequirement.content, + exp.workOptions, + { + write: true, + writePackageContainer: true, + } + ) +} diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index 1894e38d..4e81c090 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -5,7 +5,7 @@ import { GenericAccessorHandle } from '../../../accessorHandlers/genericHandle' import { GenericWorker } from '../../../worker' import { compareActualExpectVersions, findBestPackageContainerWithAccessToPackage } from '../lib/lib' import { Diff } from 'deep-diff' -import { Expectation, ReturnTypeDoYouSupportExpectation } from '@shared/api' +import { Expectation, Reason, ReturnTypeDoYouSupportExpectation } from '@shared/api' /** Check that a worker has access to the packageContainers through its accessors */ export function checkWorkerHasAccessToPackageContainersOnPackage( @@ -18,60 +18,84 @@ export function checkWorkerHasAccessToPackageContainersOnPackage( let accessSourcePackageContainer: ReturnType // Check that we have access to the packageContainers if (checks.sources !== undefined) { + if (checks.sources.length === 0) { + return { + support: false, + reason: { + user: `No sources configured`, + tech: `No sources configured`, + }, + } + } accessSourcePackageContainer = findBestPackageContainerWithAccessToPackage(genericWorker, checks.sources) if (!accessSourcePackageContainer) { return { support: false, - reason: `Doesn't have access to any of the source packageContainers (${checks.sources - .map((o) => o.containerId) - .join(', ')})`, + reason: { + user: `There is an issue with the configuration of the Worker, it doesn't have access to any of the source PackageContainers`, + tech: `Worker doesn't have access to any of the source packageContainers (${checks.sources + .map((o) => `${o.containerId} "${o.label}"`) + .join(', ')})`, + }, } } } let accessTargetPackageContainer: ReturnType if (checks.targets !== undefined) { + if (checks.targets.length === 0) { + return { + support: false, + reason: { + user: `No targets configured`, + tech: `No targets configured`, + }, + } + } accessTargetPackageContainer = findBestPackageContainerWithAccessToPackage(genericWorker, checks.targets) if (!accessTargetPackageContainer) { return { support: false, - reason: `Doesn't have access to any of the target packageContainers (${checks.targets - .map((o) => o.containerId) - .join(', ')})`, + reason: { + user: `There is an issue with the configuration of the Worker, it doesn't have access to any of the target PackageContainers`, + tech: `Worker doesn't have access to any of the target packageContainers (${checks.targets + .map((o) => o.containerId) + .join(', ')})`, + }, } } } - const hasAccessTo: string[] = [] - if (accessSourcePackageContainer) { - hasAccessTo.push( - `source "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"` - ) - } - if (accessTargetPackageContainer) { - hasAccessTo.push( - `target "${accessTargetPackageContainer.packageContainer.label}" through accessor "${accessTargetPackageContainer.accessorId}"` - ) - } + // const hasAccessTo: string[] = [] + // if (accessSourcePackageContainer) { + // hasAccessTo.push( + // `source "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"` + // ) + // } + // if (accessTargetPackageContainer) { + // hasAccessTo.push( + // `target "${accessTargetPackageContainer.packageContainer.label}" through accessor "${accessTargetPackageContainer.accessorId}"` + // ) + // } return { support: true, - reason: `Has access to ${hasAccessTo.join(' and ')}`, + // reason: `Has access to ${hasAccessTo.join(' and ')}`, } } export type LookupPackageContainer = | { + ready: true accessor: AccessorOnPackage.Any handle: GenericAccessorHandle - ready: true - reason: string + // reason: Reason } | { - accessor: undefined - handle: undefined ready: false - reason: string + accessor: undefined + // handle: undefined + reason: Reason } interface LookupChecks { /** Check that the accessor-handle supports reading */ @@ -95,7 +119,7 @@ export async function lookupAccessorHandles( checks: LookupChecks ): Promise> { /** undefined if all good, error string otherwise */ - let errorReason: undefined | string = 'No target found' + let errorReason: undefined | Reason = { user: 'No target found', tech: 'No target found' } // See if the file is available at any of the targets: for (const { packageContainer, accessorId, accessor } of prioritizeAccessors(packageContainers)) { @@ -111,22 +135,33 @@ export async function lookupAccessorHandles( if (checks.read) { // Check that the accessor-handle supports reading: - const issueHandleRead = handle.checkHandleRead() - if (issueHandleRead) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issueHandleRead}` + const readResult = handle.checkHandleRead() + if (!readResult.success) { + errorReason = { + user: `There is an issue with the configuration for the PackageContainer "${ + packageContainer.label + }" (on accessor "${accessor.label || accessorId}"): ${readResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + readResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.readPackage) { // Check that the Package can be read: - const issuePackageReadAccess = await handle.checkPackageReadAccess() - if (issuePackageReadAccess) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issuePackageReadAccess}` + const readResult = await handle.checkPackageReadAccess() + if (!readResult.success) { + errorReason = { + user: `Can't read the Package from PackageContainer "${packageContainer.label}" (on accessor "${ + accessor.label || accessorId + }"), due to: ${readResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + readResult.reason.tech + }`, + } + continue // Maybe next accessor works? } } @@ -134,30 +169,45 @@ export async function lookupAccessorHandles( // Check that the version of the Package is correct: const actualSourceVersion = await handle.getPackageActualVersion() - const issuePackageVersion = compareActualExpectVersions(actualSourceVersion, checks.packageVersion) - if (issuePackageVersion) { - errorReason = `${packageContainer.label}: Accessor "${ - accessor.label || accessorId - }": ${issuePackageVersion}` + const compareVersionResult = compareActualExpectVersions(actualSourceVersion, checks.packageVersion) + if (!compareVersionResult.success) { + errorReason = { + user: `Won't read from the package, due to: ${compareVersionResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + compareVersionResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.write) { // Check that the accessor-handle supports writing: - const issueHandleWrite = handle.checkHandleWrite() - if (issueHandleWrite) { - errorReason = `${packageContainer.label}: lookupTargets: Accessor "${ - accessor.label || accessorId - }": ${issueHandleWrite}` + const writeResult = handle.checkHandleWrite() + if (!writeResult.success) { + errorReason = { + user: `There is an issue with the configuration for the PackageContainer "${ + packageContainer.label + }" (on accessor "${accessor.label || accessorId}"): ${writeResult.reason.user}`, + tech: `${packageContainer.label}: lookupTargets: Accessor "${accessor.label || accessorId}": ${ + writeResult.reason.tech + }`, + } continue // Maybe next accessor works? } } if (checks.writePackageContainer) { // Check that it is possible to write to write to the package container: - const issuePackage = await handle.checkPackageContainerWriteAccess() - if (issuePackage) { - errorReason = `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${issuePackage}` + const writeAccessResult = await handle.checkPackageContainerWriteAccess() + if (!writeAccessResult.success) { + errorReason = { + user: `Can't write to the PackageContainer "${packageContainer.label}" (on accessor "${ + accessor.label || accessorId + }"), due to: ${writeAccessResult.reason.user}`, + tech: `${packageContainer.label}: Accessor "${accessor.label || accessorId}": ${ + writeAccessResult.reason.tech + }`, + } continue // Maybe next accessor works? } } @@ -168,24 +218,19 @@ export async function lookupAccessorHandles( accessor: accessor, handle: handle, ready: true, - reason: `Can access target "${packageContainer.label}" through accessor "${ - accessor.label || accessorId - }"`, + // reason: `Can access target "${packageContainer.label}" through accessor "${ + // accessor.label || accessorId + // }"`, } } } return { accessor: undefined, - handle: undefined, ready: false, reason: errorReason, } } -export function waitTime(duration: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, duration) - }) -} + /** Converts a diff to some kind of user-readable string */ export function userReadableDiff(diffs: Diff[]): string { const strs: string[] = [] diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts index 5a730710..7c4ca56a 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/ffmpeg.ts @@ -1,28 +1,28 @@ import { ChildProcess, spawn } from 'child_process' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../../accessorHandlers/accessor' import { FileShareAccessorHandle } from '../../../../accessorHandlers/fileShare' -import { HTTPAccessorHandle } from '../../../../accessorHandlers/http' +import { HTTPProxyAccessorHandle } from '../../../../accessorHandlers/httpProxy' import { LocalFolderAccessorHandle } from '../../../../accessorHandlers/localFolder' -import { assertNever } from '../../../../lib/lib' +import { assertNever } from '@shared/api' import { WorkInProgress } from '../../../../lib/workInProgress' export interface FFMpegProcess { cancel: () => void } -/** Check if FFMpeg is available */ -export function hasFFMpeg(): Promise { - return hasFFExecutable(process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg') +/** Check if FFMpeg is available, returns null if no error found */ +export function testFFMpeg(): Promise { + return testFFExecutable(process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg') } /** Check if FFProbe is available */ -export function hasFFProbe(): Promise { - return hasFFExecutable(process.platform === 'win32' ? 'ffprobe.exe' : 'ffprobe') +export function testFFProbe(): Promise { + return testFFExecutable(process.platform === 'win32' ? 'ffprobe.exe' : 'ffprobe') } -export function hasFFExecutable(ffExecutable: string): Promise { - return new Promise((resolve, reject) => { +export function testFFExecutable(ffExecutable: string): Promise { + return new Promise((resolve) => { const ffMpegProcess: ChildProcess = spawn(ffExecutable, ['-version'], { shell: true, }) @@ -36,15 +36,19 @@ export function hasFFExecutable(ffExecutable: string): Promise { output += str }) ffMpegProcess.on('error', (err) => { - reject(err) + resolve(`Process ${ffExecutable} emitted error: ${err}`) }) ffMpegProcess.on('close', (code) => { const m = output.match(/version ([\w-]+)/) // version N-102494-g2899fb61d2 - if (code === 0 && m) { - resolve(m[1]) + if (code === 0) { + if (m) { + resolve(null) + } else { + resolve(`Process ${ffExecutable} bad version: ${output}`) + } } else { - reject(null) + resolve(`Process ${ffExecutable} exited with code ${code}`) } }) }) @@ -58,7 +62,7 @@ export async function runffMpeg( targetHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle, + | HTTPProxyAccessorHandle, actualSourceVersionHash: string, onDone: () => Promise ): Promise { @@ -79,7 +83,7 @@ export async function runffMpeg( } else if (isFileShareAccessorHandle(targetHandle)) { await targetHandle.prepareFileAccess() args.push(`"${targetHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(targetHandle)) { + } else if (isHTTPProxyAccessorHandle(targetHandle)) { pipeStdOut = true args.push('pipe:1') // pipe output to stdout } else { @@ -102,6 +106,7 @@ export async function runffMpeg( }) writeStream.once('close', () => { uploadIsDone = true + maybeDone() }) } else { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts index 15688abd..60dc4f06 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib/scan.ts @@ -1,19 +1,18 @@ import { exec, ChildProcess, spawn } from 'child_process' -import { Expectation } from '@shared/api' +import { Expectation, assertNever } from '@shared/api' import { isQuantelClipAccessorHandle, isLocalFolderAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, } from '../../../../accessorHandlers/accessor' import { LocalFolderAccessorHandle } from '../../../../accessorHandlers/localFolder' import { QuantelAccessorHandle } from '../../../../accessorHandlers/quantel' import { CancelablePromise } from '../../../../lib/cancelablePromise' -import { assertNever } from '../../../../lib/lib' import { FieldOrder, ScanAnomaly } from './coreApi' import { generateFFProbeFromClipData } from './quantelFormats' import { FileShareAccessorHandle } from '../../../../accessorHandlers/fileShare' -import { HTTPAccessorHandle } from '../../../../accessorHandlers/http' +import { HTTPProxyAccessorHandle } from '../../../../accessorHandlers/httpProxy' interface FFProbeScanResult { // to be defined... @@ -25,23 +24,27 @@ export function scanWithFFProbe( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle ): CancelablePromise { return new CancelablePromise(async (resolve, reject, onCancel) => { if ( isLocalFolderAccessorHandle(sourceHandle) || isFileShareAccessorHandle(sourceHandle) || - isHTTPAccessorHandle(sourceHandle) + isHTTPProxyAccessorHandle(sourceHandle) ) { let inputPath: string + let filePath: string if (isLocalFolderAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullPath + filePath = sourceHandle.filePath } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() inputPath = sourceHandle.fullPath - } else if (isHTTPAccessorHandle(sourceHandle)) { + filePath = sourceHandle.filePath + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullUrl + filePath = sourceHandle.filePath } else { assertNever(sourceHandle) throw new Error('Unknown handle') @@ -73,6 +76,7 @@ export function scanWithFFProbe( reject(new Error(`File doesn't seem to be a media file`)) return } + json.filePath = filePath resolve(json) }) } else if (isQuantelClipAccessorHandle(sourceHandle)) { @@ -98,7 +102,7 @@ export function scanFieldOrder( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle, targetVersion: Expectation.PackageDeepScan['endRequirement']['version'] ): CancelablePromise { @@ -125,12 +129,12 @@ export function scanFieldOrder( } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() args.push(`-i "${sourceHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { args.push(`-i "${sourceHandle.fullUrl}"`) } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Source Clip not found`) + if (!httpStreamURL.success) throw new Error(`Source Clip not found (${httpStreamURL.reason.tech})`) args.push('-seekable 0') args.push(`-i "${httpStreamURL.fullURL}"`) @@ -173,7 +177,7 @@ export function scanMoreInfo( sourceHandle: | LocalFolderAccessorHandle | FileShareAccessorHandle - | HTTPAccessorHandle + | HTTPProxyAccessorHandle | QuantelAccessorHandle, previouslyScanned: FFProbeScanResult, targetVersion: Expectation.PackageDeepScan['endRequirement']['version'], @@ -225,12 +229,12 @@ export function scanMoreInfo( } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() args.push(`-i "${sourceHandle.fullPath}"`) - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { args.push(`-i "${sourceHandle.fullUrl}"`) } else if (isQuantelClipAccessorHandle(sourceHandle)) { const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Source Clip not found`) + if (!httpStreamURL.success) throw new Error(`Source Clip not found (${httpStreamURL.reason.tech})`) args.push('-seekable 0') args.push(`-i "${httpStreamURL.fullURL}"`) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index 240e5ad7..a72da83c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -13,7 +13,7 @@ import { } from '@shared/api' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' @@ -30,7 +30,14 @@ export const MediaFilePreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, targets: exp.endRequirement.targets, @@ -54,13 +61,13 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -72,13 +79,32 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupSource = await lookupPreviewSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Preview does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -86,11 +112,23 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No preview file found' } + return { + fulfilled: false, + reason: { user: `The preview needs to be re-generated`, tech: `No preview metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Preview version doesn't match preview file` } + return { + fulfilled: false, + reason: { + user: `The preview needs to be re-generated`, + tech: `Preview version doesn't match source file`, + }, + } } else { - return { fulfilled: true, reason: 'Preview already matches preview file' } + return { + fulfilled: true, + // reason: { user: `Preview already matches preview file`, tech: `Preview already matches preview file` }, + } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -100,10 +138,10 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupPreviewSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupPreviewTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle @@ -112,14 +150,14 @@ export const MediaFilePreview: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { // We can read the source and write the preview directly. if (!isLocalFolderAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) @@ -128,8 +166,8 @@ export const MediaFilePreview: ExpectationWindowsHandler = { // On cancel ffMpegProcess?.cancel() }).do(async () => { - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -173,12 +211,16 @@ export const MediaFilePreview: ExpectationWindowsHandler = { async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } @@ -198,16 +240,31 @@ export const MediaFilePreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove preview file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed preview file "${exp.endRequirement.content.filePath}" from target` } + return { + removed: true, + // reason: { user: ``, tech: `Removed preview file "${exp.endRequirement.content.filePath}" from target` }, + } }, } function isMediaFilePreview(exp: Expectation.Any): exp is Expectation.MediaFilePreview { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index 739a9aa4..2f94204d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -7,13 +7,14 @@ import { ReturnTypeIsExpectationFullfilled, ReturnTypeIsExpectationReadyToStartWorkingOn, ReturnTypeRemoveExpectation, + assertNever, } from '@shared/api' import { getStandardCost } from '../lib/lib' import { GenericWorker } from '../../../worker' import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' @@ -23,7 +24,6 @@ import { lookupAccessorHandles, LookupPackageContainer, } from './lib' -import { assertNever } from '../../../lib/lib' import { FFMpegProcess, runffMpeg } from './lib/ffmpeg' import { WindowsWorker } from '../windowsWorker' @@ -36,7 +36,14 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -59,13 +66,13 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -77,13 +84,32 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const lookupSource = await lookupThumbnailSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Thumbnail does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -91,11 +117,20 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No thumbnail file found' } + return { + fulfilled: false, + reason: { user: `The thumbnail needs to be re-generated`, tech: `No thumbnail metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Thumbnail version doesn't match thumbnail file` } + return { + fulfilled: false, + reason: { + user: `The thumbnail needs to be re-generated`, + tech: `Thumbnail version doesn't match thumbnail file`, + }, + } } else { - return { fulfilled: true, reason: 'Thumbnail already matches thumbnail file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -105,10 +140,10 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupThumbnailSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let ffMpegProcess: FFMpegProcess | undefined const workInProgress = new WorkInProgress({ workLabel: 'Generating thumbnail' }, async () => { @@ -118,29 +153,29 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) && + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) + !isHTTPProxyAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) { - throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) { + throw new Error(tryReadPackage.reason.tech) } const actualSourceVersion = await sourceHandle.getPackageActualVersion() @@ -172,7 +207,7 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { } else if (isFileShareAccessorHandle(sourceHandle)) { await sourceHandle.prepareFileAccess() inputPath = sourceHandle.fullPath - } else if (isHTTPAccessorHandle(sourceHandle)) { + } else if (isHTTPProxyAccessorHandle(sourceHandle)) { inputPath = sourceHandle.fullUrl } else { assertNever(sourceHandle) @@ -195,12 +230,16 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { ffMpegProcess = await runffMpeg(workInProgress, args, targetHandle, sourceVersionHash, async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) }) @@ -216,11 +255,29 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isMediaFileThumbnail(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } - await lookupTarget.handle.removePackage() + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed thumbnail' } + return { removed: true } }, } function isMediaFileThumbnail(exp: Expectation.Any): exp is Expectation.MediaFileThumbnail { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts index 8fc6d492..fbfa456d 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageDeepScan.ts @@ -14,7 +14,7 @@ import { import { isCorePackageInfoAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -35,7 +35,14 @@ export const PackageDeepScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFProbe) return { support: false, reason: 'Cannot access FFProbe executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -59,13 +66,13 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const lookupTarget = await lookupDeepScanSources(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -77,10 +84,22 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const lookupSource = await lookupDeepScanSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupDeepScanTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -100,7 +119,7 @@ export const PackageDeepScan: ExpectationWindowsHandler = { } return { fulfilled: false, reason: packageInfoSynced.reason } } else { - return { fulfilled: true, reason: packageInfoSynced.reason } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -109,10 +128,10 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupDeepScanSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupDeepScanTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let currentProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Scanning file' }, async () => { @@ -124,14 +143,14 @@ export const PackageDeepScan: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupSource.accessor.type === Accessor.AccessType.HTTP || + lookupSource.accessor.type === Accessor.AccessType.HTTP_PROXY || lookupSource.accessor.type === Accessor.AccessType.QUANTEL) && lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO ) { if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) && + !isHTTPProxyAccessorHandle(sourceHandle) && !isQuantelClipAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) @@ -139,8 +158,8 @@ export const PackageDeepScan: ExpectationWindowsHandler = { if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const sourceVersionHash = hashObj(actualSourceVersion) @@ -187,7 +206,10 @@ export const PackageDeepScan: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Scan completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Scan completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } else { @@ -202,12 +224,29 @@ export const PackageDeepScan: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isPackageDeepScan(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupDeepScanTargets(worker, exp) - if (!lookupTarget.ready) return { removed: false, reason: `Not able to access target: ${lookupTarget.reason}` } + if (!lookupTarget.ready) + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } if (!isCorePackageInfoAccessorHandle(lookupTarget.handle)) throw new Error(`Target AccessHandler type is wrong`) - await lookupTarget.handle.removePackageInfo('deepScan', exp) + try { + await lookupTarget.handle.removePackageInfo('deepScan', exp) + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove the scan result due to an internal error`, + tech: `Cannot remove CorePackageInfo: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed scan info from Store' } + return { removed: true } }, } function isPackageDeepScan(exp: Expectation.Any): exp is Expectation.PackageDeepScan { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts index c436ea1a..f1f89e5f 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/packageScan.ts @@ -14,7 +14,7 @@ import { import { isCorePackageInfoAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -33,7 +33,14 @@ export const PackageScan: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFProbe) return { support: false, reason: 'Cannot access FFProbe executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -57,13 +64,12 @@ export const PackageScan: ExpectationWindowsHandler = { const lookupTarget = await lookupScanSources(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -75,10 +81,22 @@ export const PackageScan: ExpectationWindowsHandler = { const lookupSource = await lookupScanSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupScanTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to ${lookupTarget.reason.user}`, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() @@ -98,7 +116,7 @@ export const PackageScan: ExpectationWindowsHandler = { } return { fulfilled: false, reason: packageInfoSynced.reason } } else { - return { fulfilled: true, reason: packageInfoSynced.reason } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -107,10 +125,10 @@ export const PackageScan: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupScanSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupScanTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let currentProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Scanning file' }, async () => { @@ -123,22 +141,22 @@ export const PackageScan: ExpectationWindowsHandler = { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupSource.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupSource.accessor.type === Accessor.AccessType.HTTP || + lookupSource.accessor.type === Accessor.AccessType.HTTP_PROXY || lookupSource.accessor.type === Accessor.AccessType.QUANTEL) && lookupTarget.accessor.type === Accessor.AccessType.CORE_PACKAGE_INFO ) { if ( !isLocalFolderAccessorHandle(sourceHandle) && !isFileShareAccessorHandle(sourceHandle) && - !isHTTPAccessorHandle(sourceHandle) && + !isHTTPProxyAccessorHandle(sourceHandle) && !isQuantelClipAccessorHandle(sourceHandle) ) throw new Error(`Source AccessHandler type is wrong`) if (!isCorePackageInfoAccessorHandle(targetHandle)) throw new Error(`Target AccessHandler type is wrong`) - const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + const tryReadPackage = await sourceHandle.checkPackageReadAccess() + if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const sourceVersionHash = hashObj(actualSourceVersion) @@ -164,7 +182,10 @@ export const PackageScan: ExpectationWindowsHandler = { const duration = Date.now() - startTime workInProgress._reportComplete( sourceVersionHash, - `Scan completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Scan completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } else { @@ -179,12 +200,29 @@ export const PackageScan: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isPackageScan(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupScanTargets(worker, exp) - if (!lookupTarget.ready) return { removed: false, reason: `Not able to access target: ${lookupTarget.reason}` } + if (!lookupTarget.ready) + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } if (!isCorePackageInfoAccessorHandle(lookupTarget.handle)) throw new Error(`Target AccessHandler type is wrong`) - await lookupTarget.handle.removePackageInfo('scan', exp) + try { + await lookupTarget.handle.removePackageInfo('scan', exp) + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove the scan result due to an internal error`, + tech: `Cannot remove CorePackageInfo: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed scan info from Store' } + return { removed: true } }, } function isPackageScan(exp: Expectation.Any): exp is Expectation.PackageScan { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts index 2a48c6d0..0666711c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipCopy.ts @@ -41,7 +41,14 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } if (lookupTarget.accessor.type === Accessor.AccessType.QUANTEL) { - if (!lookupTarget.accessor.serverId) return { ready: false, reason: `Target Accessor has no serverId set` } + if (!lookupTarget.accessor.serverId) + return { + ready: false, + reason: { + user: `There is an issue in the settings: The Accessor "${lookupTarget.handle.accessorId}" has no serverId set`, + tech: `Target Accessor "${lookupTarget.handle.accessorId}" has no serverId set`, + }, + } } // // Do a check, to ensure that the source and targets are Quantel: @@ -51,13 +58,13 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { // return { ready: false, reason: `Target Accessor type not supported: ${lookupSource.accessor.type}` } // Also check if we actually can read from the package: - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryReading = await lookupSource.handle.tryPackageRead() + if (!tryReading.success) return { ready: false, reason: tryReading.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, + // reason: `${lookupSource.reason.user}, ${lookupTarget.reason.tech}`, } }, isExpectationFullfilled: async ( @@ -69,19 +76,35 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issuePackage = await lookupTarget.handle.checkPackageReadAccess() - if (issuePackage) { - return { fulfilled: false, reason: `Clip not found: ${issuePackage.toString()}` } + if (!issuePackage.success) { + return { + fulfilled: false, + reason: { + user: `Target package: ${issuePackage.reason.user}`, + tech: `Target package: ${issuePackage.reason.tech}`, + }, + } } // Does the clip exist on the target? const actualTargetVersion = await lookupTarget.handle.getPackageActualVersion() - if (!actualTargetVersion) return { fulfilled: false, reason: `No package found on target` } + if (!actualTargetVersion) + return { + fulfilled: false, + reason: { user: `No clip found on target`, tech: `No clip found on target` }, + } const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) return { fulfilled: false, reason: lookupSource.reason } // Check that the target clip is of the right version: @@ -91,15 +114,12 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { makeUniversalVersion(actualSourceVersion), makeUniversalVersion(actualTargetVersion) ) - if (issueVersions) { - return { fulfilled: false, reason: issueVersions } + if (!issueVersions.success) { + return { fulfilled: false, reason: issueVersions.reason } } return { fulfilled: true, - reason: `Clip "${ - exp.endRequirement.content.guid || exp.endRequirement.content.title - }" already exists on target`, } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -109,10 +129,10 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupCopySources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupCopyTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -203,20 +223,22 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { wasCompleted = true setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - - targetHandle - .updateMetadata(actualSourceUVersion) - .then(() => { - workInProgress._reportComplete( - actualSourceVersionHash, - `Copy completed in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await targetHandle.finalizePackage() + await targetHandle.updateMetadata(actualSourceUVersion) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + actualSourceVersionHash, + { + user: `Copy completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) @@ -233,18 +255,29 @@ export const QuantelClipCopy: ExpectationWindowsHandler = { const lookupTarget = await lookupCopyTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove clip: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove clip due to an internal error`, + tech: `Cannot remove preview clip: ${err.toString()}`, + }, + } } return { removed: true, - reason: `Removed clip "${exp.endRequirement.content.guid || exp.endRequirement.content.title}" from target`, } }, } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts index 26570157..92a1ad8f 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipPreview.ts @@ -13,7 +13,7 @@ import { } from '@shared/api' import { isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' @@ -29,7 +29,14 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, targets: exp.endRequirement.targets, @@ -53,22 +60,26 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryResult = await lookupSource.handle.tryPackageRead() + if (!tryResult.success) return { ready: false, reason: tryResult.reason } // This is a bit special, as we use the Quantel HTTP-transformer to get a HLS-stream of the video: if (!isQuantelClipAccessorHandle(lookupSource.handle)) throw new Error(`Source AccessHandler type is wrong`) + const httpStreamURL = await lookupSource.handle.getTransformerStreamURL() - if (!httpStreamURL) return { ready: false, reason: `Preview source not found` } + if (!httpStreamURL.success) + return { + ready: false, + reason: httpStreamURL.reason, + } const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, httpStreamURL) - const issueReadingHTTP = await sourceHTTPHandle.tryPackageRead() - if (issueReadingHTTP) return { ready: false, reason: issueReadingHTTP } + const tryReadingHTTP = await sourceHTTPHandle.tryPackageRead() + if (!tryReadingHTTP.success) return { ready: false, reason: tryReadingHTTP.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -80,13 +91,32 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupSource = await lookupPreviewSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Preview does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -94,11 +124,20 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const metadata = await lookupTarget.handle.fetchMetadata() if (!metadata) { - return { fulfilled: false, reason: 'No preview file found' } + return { + fulfilled: false, + reason: { user: `The preview needs to be re-generated`, tech: `No preview metadata file found` }, + } } else if (metadata.sourceVersionHash !== actualSourceVersionHash) { - return { fulfilled: false, reason: `Preview version doesn't match preview file` } + return { + fulfilled: false, + reason: { + user: `The preview needs to be re-generated`, + tech: `Preview version doesn't match source file`, + }, + } } else { - return { fulfilled: true, reason: 'Preview already matches preview file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -108,10 +147,10 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupPreviewSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupPreviewTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle @@ -119,20 +158,20 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.QUANTEL && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { // We can read the source and write the preview directly. if (!isQuantelClipAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) // This is a bit special, as we use the Quantel HTTP-transformer to get a HLS-stream of the video: const httpStreamURL = await sourceHandle.getTransformerStreamURL() - if (!httpStreamURL) throw new Error(`Preview source not found`) + if (!httpStreamURL.success) throw new Error(httpStreamURL.reason.tech) const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, httpStreamURL) let ffMpegProcess: FFMpegProcess | undefined @@ -141,7 +180,7 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { ffMpegProcess?.cancel() }).do(async () => { const issueReadPackage = await sourceHandle.checkPackageReadAccess() - if (issueReadPackage) throw new Error(issueReadPackage) + if (!issueReadPackage.success) throw new Error(issueReadPackage.reason.tech) const actualSourceVersion = await sourceHandle.getPackageActualVersion() const actualSourceVersionHash = hashObj(actualSourceVersion) @@ -185,12 +224,16 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { async () => { // Called when ffmpeg has finished ffMpegProcess = undefined + await targetHandle.finalizePackage() await targetHandle.updateMetadata(metadata) const duration = Date.now() - startTime workInProgress._reportComplete( actualSourceVersionHash, - `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + { + user: `Preview generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, undefined ) } @@ -210,16 +253,28 @@ export const QuantelClipPreview: ExpectationWindowsHandler = { const lookupTarget = await lookupPreviewTargets(worker, exp) if (!lookupTarget.ready) { - return { removed: false, reason: `No access to target: ${lookupTarget.reason}` } + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } } try { await lookupTarget.handle.removePackage() } catch (err) { - return { removed: false, reason: `Cannot remove preview file: ${err.toString()}` } + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } } - return { removed: true, reason: `Removed preview file "${exp.endRequirement.content.filePath}" from target` } + return { removed: true } }, } function isQuantelClipPreview(exp: Expectation.Any): exp is Expectation.QuantelClipPreview { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts index 58f7ad7d..6bfaab1a 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/quantelClipThumbnail.ts @@ -8,6 +8,7 @@ import { ReturnTypeIsExpectationReadyToStartWorkingOn, ReturnTypeRemoveExpectation, literal, + Reason, } from '@shared/api' import { getStandardCost } from '../lib/lib' import { GenericWorker } from '../../../worker' @@ -15,14 +16,14 @@ import { ExpectationWindowsHandler } from './expectationWindowsHandler' import { getAccessorHandle, isFileShareAccessorHandle, - isHTTPAccessorHandle, + isHTTPProxyAccessorHandle, isLocalFolderAccessorHandle, isQuantelClipAccessorHandle, } from '../../../accessorHandlers/accessor' import { IWorkInProgress, WorkInProgress } from '../../../lib/workInProgress' import { checkWorkerHasAccessToPackageContainersOnPackage, lookupAccessorHandles, LookupPackageContainer } from './lib' import { GenericAccessorHandle, PackageReadStream, PutPackageHandler } from '../../../accessorHandlers/genericHandle' -import { HTTPAccessorHandle } from '../../../accessorHandlers/http' +import { HTTPProxyAccessorHandle } from '../../../accessorHandlers/httpProxy' import { WindowsWorker } from '../windowsWorker' /** @@ -34,7 +35,14 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { genericWorker: GenericWorker, windowsWorker: WindowsWorker ): ReturnTypeDoYouSupportExpectation { - if (!windowsWorker.hasFFMpeg) return { support: false, reason: 'Cannot access FFMpeg executable' } + if (windowsWorker.testFFMpeg) + return { + support: false, + reason: { + user: 'There is an issue with the Worker (FFMpeg)', + tech: `Cannot access FFMpeg executable: ${windowsWorker.testFFMpeg}`, + }, + } return checkWorkerHasAccessToPackageContainersOnPackage(genericWorker, { sources: exp.startRequirement.sources, }) @@ -57,21 +65,24 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) return { ready: lookupTarget.ready, reason: lookupTarget.reason } - const issueReading = await lookupSource.handle.tryPackageRead() - if (issueReading) return { ready: false, reason: issueReading } + const tryResult = await lookupSource.handle.tryPackageRead() + if (!tryResult.success) return { ready: false, reason: tryResult.reason } // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail: const thumbnailURL = await getThumbnailURL(exp, lookupSource) - if (!thumbnailURL) return { ready: false, reason: `Thumbnail source not found` } + if (!thumbnailURL.success) + return { + ready: false, + reason: thumbnailURL.reason, + } const sourceHTTPHandle = getSourceHTTPHandle(worker, lookupSource.handle, thumbnailURL) - const issueReadingHTTP = await sourceHTTPHandle.tryPackageRead() - if (issueReadingHTTP) return { ready: false, reason: issueReadingHTTP } + const tryReadingHTTP = await sourceHTTPHandle.tryPackageRead() + if (!tryReadingHTTP.success) return { ready: false, reason: tryReadingHTTP.reason } return { ready: true, sourceExists: true, - reason: `${lookupSource.reason}, ${lookupTarget.reason}`, } }, isExpectationFullfilled: async ( @@ -83,13 +94,32 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const lookupSource = await lookupThumbnailSources(worker, exp) if (!lookupSource.ready) - return { fulfilled: false, reason: `Not able to access source: ${lookupSource.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access source, due to ${lookupSource.reason.user}`, + tech: `Not able to access source: ${lookupSource.reason.tech}`, + }, + } const lookupTarget = await lookupThumbnailTargets(worker, exp) if (!lookupTarget.ready) - return { fulfilled: false, reason: `Not able to access target: ${lookupTarget.reason}` } + return { + fulfilled: false, + reason: { + user: `Not able to access target, due to: ${lookupTarget.reason.user} `, + tech: `Not able to access target: ${lookupTarget.reason.tech}`, + }, + } const issueReadPackage = await lookupTarget.handle.checkPackageReadAccess() - if (issueReadPackage) return { fulfilled: false, reason: `Thumbnail does not exist: ${issueReadPackage}` } + if (!issueReadPackage.success) + return { + fulfilled: false, + reason: { + user: `Issue with target: ${issueReadPackage.reason.user}`, + tech: `Issue with target: ${issueReadPackage.reason.tech}`, + }, + } const actualSourceVersion = await lookupSource.handle.getPackageActualVersion() const expectedTargetMetadata: Metadata = getMetadata(exp, actualSourceVersion) @@ -97,11 +127,20 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const targetMetadata = await lookupTarget.handle.fetchMetadata() if (!targetMetadata) { - return { fulfilled: false, reason: 'No thumbnail file found' } + return { + fulfilled: false, + reason: { user: `The thumbnail needs to be re-generated`, tech: `No thumbnail metadata file found` }, + } } else if (targetMetadata.sourceVersionHash !== expectedTargetMetadata.sourceVersionHash) { - return { fulfilled: false, reason: `Thumbnail version hash doesn't match thumnail file` } + return { + fulfilled: false, + reason: { + user: `The thumbnail needs to be re-generated`, + tech: `Thumbnail version doesn't match thumbnail file`, + }, + } } else { - return { fulfilled: true, reason: 'Thumbnail already matches thumnail file' } + return { fulfilled: true } } }, workOnExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { @@ -111,10 +150,10 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { const startTime = Date.now() const lookupSource = await lookupThumbnailSources(worker, exp) - if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason}`) + if (!lookupSource.ready) throw new Error(`Can't start working due to source: ${lookupSource.reason.tech}`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) const sourceHandle = lookupSource.handle const targetHandle = lookupTarget.handle @@ -123,20 +162,20 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { lookupSource.accessor.type === Accessor.AccessType.QUANTEL && (lookupTarget.accessor.type === Accessor.AccessType.LOCAL_FOLDER || lookupTarget.accessor.type === Accessor.AccessType.FILE_SHARE || - lookupTarget.accessor.type === Accessor.AccessType.HTTP) + lookupTarget.accessor.type === Accessor.AccessType.HTTP_PROXY) ) { if (!isQuantelClipAccessorHandle(sourceHandle)) throw new Error(`Source AccessHandler type is wrong`) if ( !isLocalFolderAccessorHandle(targetHandle) && !isFileShareAccessorHandle(targetHandle) && - !isHTTPAccessorHandle(targetHandle) + !isHTTPProxyAccessorHandle(targetHandle) ) throw new Error(`Target AccessHandler type is wrong`) // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail: const thumbnailURL = await getThumbnailURL(exp, lookupSource) - if (!thumbnailURL) throw new Error(`Can't start working due to source: Thumbnail url not found`) + if (!thumbnailURL.success) throw new Error(`Can't start working due to source: ${thumbnailURL.reason.tech}`) const sourceHTTPHandle = getSourceHTTPHandle(worker, sourceHandle, thumbnailURL) let wasCancelled = false @@ -176,20 +215,23 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { if (wasCancelled) return // ignore setImmediate(() => { // Copying is done - const duration = Date.now() - startTime - lookupTarget.handle - .updateMetadata(targetMetadata) - .then(() => { - workInProgress._reportComplete( - targetMetadata.sourceVersionHash, - `Thumbnail fetched in ${Math.round(duration / 100) / 10}s`, - undefined - ) - }) - .catch((err) => { - workInProgress._reportError(err) - }) + ;(async () => { + await lookupTarget.handle.finalizePackage() + await lookupTarget.handle.updateMetadata(targetMetadata) + + const duration = Date.now() - startTime + workInProgress._reportComplete( + targetMetadata.sourceVersionHash, + { + user: `Thumbnail generation completed in ${Math.round(duration / 100) / 10}s`, + tech: `Completed at ${Date.now()}`, + }, + undefined + ) + })().catch((err) => { + workInProgress._reportError(err) + }) }) }) }) @@ -204,11 +246,29 @@ export const QuantelThumbnail: ExpectationWindowsHandler = { removeExpectation: async (exp: Expectation.Any, worker: GenericWorker): Promise => { if (!isQuantelClipThumbnail(exp)) throw new Error(`Wrong exp.type: "${exp.type}"`) const lookupTarget = await lookupThumbnailTargets(worker, exp) - if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason}`) + if (!lookupTarget.ready) { + return { + removed: false, + reason: { + user: `Can't access target, due to: ${lookupTarget.reason.user}`, + tech: `No access to target: ${lookupTarget.reason.tech}`, + }, + } + } - await lookupTarget.handle.removePackage() + try { + await lookupTarget.handle.removePackage() + } catch (err) { + return { + removed: false, + reason: { + user: `Cannot remove file due to an internal error`, + tech: `Cannot remove preview file: ${err.toString()}`, + }, + } + } - return { removed: true, reason: 'Removed thumbnail' } + return { removed: true } }, } function isQuantelClipThumbnail(exp: Expectation.Any): exp is Expectation.QuantelClipThumbnail { @@ -233,13 +293,20 @@ function getMetadata(exp: Expectation.QuantelClipThumbnail, actualSourceVersion: async function getThumbnailURL( exp: Expectation.QuantelClipThumbnail, lookupSource: LookupPackageContainer -): Promise<{ baseURL: string; url: string } | undefined> { +): Promise<{ success: true; baseURL: string; url: string } | { success: false; reason: Reason }> { if (!lookupSource.accessor) throw new Error(`Source accessor not set!`) if (lookupSource.accessor.type !== Accessor.AccessType.QUANTEL) throw new Error(`Source accessor should have been a Quantel ("${lookupSource.accessor.type}")`) if (!isQuantelClipAccessorHandle(lookupSource.handle)) throw new Error(`Source AccessHandler type is wrong`) - if (!lookupSource.accessor.transformerURL) return undefined + if (!lookupSource.accessor.transformerURL) + return { + success: false, + reason: { + user: `transformerURL is not set in settings`, + tech: `transformerURL not set on accessor ${lookupSource.handle.accessorId}`, + }, + } const clip = await lookupSource.handle.getClip() if (clip) { @@ -255,25 +322,33 @@ async function getThumbnailURL( } return { + success: true, baseURL: lookupSource.accessor.transformerURL, url: `/quantel/homezone/clips/stills/${clip.ClipID}/${frame}.${width ? width + '.' : ''}jpg`, } + } else { + return { + success: false, + reason: { + user: `Source clip not found`, + tech: `Source clip not found`, + }, + } } - return undefined } export function getSourceHTTPHandle( worker: GenericWorker, sourceHandle: GenericAccessorHandle, thumbnailURL: { baseURL: string; url: string } -): HTTPAccessorHandle { +): HTTPProxyAccessorHandle { // This is a bit special, as we use the Quantel HTTP-transformer to extract the thumbnail, // so we have a QUANTEL source, but we construct an HTTP source from it to use instead: const handle = getAccessorHandle( worker, sourceHandle.accessorId + '__http', - literal({ - type: Accessor.AccessType.HTTP, + literal({ + type: Accessor.AccessType.HTTP_PROXY, baseUrl: thumbnailURL.baseURL, // networkId?: string url: thumbnailURL.url, @@ -281,7 +356,7 @@ export function getSourceHTTPHandle( { filePath: thumbnailURL.url }, {} ) - if (!isHTTPAccessorHandle(handle)) throw new Error(`getSourceHTTPHandle: got a non-HTTP handle!`) + if (!isHTTPProxyAccessorHandle(handle)) throw new Error(`getSourceHTTPHandle: got a non-HTTP handle!`) return handle } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts index ebfbbdd5..8e29fb4e 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/lib/lib.ts @@ -7,11 +7,12 @@ import { getAccessorCost, getAccessorStaticHandle } from '../../../accessorHandl import { GenericWorker } from '../../../worker' import { Expectation } from '@shared/api' import { prioritizeAccessors } from '../../../lib/lib' +import { AccessorHandlerResult } from '../../../accessorHandlers/genericHandle' export function compareActualExpectVersions( actualVersion: Expectation.Version.Any, expectVersion: Expectation.Version.ExpectAny -): undefined | string { +): AccessorHandlerResult { const expectProperties = makeUniversalVersion(expectVersion) const actualProperties = makeUniversalVersion(actualVersion) @@ -20,25 +21,37 @@ export function compareActualExpectVersions( const actual = actualProperties[key] as VersionProperty if (expect.value !== undefined && actual.value && expect.value !== actual.value) { - return `Actual ${actual.name} differ from expected (${expect.value}, ${actual.value})` + return { + success: false, + reason: { + user: 'Actual version differs from expected', + tech: `Actual ${actual.name} differ from expected (${expect.value}, ${actual.value})`, + }, + } } } - return undefined // All good! + return { success: true } } export function compareUniversalVersions( sourceVersion: UniversalVersion, targetVersion: UniversalVersion -): undefined | string { +): AccessorHandlerResult { for (const key of Object.keys(sourceVersion)) { const source = sourceVersion[key] as VersionProperty const target = targetVersion[key] as VersionProperty if (source.value !== target.value) { - return `Target ${source.name} differ from source (${target.value}, ${source.value})` + return { + success: false, + reason: { + user: 'Target version differs from Source', + tech: `Target ${source.name} differ from source (${target.value}, ${source.value})`, + }, + } } } - return undefined // All good! + return { success: true } } export function makeUniversalVersion( diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts b/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts index b642f886..ff97420c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/lib/robocopy.ts @@ -53,7 +53,10 @@ export function roboCopyFile(src: string, dst: string, progress?: (progress: num rbcpy.on('close', (code) => { rbcpy = undefined - if (code && (code & 1) === 1) { + if ( + code === 0 || // No errors occurred, and no copying was done. + (code && (code & 1) === 1) // One or more files were copied successfully (that is, new files have arrived). + ) { // Robocopy's code for succesfully copying files is 1 at LSB: https://ss64.com/nt/robocopy-exit.html if (srcFileName !== dstFileName) { fs.rename(path.join(dstFolder, srcFileName), path.join(dstFolder, dstFileName), (err) => { diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts index 58561d7b..79676dc0 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/packageContainerExpectationHandler.ts @@ -4,6 +4,7 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, ReturnTypeDisposePackageContainerMonitors, + Reason, } from '@shared/api' import { Accessor, PackageContainer, PackageContainerOnPackage } from '@sofie-automation/blueprints-integration' import { GenericAccessorHandle } from '../../accessorHandlers/genericHandle' @@ -21,36 +22,45 @@ export async function runPackageContainerCronJob( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { + // Quick-check: If there are no cronjobs at all, no need to check: + if (!Object.keys(packageContainer.cronjobs).length) { + return { success: true } // all good + } + const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'cronjob') - if (!lookup.ready) return { completed: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.runCronJob(packageContainer) - if (result) return { completed: false, reason: result } - else return { completed: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else return { success: true } // all good } export async function setupPackageContainerMonitors( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'monitor') - if (!lookup.ready) return { setupOk: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.setupPackageContainerMonitors(packageContainer) - if (result) return { setupOk: false, reason: result, monitors: {} } - else return { setupOk: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else + return { + success: true, + monitors: {}, // To me implemented: monitor ids + } } export async function disposePackageContainerMonitors( packageContainer: PackageContainerExpectation, genericWorker: GenericWorker ): Promise { const lookup = await lookupPackageContainer(genericWorker, packageContainer, 'monitor') - if (!lookup.ready) return { disposed: lookup.ready, reason: lookup.reason } + if (!lookup.ready) return { success: lookup.ready, reason: lookup.reason } const result = await lookup.handle.disposePackageContainerMonitors(packageContainer) - if (result) return { disposed: false, reason: result } - else return { disposed: true } // all good + if (!result.success) return { success: false, reason: result.reason } + else return { success: true } // all good } function checkWorkerHasAccessToPackageContainer( @@ -67,12 +77,15 @@ function checkWorkerHasAccessToPackageContainer( if (accessSourcePackageContainer) { return { support: true, - reason: `Has access to packageContainer "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"`, + // reason: `Has access to packageContainer "${accessSourcePackageContainer.packageContainer.label}" through accessor "${accessSourcePackageContainer.accessorId}"`, } } else { return { support: false, - reason: `Doesn't have access to the packageContainer (${containerId})`, + reason: { + user: `Worker doesn't support working with PackageContainer "${containerId}" (check settings?)`, + tech: `Worker doesn't have any access to the PackageContainer "${containerId}"`, + }, } } } @@ -116,11 +129,11 @@ export type LookupPackageContainer = accessor: Accessor.Any handle: GenericAccessorHandle ready: true - reason: string + reason: Reason } | { accessor: undefined handle: undefined ready: false - reason: string + reason: Reason } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts index cba7b30b..8245dc1c 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/windowsWorker.ts @@ -1,6 +1,7 @@ import { Expectation, ExpectationManagerWorkerAgent, + LoggerInstance, PackageContainerExpectation, ReturnTypeDisposePackageContainerMonitors, ReturnTypeDoYouSupportExpectation, @@ -12,6 +13,7 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, WorkerAgentConfig, + assertNever, } from '@shared/api' import { GenericWorker, WorkerLocation } from '../../worker' import { FileCopy } from './expectationHandlers/fileCopy' @@ -25,37 +27,49 @@ import { QuantelClipCopy } from './expectationHandlers/quantelClipCopy' import * as PackageContainerExpHandler from './packageContainerExpectationHandler' import { QuantelClipPreview } from './expectationHandlers/quantelClipPreview' import { QuantelThumbnail } from './expectationHandlers/quantelClipThumbnail' -import { assertNever } from '../../lib/lib' -import { hasFFMpeg, hasFFProbe } from './expectationHandlers/lib/ffmpeg' +import { testFFMpeg, testFFProbe } from './expectationHandlers/lib/ffmpeg' +import { JsonDataCopy } from './expectationHandlers/jsonDataCopy' /** This is a type of worker that runs on a windows machine */ export class WindowsWorker extends GenericWorker { static readonly type = 'windowsWorker' - public hasFFMpeg = false - public hasFFProbe = false + /** null = all is well */ + public testFFMpeg: null | string = 'Not initialized' + /** null = all is well */ + public testFFProbe: null | string = 'Not initialized' + + private monitor: NodeJS.Timer | undefined constructor( + logger: LoggerInstance, public readonly config: WorkerAgentConfig, sendMessageToManager: ExpectationManagerWorkerAgent.MessageFromWorker, location: WorkerLocation ) { - super(config, location, sendMessageToManager, WindowsWorker.type) + super(logger, config, location, sendMessageToManager, WindowsWorker.type) } async doYouSupportExpectation(exp: Expectation.Any): Promise { - try { - return this.getExpectationHandler(exp).doYouSupportExpectation(exp, this, this) - } catch (err) { - // Does not support the type - return { - support: false, - reason: err.toString(), - } - } + return this.getExpectationHandler(exp).doYouSupportExpectation(exp, this, this) } async init(): Promise { - this.hasFFMpeg = !!(await hasFFMpeg()) - this.hasFFProbe = !!(await hasFFProbe()) + await this.checkExecutables() + this.monitor = setInterval(() => { + this.checkExecutables().catch((err) => { + this.logger.error('Error in checkExecutables') + this.logger.error(err) + }) + }, 10 * 1000) + } + terminate(): void { + if (this.monitor) { + clearInterval(this.monitor) + delete this.monitor + } + } + private async checkExecutables() { + this.testFFMpeg = await testFFMpeg() + this.testFFProbe = await testFFProbe() } getCostFortExpectation(exp: Expectation.Any): Promise { return this.getExpectationHandler(exp).getCostForExpectation(exp, this, this) @@ -90,6 +104,8 @@ export class WindowsWorker extends GenericWorker { return QuantelThumbnail case Expectation.Type.QUANTEL_CLIP_PREVIEW: return QuantelClipPreview + case Expectation.Type.JSON_DATA_COPY: + return JsonDataCopy default: assertNever(exp) // @ts-expect-error exp.type is never diff --git a/shared/packages/worker/src/workerAgent.ts b/shared/packages/worker/src/workerAgent.ts index f51d0546..e4575508 100644 --- a/shared/packages/worker/src/workerAgent.ts +++ b/shared/packages/worker/src/workerAgent.ts @@ -16,7 +16,10 @@ import { ReturnTypeRunPackageContainerCronJob, ReturnTypeSetupPackageContainerMonitors, ReturnTypeDisposePackageContainerMonitors, + LogLevel, + APPCONTAINER_PING_TIME, } from '@shared/api' +import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' import { IWorkInProgress } from './worker/lib/workInProgress' import { GenericWorker } from './worker/worker' @@ -31,11 +34,13 @@ export class WorkerAgent { // private _busyMethodCount = 0 private currentJobs: { cost: ExpectationManagerWorkerAgent.ExpectationCost; progress: number }[] = [] private workforceAPI: WorkforceAPI + private appContainerAPI: AppContainerAPI private wipI = 0 private worksInProgress: { [wipId: string]: IWorkInProgress } = {} - private id: string - private connectionOptions: ClientConnectionOptions + public readonly id: string + private workForceConnectionOptions: ClientConnectionOptions + private appContainerConnectionOptions: ClientConnectionOptions | null private expectationManagers: { [id: string]: { @@ -49,12 +54,18 @@ export class WorkerAgent { ExpectationManagerWorkerAgent.WorkerAgent > } = {} + private terminated = false + private spinDownTime = 0 + private intervalCheckTimer: NodeJS.Timer | null = null + private lastWorkTime = 0 + private activeMonitors: { [monitorId: string]: true } = {} constructor(private logger: LoggerInstance, private config: WorkerConfig) { - this.workforceAPI = new WorkforceAPI() + this.workforceAPI = new WorkforceAPI(this.logger) + this.appContainerAPI = new AppContainerAPI(this.logger) this.id = config.worker.workerId - this.connectionOptions = this.config.worker.workforceURL + this.workForceConnectionOptions = this.config.worker.workforceURL ? { type: 'websocket', url: this.config.worker.workforceURL, @@ -62,9 +73,15 @@ export class WorkerAgent { : { type: 'internal', } - + this.appContainerConnectionOptions = this.config.worker.appContainerURL + ? { + type: 'websocket', + url: this.config.worker.appContainerURL, + } + : null // Todo: Different types of workers: this._worker = new WindowsWorker( + this.logger, this.config.worker, async (managerId: string, message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { // Forward the message to the expectationManager: @@ -72,7 +89,7 @@ export class WorkerAgent { const manager = this.expectationManagers[managerId] if (!manager) throw new Error(`ExpectationManager "${managerId}" not found`) - return await manager.api.messageFromWorker(message) + return manager.api.messageFromWorker(message) }, { // todo: tmp: @@ -82,17 +99,42 @@ export class WorkerAgent { ) } async init(): Promise { - await this.workforceAPI.init(this.id, this.connectionOptions, this) + await this._worker.init() + + // Connect to WorkForce: + if (this.workForceConnectionOptions.type === 'websocket') { + this.logger.info(`Worker: Connecting to Workforce at "${this.workForceConnectionOptions.url}"`) + } + await this.workforceAPI.init(this.id, this.workForceConnectionOptions, this) + + // Connect to AppContainer (if applicable) + if (this.appContainerConnectionOptions) { + if (this.appContainerConnectionOptions.type === 'websocket') { + this.logger.info(`Worker: Connecting to AppContainer at "${this.appContainerConnectionOptions.url}"`) + } + await this.appContainerAPI.init(this.id, this.appContainerConnectionOptions, this) + } const list = await this.workforceAPI.getExpectationManagerList() await this.updateListOfExpectationManagers(list) - await this._worker.init() + this.IDidSomeWork() } terminate(): void { + this.terminated = true this.workforceAPI.terminate() - Object.values(this.expectationManagers).forEach((expectationManager) => expectationManager.api.terminate()) - // this._worker.terminate() + + for (const expectationManager of Object.values(this.expectationManagers)) { + expectationManager.api.terminate() + } + for (const wipId of Object.keys(this.worksInProgress)) { + this.cancelWorkInProgress(wipId).catch((error) => { + this.logger.error('WorkerAgent.terminate: Error in cancelWorkInProgress') + this.logger.error(error) + }) + } + if (this.intervalCheckTimer) clearInterval(this.intervalCheckTimer) + this._worker.terminate() } /** Called when running in the same-process-mode, it */ hookToWorkforce(hook: Hook): void { @@ -120,6 +162,10 @@ export class WorkerAgent { // isFree(): boolean { // return this._busyMethodCount === 0 // } + async doYouSupportExpectation(exp: Expectation.Any): Promise { + this.IDidSomeWork() + return this._worker.doYouSupportExpectation(exp) + } async expectationManagerAvailable(id: string, url: string): Promise { const existing = this.expectationManagers[id] if (existing) { @@ -131,16 +177,33 @@ export class WorkerAgent { async expectationManagerGone(id: string): Promise { delete this.expectationManagers[id] } + public async setLogLevel(logLevel: LogLevel): Promise { + this.logger.level = logLevel + } + async _debugKill(): Promise { + this.terminate() + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } + public async setSpinDownTime(spinDownTime: number): Promise { + this.spinDownTime = spinDownTime + this.IDidSomeWork() + + this.setupIntervalCheck() + } private async connectToExpectationManager(id: string, url: string): Promise { - this.logger.info(`Connecting to Expectation Manager "${id}" at url "${url}"`) + this.logger.info(`Worker: Connecting to Expectation Manager "${id}" at url "${url}"`) const expectedManager = (this.expectationManagers[id] = { url: url, - api: new ExpectationManagerAPI(), + api: new ExpectationManagerAPI(this.logger), }) const methods: ExpectationManagerWorkerAgent.WorkerAgent = literal({ doYouSupportExpectation: async (exp: Expectation.Any): Promise => { - return await this._worker.doYouSupportExpectation(exp) + return this._worker.doYouSupportExpectation(exp) }, getCostForExpectation: async ( exp: Expectation.Any @@ -161,82 +224,149 @@ export class WorkerAgent { exp: Expectation.Any, wasFullfilled: boolean ): Promise => { + this.IDidSomeWork() return this._worker.isExpectationFullfilled(exp, wasFullfilled) }, workOnExpectation: async ( exp: Expectation.Any, cost: ExpectationManagerWorkerAgent.ExpectationCost ): Promise => { + this.IDidSomeWork() const currentjob = { cost: cost, progress: 0, // callbacksOnDone: [], } + const wipId = this.wipI++ + this.logger.debug( + `Worker "${this.id}" starting job ${wipId}, (${exp.id}). (${this.currentJobs.length})` + ) this.currentJobs.push(currentjob) - const wipId = this.wipI++ + try { + const workInProgress = await this._worker.workOnExpectation(exp) - const workInProgress = await this._worker.workOnExpectation(exp) + this.worksInProgress[`${wipId}`] = workInProgress - this.worksInProgress[`${wipId}`] = workInProgress + workInProgress.on('progress', (actualVersionHash, progress: number) => { + this.IDidSomeWork() + currentjob.progress = progress + expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch((err) => { + if (!this.terminated) { + this.logger.error('Error in wipEventProgress') + this.logger.error(err) + } + }) + }) + workInProgress.on('error', (error: string) => { + this.IDidSomeWork() + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + this.logger.debug( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to error. (${this.currentJobs.length})` + ) - workInProgress.on('progress', (actualVersionHash, progress: number) => { - currentjob.progress = progress - expectedManager.api.wipEventProgress(wipId, actualVersionHash, progress).catch(console.error) - }) - workInProgress.on('error', (error) => { - this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + expectedManager.api + .wipEventError(wipId, { + user: 'Work aborted due to an error', + tech: error, + }) + .catch((err) => { + if (!this.terminated) { + this.logger.error('Error in wipEventError') + this.logger.error(err) + } + }) + delete this.worksInProgress[`${wipId}`] + }) + workInProgress.on('done', (actualVersionHash, reason, result) => { + this.IDidSomeWork() + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + this.logger.debug( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), done. (${this.currentJobs.length})` + ) - expectedManager.api.wipEventError(wipId, error).catch(console.error) - delete this.worksInProgress[`${wipId}`] - }) - workInProgress.on('done', (actualVersionHash, reason, result) => { - this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch((err) => { + if (!this.terminated) { + this.logger.error('Error in wipEventDone') + this.logger.error(err) + } + }) + delete this.worksInProgress[`${wipId}`] + }) - expectedManager.api.wipEventDone(wipId, actualVersionHash, reason, result).catch(console.error) - delete this.worksInProgress[`${wipId}`] - }) + return { + wipId: wipId, + properties: workInProgress.properties, + } + } catch (err) { + // The workOnExpectation failed. - return { - wipId: wipId, - properties: workInProgress.properties, - } + this.currentJobs = this.currentJobs.filter((job) => job !== currentjob) + this.logger.debug( + `Worker "${this.id}" stopped job ${wipId}, (${exp.id}), due to initial error. (${this.currentJobs.length})` + ) - // return workInProgress + throw err + } }, removeExpectation: async (exp: Expectation.Any): Promise => { + this.IDidSomeWork() return this._worker.removeExpectation(exp) }, cancelWorkInProgress: async (wipId: number): Promise => { - const wip = this.worksInProgress[`${wipId}`] - if (wip) { - await wip.cancel() - } - delete this.worksInProgress[`${wipId}`] + this.IDidSomeWork() + return this.cancelWorkInProgress(wipId) }, doYouSupportPackageContainer: ( packageContainer: PackageContainerExpectation ): Promise => { + this.IDidSomeWork() return this._worker.doYouSupportPackageContainer(packageContainer) }, runPackageContainerCronJob: ( packageContainer: PackageContainerExpectation ): Promise => { + this.IDidSomeWork() return this._worker.runPackageContainerCronJob(packageContainer) }, - setupPackageContainerMonitors: ( + setupPackageContainerMonitors: async ( packageContainer: PackageContainerExpectation ): Promise => { - return this._worker.setupPackageContainerMonitors(packageContainer) + this.IDidSomeWork() + const monitors = await this._worker.setupPackageContainerMonitors(packageContainer) + if (monitors.success) { + for (const monitorId of Object.keys(monitors.monitors)) { + this.activeMonitors[monitorId] = true + } + } + return monitors }, - disposePackageContainerMonitors: ( + disposePackageContainerMonitors: async ( packageContainer: PackageContainerExpectation ): Promise => { - return this._worker.disposePackageContainerMonitors(packageContainer) + this.IDidSomeWork() + const success = await this._worker.disposePackageContainerMonitors(packageContainer) + if (success.success) { + this.activeMonitors = {} + } + return success }, }) + // Wrap the methods, so that we can cut off communication upon termination: (this is used in tests) + for (const key of Object.keys(methods) as Array) { + const fcn = methods[key] as any + methods[key] = ((...args: any[]) => { + if (this.terminated) + return new Promise((_resolve, reject) => { + // Simulate a timed out message: + setTimeout(() => { + reject('Timeout') + }, 200) + }) + return fcn(...args) + }) as any + } // Connect to the ExpectationManager: - if (url === '__internal') { // This is used for an internal connection: const managerHookHook = this.expectationManagerHooks[id] @@ -273,4 +403,37 @@ export class WorkerAgent { } } } + private async cancelWorkInProgress(wipId: string | number): Promise { + const wip = this.worksInProgress[`${wipId}`] + if (wip) { + await wip.cancel() + } + delete this.worksInProgress[`${wipId}`] + } + private setupIntervalCheck() { + if (!this.intervalCheckTimer) { + this.intervalCheckTimer = setInterval(() => { + this.intervalCheck() + }, APPCONTAINER_PING_TIME) + } + } + private intervalCheck() { + // Check the SpinDownTime: + if (this.spinDownTime) { + if (Date.now() - this.lastWorkTime > this.spinDownTime) { + this.IDidSomeWork() // so that we won't ask again until later + + // Don's spin down if a monitor is active + if (!Object.keys(this.activeMonitors).length) { + this.logger.info(`Worker: is idle, requesting spinning down`) + this.appContainerAPI.requestSpinDown() + } + } + } + // Also ping the AppContainer + this.appContainerAPI.ping() + } + private IDidSomeWork() { + this.lastWorkTime = Date.now() + } } diff --git a/shared/packages/worker/src/workforceApi.ts b/shared/packages/worker/src/workforceApi.ts index d7bd28c4..4e9d83de 100644 --- a/shared/packages/worker/src/workforceApi.ts +++ b/shared/packages/worker/src/workforceApi.ts @@ -1,4 +1,4 @@ -import { AdapterClient, WorkForceWorkerAgent } from '@shared/api' +import { AdapterClient, LoggerInstance, WorkForceWorkerAgent } from '@shared/api' /** * Exposes the API-methods of a Workforce, to be called from the WorkerAgent @@ -8,10 +8,10 @@ import { AdapterClient, WorkForceWorkerAgent } from '@shared/api' export class WorkforceAPI extends AdapterClient implements WorkForceWorkerAgent.WorkForce { - constructor() { - super('workerAgent') + constructor(logger: LoggerInstance) { + super(logger, 'workerAgent') } async getExpectationManagerList(): Promise<{ id: string; url: string }[]> { - return await this._sendMessage('getExpectationManagerList', undefined) + return this._sendMessage('getExpectationManagerList', undefined) } } diff --git a/shared/packages/workforce/package.json b/shared/packages/workforce/package.json index 21c432c8..f923d45a 100644 --- a/shared/packages/workforce/package.json +++ b/shared/packages/workforce/package.json @@ -24,7 +24,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ @@ -34,4 +34,4 @@ "eslint" ] } -} +} \ No newline at end of file diff --git a/shared/packages/workforce/src/appContainerApi.ts b/shared/packages/workforce/src/appContainerApi.ts new file mode 100644 index 00000000..fc64bbd8 --- /dev/null +++ b/shared/packages/workforce/src/appContainerApi.ts @@ -0,0 +1,37 @@ +import { WorkForceAppContainer, AdapterServer, AdapterServerOptions, LogLevel, Expectation } from '@shared/api' + +/** + * Exposes the API-methods of a AppContainer, to be called from the Workforce + * Note: The AppContainer connects to the Workforce, therefore the Workforce is the AdapterServer here. + * The corresponding other side is implemented at shared/packages/worker/src/workforceApi.ts + */ +export class AppContainerAPI + extends AdapterServer + implements WorkForceAppContainer.AppContainer { + constructor( + methods: WorkForceAppContainer.WorkForce, + options: AdapterServerOptions + ) { + super(methods, options) + } + + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + + async requestAppTypeForExpectation(exp: Expectation.Any): Promise<{ appType: string; cost: number } | null> { + return this._sendMessage('requestAppTypeForExpectation', exp) + } + async spinUp(appType: string): Promise { + return this._sendMessage('spinUp', appType) + } + async spinDown(appId: string): Promise { + return this._sendMessage('spinDown', appId) + } + async getRunningApps(): Promise<{ appId: string; appType: string }[]> { + return this._sendMessage('getRunningApps') + } +} diff --git a/shared/packages/workforce/src/expectationManagerApi.ts b/shared/packages/workforce/src/expectationManagerApi.ts index d0a63e11..a02747a6 100644 --- a/shared/packages/workforce/src/expectationManagerApi.ts +++ b/shared/packages/workforce/src/expectationManagerApi.ts @@ -1,4 +1,4 @@ -import { WorkForceExpectationManager, AdapterServer, AdapterServerOptions } from '@shared/api' +import { WorkForceExpectationManager, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' /** * Exposes the API-methods of a ExpectationManager, to be called from the Workforce @@ -15,5 +15,10 @@ export class ExpectationManagerAPI super(methods, options) } - // Note: This side of the API has no methods exposed. + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } } diff --git a/shared/packages/workforce/src/workerAgentApi.ts b/shared/packages/workforce/src/workerAgentApi.ts index 7cb984d4..0e785763 100644 --- a/shared/packages/workforce/src/workerAgentApi.ts +++ b/shared/packages/workforce/src/workerAgentApi.ts @@ -1,4 +1,4 @@ -import { WorkForceWorkerAgent, AdapterServer, AdapterServerOptions } from '@shared/api' +import { WorkForceWorkerAgent, AdapterServer, AdapterServerOptions, LogLevel } from '@shared/api' /** * Exposes the API-methods of a WorkerAgent, to be called from the Workforce @@ -15,10 +15,17 @@ export class WorkerAgentAPI super(methods, options) } + async setLogLevel(logLevel: LogLevel): Promise { + return this._sendMessage('setLogLevel', logLevel) + } + async _debugKill(): Promise { + return this._sendMessage('_debugKill') + } + async expectationManagerAvailable(id: string, url: string): Promise { - return await this._sendMessage('expectationManagerAvailable', id, url) + return this._sendMessage('expectationManagerAvailable', id, url) } async expectationManagerGone(id: string): Promise { - return await this._sendMessage('expectationManagerGone', id) + return this._sendMessage('expectationManagerGone', id) } } diff --git a/shared/packages/workforce/src/workerHandler.ts b/shared/packages/workforce/src/workerHandler.ts new file mode 100644 index 00000000..d0755eed --- /dev/null +++ b/shared/packages/workforce/src/workerHandler.ts @@ -0,0 +1,45 @@ +import { Expectation, LoggerInstance } from '@shared/api' +import { Workforce } from './workforce' + +/** The WorkerHandler is in charge of spinning up/down Workers */ +export class WorkerHandler { + private logger: LoggerInstance + + constructor(private workForce: Workforce) { + this.logger = workForce.logger + } + public terminate(): void { + // nothing? + } + public async requestResources(exp: Expectation.Any): Promise { + let best: { appContainerId: string; appType: string; cost: number } | null = null + this.logger.debug(`Workforce: Got request for resources for exp "${exp.id}"`) + for (const [appContainerId, appContainer] of Object.entries(this.workForce.appContainers)) { + this.logger.debug(`Workforce: Asking appContainer "${appContainerId}"`) + const proposal = await appContainer.api.requestAppTypeForExpectation(exp) + if (proposal) { + if (!best || proposal.cost < best.cost) { + best = { + appContainerId: appContainerId, + appType: proposal.appType, + cost: proposal.cost, + } + } + } + } + if (best) { + this.logger.debug(`Workforce: Selecting appContainer "${best.appContainerId}"`) + + const appContainer = this.workForce.appContainers[best.appContainerId] + if (!appContainer) throw new Error(`WorkerHandler: AppContainer "${best.appContainerId}" not found`) + + this.logger.debug(`Workforce: Spinning up another worker (${best.appType}) on "${best.appContainerId}"`) + + await appContainer.api.spinUp(best.appType) + return true + } else { + this.logger.debug(`Workforce: No resources available`) + return false + } + } +} diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index a26f6ad1..72b2dbb5 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -6,16 +6,23 @@ import { Hook, LoggerInstance, WorkforceConfig, + assertNever, + WorkForceAppContainer, + WorkforceStatus, + LogLevel, + Expectation, } from '@shared/api' +import { AppContainerAPI } from './appContainerApi' import { ExpectationManagerAPI } from './expectationManagerApi' import { WorkerAgentAPI } from './workerAgentApi' +import { WorkerHandler } from './workerHandler' /** * The Workforce class tracks the status of which ExpectationManagers and WorkerAgents are online, * and mediates connections between the two. */ export class Workforce { - private workerAgents: { + public workerAgents: { [workerId: string]: { api: WorkerAgentAPI } @@ -27,38 +34,87 @@ export class Workforce { url?: string } } = {} + public appContainers: { + [id: string]: { + api: AppContainerAPI + initialized: boolean + runningApps: { + appId: string + appType: string + }[] + availableApps: { + appType: string + }[] + } + } = {} private websocketServer?: WebsocketServer - constructor(private logger: LoggerInstance, config: WorkforceConfig) { - if (config.workforce.port) { + private workerHandler: WorkerHandler + + constructor(public logger: LoggerInstance, config: WorkforceConfig) { + if (config.workforce.port !== null) { this.websocketServer = new WebsocketServer(config.workforce.port, (client: ClientConnection) => { // A new client has connected - this.logger.info(`New ${client.clientType} connected, id "${client.clientId}"`) - - if (client.clientType === 'workerAgent') { - const workForceMethods = this.getWorkerAgentAPI() - const api = new WorkerAgentAPI(workForceMethods, { - type: 'websocket', - clientConnection: client, - }) - this.workerAgents[client.clientId] = { api } - } else if (client.clientType === 'expectationManager') { - const workForceMethods = this.getExpectationManagerAPI() - const api = new ExpectationManagerAPI(workForceMethods, { - type: 'websocket', - clientConnection: client, - }) - this.expectationManagers[client.clientId] = { api } - } else { - throw new Error(`Unknown clientType "${client.clientType}"`) + this.logger.info(`Workforce: New client "${client.clientType}" connected, id "${client.clientId}"`) + + switch (client.clientType) { + case 'workerAgent': { + const workForceMethods = this.getWorkerAgentAPI() + const api = new WorkerAgentAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.workerAgents[client.clientId] = { api } + client.on('close', () => { + delete this.workerAgents[client.clientId] + }) + break + } + case 'expectationManager': { + const workForceMethods = this.getExpectationManagerAPI() + const api = new ExpectationManagerAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.expectationManagers[client.clientId] = { api } + client.on('close', () => { + delete this.expectationManagers[client.clientId] + }) + break + } + case 'appContainer': { + const workForceMethods = this.getAppContainerAPI(client.clientId) + const api = new AppContainerAPI(workForceMethods, { + type: 'websocket', + clientConnection: client, + }) + this.appContainers[client.clientId] = { + api, + availableApps: [], + runningApps: [], + initialized: false, + } + client.on('close', () => { + delete this.appContainers[client.clientId] + }) + break + } + + case 'N/A': + throw new Error(`ExpectationManager: Unsupported clientType "${client.clientType}"`) + default: + assertNever(client.clientType) + throw new Error(`Workforce: Unknown clientType "${client.clientType}"`) } }) } + this.workerHandler = new WorkerHandler(this) } async init(): Promise { // Nothing to do here at the moment + // this.workerHandler.triggerUpdate() } terminate(): void { this.websocketServer?.terminate() @@ -94,6 +150,9 @@ export class Workforce { return workForceMethods } } + getPort(): number | undefined { + return this.websocketServer?.port + } /** Return the API-methods that the Workforce exposes to the WorkerAgent */ private getWorkerAgentAPI(): WorkForceWorkerAgent.WorkForce { @@ -116,17 +175,50 @@ export class Workforce { /** Return the API-methods that the Workforce exposes to the ExpectationManager */ private getExpectationManagerAPI(): WorkForceExpectationManager.WorkForce { return { + setLogLevel: async (logLevel: LogLevel): Promise => { + return this.setLogLevel(logLevel) + }, + setLogLevelOfApp: async (appId: string, logLevel: LogLevel): Promise => { + return this.setLogLevelOfApp(appId, logLevel) + }, registerExpectationManager: async (managerId: string, url: string): Promise => { await this.registerExpectationManager(managerId, url) }, + requestResources: async (exp: Expectation.Any): Promise => { + return this.requestResources(exp) + }, + + getStatus: async (): Promise => { + return this.getStatus() + }, + _debugKillApp: async (appId: string): Promise => { + return this._debugKillApp(appId) + }, + } + } + /** Return the API-methods that the Workforce exposes to the AppContainer */ + private getAppContainerAPI(clientId: string): WorkForceAppContainer.WorkForce { + return { + registerAvailableApps: async (availableApps: { appType: string }[]): Promise => { + await this.registerAvailableApps(clientId, availableApps) + }, } } + private _debugKill(): void { + // This is for testing purposes only + setTimeout(() => { + // eslint-disable-next-line no-process-exit + process.exit(42) + }, 1) + } public async registerExpectationManager(managerId: string, url: string): Promise { const em = this.expectationManagers[managerId] if (!em || em.url !== url) { // Added/Changed + this.logger.info(`Workforce: Register ExpectationManager (${managerId}) at url "${url}"`) + // Announce the new expectation manager to the workerAgents: for (const workerAgent of Object.values(this.workerAgents)) { await workerAgent.api.expectationManagerAvailable(managerId, url) @@ -134,6 +226,66 @@ export class Workforce { } this.expectationManagers[managerId].url = url } + public async requestResources(exp: Expectation.Any): Promise { + return this.workerHandler.requestResources(exp) + } + public async getStatus(): Promise { + return { + workerAgents: Object.entries(this.workerAgents).map(([workerId, _workerAgent]) => { + return { + id: workerId, + } + }), + expectationManagers: Object.entries(this.expectationManagers).map(([id, expMan]) => { + return { + id: id, + url: expMan.url, + } + }), + appContainers: Object.entries(this.appContainers).map(([id, appContainer]) => { + return { + id: id, + initialized: appContainer.initialized, + availableApps: appContainer.availableApps.map((availableApp) => { + return { + appType: availableApp.appType, + } + }), + } + }), + } + } + + public setLogLevel(logLevel: LogLevel): void { + this.logger.level = logLevel + } + public async setLogLevelOfApp(appId: string, logLevel: LogLevel): Promise { + const workerAgent = this.workerAgents[appId] + if (workerAgent) return workerAgent.api.setLogLevel(logLevel) + + const appContainer = this.appContainers[appId] + if (appContainer) return appContainer.api.setLogLevel(logLevel) + + const expectationManager = this.expectationManagers[appId] + if (expectationManager) return expectationManager.api.setLogLevel(logLevel) + + if (appId === 'workforce') return this.setLogLevel(logLevel) + throw new Error(`App with id "${appId}" not found`) + } + public async _debugKillApp(appId: string): Promise { + const workerAgent = this.workerAgents[appId] + if (workerAgent) return workerAgent.api._debugKill() + + const appContainer = this.appContainers[appId] + if (appContainer) return appContainer.api._debugKill() + + const expectationManager = this.expectationManagers[appId] + if (expectationManager) return expectationManager.api._debugKill() + + if (appId === 'workforce') return this._debugKill() + throw new Error(`App with id "${appId}" not found`) + } + public async removeExpectationManager(managerId: string): Promise { const em = this.expectationManagers[managerId] if (em) { @@ -144,4 +296,20 @@ export class Workforce { } } } + public async registerAvailableApps(clientId: string, availableApps: { appType: string }[]): Promise { + this.appContainers[clientId].availableApps = availableApps + + // Ask the AppContainer for a list of its running apps: + this.appContainers[clientId].api + .getRunningApps() + .then((runningApps) => { + this.appContainers[clientId].runningApps = runningApps + this.appContainers[clientId].initialized = true + // this.workerHandler.triggerUpdate() + }) + .catch((error) => { + this.logger.error('Workforce: Error in getRunningApps') + this.logger.error(error) + }) + } } diff --git a/apps/tests/internal-tests/README.md b/tests/internal-tests/README.md similarity index 100% rename from apps/tests/internal-tests/README.md rename to tests/internal-tests/README.md diff --git a/tests/internal-tests/jest.config.js b/tests/internal-tests/jest.config.js new file mode 100644 index 00000000..9fe8df7f --- /dev/null +++ b/tests/internal-tests/jest.config.js @@ -0,0 +1,8 @@ +const base = require('../../jest.config.base'); +const packageJson = require('./package'); + +module.exports = { + ...base, + name: packageJson.name, + displayName: packageJson.name, +}; diff --git a/apps/tests/internal-tests/package.json b/tests/internal-tests/package.json similarity index 89% rename from apps/tests/internal-tests/package.json rename to tests/internal-tests/package.json index 3e5f16a4..619ac3cb 100644 --- a/apps/tests/internal-tests/package.json +++ b/tests/internal-tests/package.json @@ -1,12 +1,12 @@ { "name": "@tests/internal-tests", "version": "1.0.0", - "description": "Package Manager, http-proxy etc.. all in one application", + "description": "Internal tests", "private": true, "scripts": { "build": "rimraf dist && yarn build:main", "build:main": "tsc -p tsconfig.json", - "test": "jest", + "test": "jest --runInBand", "precommit": "lint-staged" }, "devDependencies": { @@ -30,7 +30,7 @@ }, "prettier": "@sofie-automation/code-standard-preset/.prettierrc.json", "engines": { - "node": ">=12.3.0" + "node": ">=12.20.0" }, "lint-staged": { "*.{js,css,json,md,scss}": [ diff --git a/apps/tests/internal-tests/src/__mocks__/child_process.ts b/tests/internal-tests/src/__mocks__/child_process.ts similarity index 87% rename from apps/tests/internal-tests/src/__mocks__/child_process.ts rename to tests/internal-tests/src/__mocks__/child_process.ts index 9bcfcd67..53c8e72d 100644 --- a/apps/tests/internal-tests/src/__mocks__/child_process.ts +++ b/tests/internal-tests/src/__mocks__/child_process.ts @@ -3,11 +3,12 @@ import EventEmitter from 'events' import { promisify } from 'util' import path from 'path' +/* eslint-disable no-console */ + const fsCopyFile = promisify(fs.copyFile) -// @ts-expect-error mock -const fs__mockSetDirectory = fs.__mockSetDirectory +const fsMkdir = promisify(fs.mkdir) -const child_process = jest.createMockFromModule('child_process') as any +const child_process: any = jest.createMockFromModule('child_process') function exec(commandString: string) { if (commandString.match(/^wmic /)) { @@ -41,6 +42,8 @@ function spawn(command: string, args: string[] = []) { spawned.emit('close', 9999) }) }) + } else if (command === 'taskkill') { + // mock killing a task? } else { throw new Error(`Mock child_process.spawn: command not implemented: "${command}"`) } @@ -51,6 +54,11 @@ child_process.spawn = spawn class SpawnedProcess extends EventEmitter { public stdout = new EventEmitter() public stderr = new EventEmitter() + public pid: number + constructor() { + super() + this.pid = Date.now() + } } async function robocopy(spawned: SpawnedProcess, args: string[]) { let sourceFolder @@ -121,14 +129,14 @@ async function robocopy(spawned: SpawnedProcess, args: string[]) { const source = path.join(sourceFolder, file) const destination = path.join(destinationFolder, file) - fs__mockSetDirectory(destinationFolder) // robocopy automatically creates the destination folder + await fsMkdir(destinationFolder) // robocopy automatically creates the destination folder await fsCopyFile(source, destination) } spawned.emit('close', 1) // OK } catch (err) { - console.log(err) - spawned.emit('close', 9999) + // console.log(err) + spawned.emit('close', 16) // Serious error. Robocopy did not copy any files. } } async function ffmpeg(spawned: SpawnedProcess, args: string[]) { diff --git a/apps/tests/internal-tests/src/__mocks__/fs.ts b/tests/internal-tests/src/__mocks__/fs.ts similarity index 82% rename from apps/tests/internal-tests/src/__mocks__/fs.ts rename to tests/internal-tests/src/__mocks__/fs.ts index 45518fe4..ecd76d15 100644 --- a/apps/tests/internal-tests/src/__mocks__/fs.ts +++ b/tests/internal-tests/src/__mocks__/fs.ts @@ -1,6 +1,9 @@ // eslint-disable-next-line node/no-unpublished-import import fsMockType from 'windows-network-drive' // Note: this is a mocked module +import { EventEmitter } from 'events' // Note: this is a mocked module +// import * as Path from 'path' +/* eslint-disable no-console */ const DEBUG_LOG = false enum fsConstants { @@ -8,7 +11,7 @@ enum fsConstants { W_OK = 4, } -const fs = jest.createMockFromModule('fs') as any +const fs: any = jest.createMockFromModule('fs') type MockAny = MockDirectory | MockFile interface MockBase { @@ -33,7 +36,8 @@ const mockRoot: MockDirectory = { content: {}, } const openFileDescriptors: { [fd: string]: MockAny } = {} -let fd = 0 +let fdId = 0 +const fsMockEmitter = new EventEmitter() function getMock(path: string, orgPath?: string, dir?: MockDirectory): MockAny { dir = dir || mockRoot @@ -74,7 +78,7 @@ function getMock(path: string, orgPath?: string, dir?: MockDirectory): MockAny { path: path, }) } -function setMock(path: string, create: MockAny, autoCreateTree: boolean, dir?: MockDirectory): void { +function setMock(path: string, create: MockAny, autoCreateTree: boolean, force = false, dir?: MockDirectory): void { dir = dir || mockRoot const m = path.match(/([^/]+)\/(.*)/) @@ -118,10 +122,20 @@ function setMock(path: string, create: MockAny, autoCreateTree: boolean, dir?: M } if (nextDir) { - return setMock(nextPath, create, autoCreateTree, nextDir) + return setMock(nextPath, create, autoCreateTree, force, nextDir) } } else { const fileName = path + if (dir.content[fileName]) { + if (!dir.content[fileName].accessWrite && !force) { + throw Object.assign(new Error(`EACCESS: Not able to write to parent folder "${path}"`), { + errno: 0, + code: 'EACCESS', + syscall: 'mock', + path: path, + }) + } + } dir.content[fileName] = create if (DEBUG_LOG) console.log('setMock', path, create) @@ -174,8 +188,6 @@ export function __printAllFiles(): string { const getPaths = (dir: MockDirectory, indent: string): string => { const strs: any[] = [] for (const [name, file] of Object.entries(dir.content)) { - // const path = `${prevPath}/${name}` - if (file.isDirectory) { strs.push(`${indent}${name}/`) strs.push(getPaths(file, indent + ' ')) @@ -211,47 +223,53 @@ function fixPath(path: string) { export function __mockReset(): void { Object.keys(mockRoot.content).forEach((filePath) => delete mockRoot.content[filePath]) + fsMockEmitter.removeAllListeners() } fs.__mockReset = __mockReset -export function __mockSetFile(path: string, size: number): void { +export function __mockSetFile(path: string, size: number, accessOptions?: FileAccess): void { path = fixPath(path) setMock( path, { - accessRead: true, - accessWrite: true, + accessRead: accessOptions?.accessRead ?? true, + accessWrite: accessOptions?.accessWrite ?? true, + isDirectory: false, content: 'mockContent', size: size, }, + true, true ) } fs.__mockSetFile = __mockSetFile -export function __mockSetDirectory(path: string): void { +export function __mockSetDirectory(path: string, accessOptions?: FileAccess): void { path = fixPath(path) setMock( path, { - accessRead: true, - accessWrite: true, + accessRead: accessOptions?.accessRead ?? true, + accessWrite: accessOptions?.accessWrite ?? true, + isDirectory: true, content: {}, }, + true, true ) } fs.__mockSetDirectory = __mockSetDirectory -// export function readdirSync(directoryPath: string) { -// return mockFiles[directoryPath] || [] -// } -// fs.readdirSync = readdirSync +export function __emitter(): EventEmitter { + return fsMockEmitter +} +fs.__emitter = __emitter export function stat(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.stat', path) + fsMockEmitter.emit('stat', path) try { const mockFile = getMock(path) if (mockFile.isDirectory) { @@ -272,6 +290,7 @@ fs.stat = stat export function access(path: string, mode: number | undefined, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.access', path, mode) + fsMockEmitter.emit('access', path, mode) try { const mockFile = getMock(path) if (mode === fsConstants.R_OK && !mockFile.accessRead) { @@ -290,6 +309,7 @@ fs.access = access export function unlink(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.unlink', path) + fsMockEmitter.emit('unlink', path) try { deleteMock(path) return callback(undefined, null) @@ -302,6 +322,7 @@ fs.unlink = unlink export function mkdir(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.mkdir', path) + fsMockEmitter.emit('mkdir', path) try { setMock( path, @@ -324,6 +345,7 @@ fs.mkdir = mkdir export function readdir(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.readdir', path) + fsMockEmitter.emit('readdir', path) try { const mockFile = getMock(path) if (!mockFile.isDirectory) { @@ -340,6 +362,7 @@ fs.readdir = readdir export function lstat(path: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.lstat', path) + fsMockEmitter.emit('lstat', path) try { const mockFile = getMock(path) return callback(undefined, { @@ -355,6 +378,7 @@ fs.lstat = lstat export function writeFile(path: string, data: Buffer | string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.writeFile', path) + fsMockEmitter.emit('writeFile', path, data) try { setMock( path, @@ -373,9 +397,19 @@ export function writeFile(path: string, data: Buffer | string, callback: (error: } } fs.writeFile = writeFile -function readFile(path: string, callback: (error: any, result?: any) => void): void { +function readFile(path: string, ...args: any[]): void { path = fixPath(path) + + let callback: (error: any, result?: any) => void + if (args.length === 1) { + callback = args[0] + } else if (args.length === 2) { + // const options = args[0] + callback = args[1] + } else throw new Error(`Mock poorly implemented: ` + args) + if (DEBUG_LOG) console.log('fs.readFile', path) + fsMockEmitter.emit('readFile', path) try { const file = getMock(path) return callback(undefined, file.content) @@ -388,12 +422,13 @@ fs.readFile = readFile export function open(path: string, _options: string, callback: (error: any, result?: any) => void): void { path = fixPath(path) if (DEBUG_LOG) console.log('fs.open', path) + fsMockEmitter.emit('open', path) try { const file = getMock(path) - fd++ - openFileDescriptors[fd + ''] = file + fdId++ + openFileDescriptors[fdId + ''] = file - return callback(undefined, fd) + return callback(undefined, fdId) } catch (err) { callback(err) } @@ -401,6 +436,7 @@ export function open(path: string, _options: string, callback: (error: any, resu fs.open = open export function close(fd: number, callback: (error: any, result?: any) => void): void { if (DEBUG_LOG) console.log('fs.close') + fsMockEmitter.emit('close', fd) if (openFileDescriptors[fd + '']) { delete openFileDescriptors[fd + ''] return callback(undefined, null) @@ -413,6 +449,7 @@ export function copyFile(source: string, destination: string, callback: (error: source = fixPath(source) destination = fixPath(destination) if (DEBUG_LOG) console.log('fs.copyFile', source, destination) + fsMockEmitter.emit('copyFile', source, destination) try { const mockFile = getMock(source) if (DEBUG_LOG) console.log('source', source) @@ -429,6 +466,7 @@ export function rename(source: string, destination: string, callback: (error: an source = fixPath(source) destination = fixPath(destination) if (DEBUG_LOG) console.log('fs.rename', source, destination) + fsMockEmitter.emit('rename', source, destination) try { const mockFile = getMock(source) setMock(destination, mockFile, false) @@ -440,4 +478,9 @@ export function rename(source: string, destination: string, callback: (error: an } fs.rename = rename +interface FileAccess { + accessRead: boolean + accessWrite: boolean +} + module.exports = fs diff --git a/apps/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts b/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts similarity index 99% rename from apps/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts rename to tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts index 3a419263..7b24f27f 100644 --- a/apps/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts +++ b/tests/internal-tests/src/__mocks__/tv-automation-quantel-gateway-client.ts @@ -2,6 +2,8 @@ import EventEmitter from 'events' // eslint-disable-next-line node/no-unpublished-import import { Q, ClipSearchQuery } from 'tv-automation-quantel-gateway-client' // note: this is a mocked module +/* eslint-disable no-console */ + const client = jest.createMockFromModule('tv-automation-quantel-gateway-client') as any const DEBUG_LOG = false diff --git a/apps/tests/internal-tests/src/__mocks__/windows-network-drive.ts b/tests/internal-tests/src/__mocks__/windows-network-drive.ts similarity index 95% rename from apps/tests/internal-tests/src/__mocks__/windows-network-drive.ts rename to tests/internal-tests/src/__mocks__/windows-network-drive.ts index 4ceb41de..d6384d7d 100644 --- a/apps/tests/internal-tests/src/__mocks__/windows-network-drive.ts +++ b/tests/internal-tests/src/__mocks__/windows-network-drive.ts @@ -1,5 +1,7 @@ const wnd = jest.createMockFromModule('windows-network-drive') as any +/* eslint-disable no-console */ + const DEBUG_LOG = false const mountedDrives: { [key: string]: string } = {} diff --git a/tests/internal-tests/src/__tests__/basic.spec.ts b/tests/internal-tests/src/__tests__/basic.spec.ts new file mode 100644 index 00000000..179f305d --- /dev/null +++ b/tests/internal-tests/src/__tests__/basic.spec.ts @@ -0,0 +1,225 @@ +import fsOrg from 'fs' +import { promisify } from 'util' +import WNDOrg from 'windows-network-drive' +import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' +import * as QGatewayClientOrg from 'tv-automation-quantel-gateway-client' +import { Expectation, literal } from '@shared/api' +import type * as fsMockType from '../__mocks__/fs' +import type * as WNDType from '../__mocks__/windows-network-drive' +import type * as QGatewayClientType from '../__mocks__/tv-automation-quantel-gateway-client' +import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' +import { waitTime } from './lib/lib' +import { + getFileShareSource, + getLocalSource, + getLocalTarget, + getQuantelSource, + getQuantelTarget, +} from './lib/containers' +jest.mock('fs') +jest.mock('child_process') +jest.mock('windows-network-drive') +jest.mock('tv-automation-quantel-gateway-client') + +const fs = (fsOrg as any) as typeof fsMockType +const WND = (WNDOrg as any) as typeof WNDType +const QGatewayClient = (QGatewayClientOrg as any) as typeof QGatewayClientType + +const fsStat = promisify(fs.stat) + +describe('Basic', () => { + let env: TestEnviromnent + + beforeAll(async () => { + env = await prepareTestEnviromnent(false) // set to true to enable debug-logging + // Verify that the fs mock works: + expect(fs.lstat).toBeTruthy() + expect(fs.__mockReset).toBeTruthy() + }) + afterAll(() => { + env.terminate() + }) + beforeEach(() => { + fs.__mockReset() + env.reset() + QGatewayClient.resetMock() + }) + test('Be able to copy local file', async () => { + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target0') + // console.log(fs.__printAllFiles()) + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: [getLocalSource('source0', 'file0Source.mp4')], + }, + endRequirement: { + targets: [getLocalTarget('target0', 'file0Target.mp4')], + content: { + filePath: 'file0Target.mp4', + }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test('Be able to copy Networked file to local', async () => { + fs.__mockSetFile('\\\\networkShare/sources/source1/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target1') + // console.log(fs.__printAllFiles()) + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: [getFileShareSource('source1', 'file0Source.mp4')], + }, + endRequirement: { + targets: [getLocalTarget('target1', 'subFolder0/file0Target.mp4')], + content: { + filePath: 'subFolder0/file0Target.mp4', + }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + // console.log(fs.__printAllFiles()) + + expect(env.containerStatuses['target1']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + + expect(await WND.list()).toEqual({ + X: '\\\\networkShare\\sources\\source1\\', + }) + + expect(await fsStat('/targets/target1/subFolder0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test('Be able to copy Quantel clips', async () => { + const orgClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123')[0] + + env.expectationManager.updateExpectations({ + copy0: literal({ + id: 'copy0', + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.QUANTEL_CLIP_COPY, + statusReport: { + label: `Copy quantel clip0`, + description: `Copy clip0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: [getQuantelSource('source0')], + }, + endRequirement: { + targets: [getQuantelTarget('target1', 1001)], + content: { + guid: 'abc123', + }, + version: { type: Expectation.Version.Type.QUANTEL_CLIP }, + }, + workOptions: {}, + }), + }) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target1']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target1'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + + const newClip = QGatewayClient.searchClip((clip) => clip.ClipGUID === 'abc123' && clip !== orgClip.clip)[0] + expect(newClip).toBeTruthy() + + expect(newClip).toMatchObject({ + server: { + ident: 1001, + }, + clip: { + ClipGUID: 'abc123', + CloneId: orgClip.clip.ClipID, + }, + }) + }) + test.skip('Be able to copy local file to http', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Be able to handle 1000 expectations', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Media file preview from local to file share', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Media file preview from local to file share', async () => { + // To be written + expect(1).toEqual(1) + }) +}) + +export {} // Just to get rid of a "not a module" warning diff --git a/tests/internal-tests/src/__tests__/issues.spec.ts b/tests/internal-tests/src/__tests__/issues.spec.ts new file mode 100644 index 00000000..5e183933 --- /dev/null +++ b/tests/internal-tests/src/__tests__/issues.spec.ts @@ -0,0 +1,281 @@ +import fsOrg from 'fs' +import { promisify } from 'util' +import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' +import { Expectation, literal } from '@shared/api' +import type * as fsMockType from '../__mocks__/fs' +import { prepareTestEnviromnent, TestEnviromnent } from './lib/setupEnv' +import { waitTime } from './lib/lib' +import { getLocalSource, getLocalTarget } from './lib/containers' +import { WorkerAgent } from '@shared/worker' +jest.mock('fs') +jest.mock('child_process') +jest.mock('windows-network-drive') +jest.mock('tv-automation-quantel-gateway-client') + +const fs = (fsOrg as any) as typeof fsMockType + +const fsStat = promisify(fs.stat) + +// const fsStat = promisify(fs.stat) + +describe('Handle unhappy paths', () => { + let env: TestEnviromnent + + beforeAll(async () => { + env = await prepareTestEnviromnent(false) // set to true to enable debug-logging + // Verify that the fs mock works: + expect(fs.lstat).toBeTruthy() + expect(fs.__mockReset).toBeTruthy() + + jest.setTimeout(env.WAIT_JOB_TIME * 10 + env.WAIT_SCAN_TIME * 2) + }) + afterAll(() => { + env.terminate() + }) + + beforeEach(() => { + fs.__mockReset() + env.reset() + }) + + test('Wait for non-existing local file', async () => { + fs.__mockSetDirectory('/sources/source0/') + fs.__mockSetDirectory('/targets/target0') + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'waiting', + statusReason: { + tech: /not able to access file/i, + }, + }, + }) + + // Now the file suddenly pops up: + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + + await waitTime(env.WAIT_SCAN_TIME) + await waitTime(env.WAIT_JOB_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test.skip('Wait for non-existing network-shared, file', async () => { + // To be written + + // Handle Drive-letters: Issue when when files aren't found? (Johan knows) + + expect(1).toEqual(1) + }) + test.skip('Wait for growing file', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Wait for non-existing file', async () => { + // To be written + expect(1).toEqual(1) + }) + test('Wait for read access on source', async () => { + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234, { + accessRead: false, + accessWrite: false, + }) + fs.__mockSetDirectory('/targets/target0', { + accessRead: true, + accessWrite: false, + }) + // fs.__printAllFiles() + + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'waiting', + statusReason: { + tech: /not able to access file/i, + }, + }, + }) + + // Now the file can be read from: + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the Expectation to be waiting: + expect(env.expectationStatuses['copy0']).toMatchObject({ + actualVersionHash: null, + statusInfo: { + status: 'new', + statusReason: { + tech: /not able to access target/i, + }, + }, + }) + + // Now the target can be written to: + fs.__mockSetDirectory('/targets/target0', { + accessRead: true, + accessWrite: true, + }) + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the copy to have completed by now: + + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(await fsStat('/targets/target0/file0Target.mp4')).toMatchObject({ + size: 1234, + }) + }) + test.skip('Wait for write access on target', async () => { + // To be written + expect(1).toEqual(1) + }) + test.skip('Expectation changed', async () => { + // Expectation is changed while waiting for a file + // Expectation is changed while work-in-progress + // Expectation is changed after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test.skip('Aborting a job', async () => { + // Expectation is aborted while waiting for a file + // Expectation is aborted while work-in-progress + // Expectation is aborted after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test.skip('Restarting a job', async () => { + // Expectation is restarted while waiting for a file + // Expectation is restarted while work-in-progress + // Expectation is restarted after fullfilled + + // To be written + expect(1).toEqual(1) + }) + test('A worker crashes', async () => { + // A worker crashes while expectation waiting for a file + // A worker crashes while expectation work-in-progress + + expect(env.workerAgents).toHaveLength(1) + fs.__mockSetFile('/sources/source0/file0Source.mp4', 1234) + fs.__mockSetDirectory('/targets/target0') + let killedWorker: WorkerAgent | undefined + const listenToCopyFile = jest.fn(() => { + // While the copy is underway, kill off the worker: + // This simulates that the worker crashes, without telling anyone. + killedWorker = env.workerAgents[0] + killedWorker.terminate() + }) + + fs.__emitter().once('copyFile', listenToCopyFile) + + addCopyFileExpectation( + env, + 'copy0', + [getLocalSource('source0', 'file0Source.mp4')], + [getLocalTarget('target0', 'file0Target.mp4')] + ) + + await waitTime(env.WAIT_JOB_TIME) + // Expect the Expectation to be still working + // (the worker has crashed, byt expectationManger hasn't noticed yet) + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('working') + expect(listenToCopyFile).toHaveBeenCalledTimes(1) + + await waitTime(env.WORK_TIMEOUT_TIME) + await waitTime(env.WAIT_JOB_TIME) + // By now, the work should have been aborted, and restarted: + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual(expect.stringMatching(/new|waiting/)) + + // Add another worker: + env.addWorker() + await waitTime(env.WAIT_SCAN_TIME) + + // Expect the copy to have completed by now: + expect(env.expectationStatuses['copy0'].statusInfo.status).toEqual('fulfilled') + expect(env.containerStatuses['target0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0']).toBeTruthy() + expect(env.containerStatuses['target0'].packages['package0'].packageStatus?.status).toEqual( + ExpectedPackageStatusAPI.PackageContainerPackageStatusStatus.READY + ) + + // Clean up: + if (killedWorker) env.removeWorker(killedWorker.id) + }) + test.skip('One of the workers reply very slowly', async () => { + // The expectation should be picked up by one of the faster workers + + // To be written + expect(1).toEqual(1) + }) +}) +function addCopyFileExpectation( + env: TestEnviromnent, + expectationId: string, + sources: Expectation.SpecificPackageContainerOnPackage.File[], + targets: [Expectation.SpecificPackageContainerOnPackage.File] +) { + env.expectationManager.updateExpectations({ + copy0: literal({ + id: expectationId, + priority: 0, + managerId: 'manager0', + fromPackages: [{ id: 'package0', expectedContentVersionHash: 'abcd1234' }], + type: Expectation.Type.FILE_COPY, + statusReport: { + label: `Copy file0`, + description: `Copy file0 because test`, + requiredForPlayout: true, + displayRank: 0, + sendReport: true, + }, + startRequirement: { + sources: sources, + }, + endRequirement: { + targets: targets, + content: { + filePath: 'file0Target.mp4', + }, + version: { type: Expectation.Version.Type.FILE_ON_DISK }, + }, + workOptions: {}, + }), + }) +} + +export {} // Just to get rid of a "not a module" warning diff --git a/apps/tests/internal-tests/src/__tests__/lib/containers.ts b/tests/internal-tests/src/__tests__/lib/containers.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/containers.ts rename to tests/internal-tests/src/__tests__/lib/containers.ts diff --git a/apps/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts b/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts similarity index 100% rename from apps/tests/internal-tests/src/__tests__/lib/coreMockAPI.ts rename to tests/internal-tests/src/__tests__/lib/coreMockAPI.ts diff --git a/tests/internal-tests/src/__tests__/lib/lib.ts b/tests/internal-tests/src/__tests__/lib/lib.ts new file mode 100644 index 00000000..872de6c3 --- /dev/null +++ b/tests/internal-tests/src/__tests__/lib/lib.ts @@ -0,0 +1,3 @@ +export function waitTime(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts new file mode 100644 index 00000000..ff3a72e1 --- /dev/null +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -0,0 +1,291 @@ +// import * as HTTPServer from '@http-server/generic' +// import * as PackageManager from '@package-manager/generic' +import * as Workforce from '@shared/workforce' +import * as Worker from '@shared/worker' +import * as Winston from 'winston' +import { Expectation, ExpectationManagerWorkerAgent, LoggerInstance, Reason, SingleAppConfig } from '@shared/api' +// import deepExtend from 'deep-extend' +import { ExpectationManager, ExpectationManagerCallbacks, ExpectationManagerOptions } from '@shared/expectation-manager' +import { CoreMockAPI } from './coreMockAPI' +import { ExpectedPackageStatusAPI } from '@sofie-automation/blueprints-integration' + +const defaultTestConfig: SingleAppConfig = { + singleApp: { + workerCount: 1, + }, + process: { + logPath: '', + unsafeSSL: false, + certificates: [], + }, + workforce: { + port: null, + }, + httpServer: { + port: 0, + basePath: '', + apiKeyRead: '', + apiKeyWrite: '', + }, + packageManager: { + coreHost: '', + corePort: 0, + deviceId: '', + deviceToken: '', + disableWatchdog: true, + port: null, + accessUrl: null, + workforceURL: null, + watchFiles: false, + }, + worker: { + workerId: 'worker', + workforceURL: null, + appContainerURL: null, + resourceId: '', + networkIds: [], + windowsDriveLetters: ['X', 'Y', 'Z'], + sourcePackageStabilityThreshold: 0, // Disabling this to speed up the tests + }, + quantelHTTPTransformerProxy: { + port: 0, + transformerURL: '', + }, + appContainer: { + appContainerId: 'appContainer0', + workforceURL: null, + port: 0, + maxRunningApps: 1, + minRunningApps: 1, + spinDownTime: 0, + resourceId: '', + networkIds: [], + windowsDriveLetters: ['X', 'Y', 'Z'], + }, +} + +export async function setupExpectationManager( + debugLogging: boolean, + workerCount: number = 1, + callbacks: ExpectationManagerCallbacks, + options?: ExpectationManagerOptions +) { + const logger = new Winston.Logger({}) as LoggerInstance + logger.add(Winston.transports.Console, { + level: debugLogging ? 'debug' : 'warn', + }) + + const expectationManager = new ExpectationManager( + logger, + 'manager0', + { type: 'internal' }, + undefined, + { type: 'internal' }, + callbacks, + options + ) + + // Initializing HTTP proxy Server: + // const httpServer = new HTTPServer.PackageProxyServer(logger, config) + // await httpServer.init() + + // Initializing Workforce: + const workforce = new Workforce.Workforce(logger, defaultTestConfig) + await workforce.init() + + // Initializing Expectation Manager: + expectationManager.hookToWorkforce(workforce.getExpectationManagerHook()) + await expectationManager.init() + + // Initialize workers: + const workerAgents: Worker.WorkerAgent[] = [] + let workerI = 0 + const addWorker = async () => { + const workerId = defaultTestConfig.worker.workerId + '_' + workerI++ + const workerAgent = new Worker.WorkerAgent(logger, { + ...defaultTestConfig, + worker: { + ...defaultTestConfig.worker, + workerId: workerId, + }, + }) + workerAgents.push(workerAgent) + + workerAgent.hookToWorkforce(workforce.getWorkerAgentHook()) + workerAgent.hookToExpectationManager(expectationManager.managerId, expectationManager.getWorkerAgentHook()) + await workerAgent.init() + + return workerId + } + const removeWorker = async (workerId: string) => { + const index = workerAgents.findIndex((wa) => wa.id === workerId) + if (index !== -1) { + const workerAgent = workerAgents[index] + + expectationManager.removeWorkerAgentHook(workerAgent.id) + + workerAgent.terminate() + // Remove from array: + workerAgents.splice(index, 1) + } + } + + for (let i = 0; i < workerCount; i++) { + await addWorker() + } + + return { + workforce, + workerAgents, + expectationManager, + addWorker, + removeWorker, + } +} + +export async function prepareTestEnviromnent(debugLogging: boolean): Promise { + const expectationStatuses: ExpectationStatuses = {} + const containerStatuses: ContainerStatuses = {} + const coreApi = new CoreMockAPI() + + const WAIT_JOB_TIME = 500 // ms + const WAIT_SCAN_TIME = 1000 // ms + const WORK_TIMEOUT_TIME = 900 // ms + + const em = await setupExpectationManager( + debugLogging, + 1, + { + reportExpectationStatus: ( + expectationId: string, + _expectaction: Expectation.Any | null, + actualVersionHash: string | null, + statusInfo: { + status?: string + progress?: number + statusReason?: Reason + } + ) => { + if (debugLogging) console.log('reportExpectationStatus', expectationId, actualVersionHash, statusInfo) + + if (!expectationStatuses[expectationId]) { + expectationStatuses[expectationId] = { + actualVersionHash: null, + statusInfo: {}, + } + } + const o = expectationStatuses[expectationId] + if (actualVersionHash) o.actualVersionHash = actualVersionHash + if (statusInfo.status) o.statusInfo.status = statusInfo.status + if (statusInfo.progress) o.statusInfo.progress = statusInfo.progress + if (statusInfo.statusReason) o.statusInfo.statusReason = statusInfo.statusReason + }, + reportPackageContainerPackageStatus: ( + containerId: string, + packageId: string, + packageStatus: Omit | null + ) => { + if (debugLogging) + console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) + if (!containerStatuses[containerId]) { + containerStatuses[containerId] = { + packages: {}, + } + } + const container = containerStatuses[containerId] + container.packages[packageId] = { + packageStatus: packageStatus, + } + }, + reportPackageContainerExpectationStatus: () => { + // todo + // if (debugLogging) console.log('reportPackageContainerExpectationStatus', containerId, packageId, packageStatus) + }, + messageFromWorker: async (message: ExpectationManagerWorkerAgent.MessageFromWorkerPayload.Any) => { + switch (message.type) { + case 'fetchPackageInfoMetadata': + return coreApi.fetchPackageInfoMetadata(...message.arguments) + case 'updatePackageInfo': + return coreApi.updatePackageInfo(...message.arguments) + case 'removePackageInfo': + return coreApi.removePackageInfo(...message.arguments) + case 'reportFromMonitorPackages': + return coreApi.reportFromMonitorPackages(...message.arguments) + default: + // @ts-expect-error message.type is never + throw new Error(`Unsupported message type "${message.type}"`) + } + }, + }, + { + constants: { + EVALUATE_INTERVAL: WAIT_SCAN_TIME - WAIT_JOB_TIME - 300, + FULLFILLED_MONITOR_TIME: WAIT_SCAN_TIME - WAIT_JOB_TIME - 300, + WORK_TIMEOUT_TIME: WORK_TIMEOUT_TIME - 300, + }, + } + ) + + return { + WAIT_JOB_TIME, + WAIT_SCAN_TIME, + WORK_TIMEOUT_TIME, + expectationManager: em.expectationManager, + workerAgents: em.workerAgents, + workforce: em.workforce, + coreApi, + expectationStatuses, + containerStatuses, + reset: () => { + if (debugLogging) { + console.log('RESET ENVIRONMENT') + } + em.expectationManager.resetWork() + Object.keys(expectationStatuses).forEach((id) => delete expectationStatuses[id]) + Object.keys(containerStatuses).forEach((id) => delete containerStatuses[id]) + coreApi.reset() + }, + terminate: () => { + em.expectationManager.terminate() + em.workforce.terminate() + em.workerAgents.forEach((workerAgent) => workerAgent.terminate()) + }, + addWorker: em.addWorker, + removeWorker: em.removeWorker, + } +} +export interface TestEnviromnent { + WAIT_JOB_TIME: number + WAIT_SCAN_TIME: number + WORK_TIMEOUT_TIME: number + expectationManager: ExpectationManager + workerAgents: Worker.WorkerAgent[] + workforce: Workforce.Workforce + coreApi: CoreMockAPI + expectationStatuses: ExpectationStatuses + containerStatuses: ContainerStatuses + reset: () => void + terminate: () => void + addWorker: () => Promise + removeWorker: (id: string) => Promise +} + +export interface ExpectationStatuses { + [expectationId: string]: { + actualVersionHash: string | null + statusInfo: { + status?: string + progress?: number + statusReason?: Reason + } + } +} +export interface ContainerStatuses { + [containerId: string]: { + packages: { + [packageId: string]: { + packageStatus: Omit | null + } + } + } +} diff --git a/apps/tests/internal-tests/src/index.ts b/tests/internal-tests/src/index.ts similarity index 100% rename from apps/tests/internal-tests/src/index.ts rename to tests/internal-tests/src/index.ts diff --git a/tests/internal-tests/tsconfig.json b/tests/internal-tests/tsconfig.json new file mode 100644 index 00000000..47857232 --- /dev/null +++ b/tests/internal-tests/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + } +} diff --git a/yarn.lock b/yarn.lock index a7dba6ce..29f460b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1478,15 +1478,13 @@ tslib "^2.0.3" underscore "1.12.0" -"@sofie-automation/blueprints-integration@1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0": - version "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0.tgz#6e9279af9dac23e88df9e76bb4b41cb9cc04c446" - integrity sha512-4SsMbHcEzUVk24G7maJQXg2i9F8rdfGn/rV0Pn95Gih4OlJxefne4T5WKS6kOY5A3C8ZGDNRolHR/WtrbcOHLA== +"@sofie-automation/blueprints-integration@1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0": + version "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/blueprints-integration/-/blueprints-integration-1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0.tgz#5b4a71e36514107a8c69ffee7e9425aab360b460" + integrity sha512-QbPOaYbkoca3UCq3UYOcYrUjjKL/sDpTH3CcLMKCCBxVgsfzLAdsRkzbfOmrgbsx1LS4s4Yv9c+YK68AKzcrvw== dependencies: - moment "2.29.1" - timeline-state-resolver-types "5.9.0-nightly-release34-20210511-085833-30ad952a1.0" + timeline-state-resolver-types "6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0" tslib "^2.1.0" - underscore "1.13.1" "@sofie-automation/code-standard-preset@^0.2.1", "@sofie-automation/code-standard-preset@^0.2.2": version "0.2.2" @@ -1507,14 +1505,14 @@ read-pkg-up "^7.0.1" shelljs "^0.8.4" -"@sofie-automation/server-core-integration@1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0": - version "1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0" - resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.35.0-nightly-feat-package-management-20210521-082825-31727c2.0.tgz#dc115837aa303bb1c7741140c1a6de5c453f32b5" - integrity sha512-yy3wTxfLef+821BVshwozVBJEwOIZhUpQqFJDcQ9AVZDUzJEl97c+q++V/FvJ2SPbT/5FqFwVp3uzlnVUIWzIA== +"@sofie-automation/server-core-integration@1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0": + version "1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0" + resolved "https://registry.yarnpkg.com/@sofie-automation/server-core-integration/-/server-core-integration-1.36.0-nightly-feat-package-management-20210630-070054-22b29d9.0.tgz#51f5cdfe38d50fc58862800eef5f9a370f7171cb" + integrity sha512-Xd3gnz/LxZ3ZlekEPovKnGM4yQL1/+xUBIB67cfMdHBcej9tjMTNbeLxgOA7ZnemjVVEeiikwmMJvwNUiLc+JA== dependencies: - data-store "3.1.0" + data-store "^4.0.3" ejson "^2.2.0" - faye-websocket "^0.11.3" + faye-websocket "^0.11.4" got "^11.8.2" tslib "^2.0.3" underscore "^1.12.1" @@ -3361,10 +3359,13 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -data-store@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/data-store/-/data-store-3.1.0.tgz#9464f2c8ac8cad5cd0ebb6992eaf354d7ea8c35c" - integrity sha512-MjbLiqz5IJFD/NZLztrrxc2LZ8KMc42kHWAUSxD/kp2ekzHE8EZfkYP4nQy15aPMwV5vac2dW21Ni72okNAwTQ== +data-store@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/data-store/-/data-store-4.0.3.tgz#d055c55f3fa776daa3a5664cf715d3fe689c3afb" + integrity sha512-AhPSqGVCSrFgi1PtkOLxYRrVhzbsma/drtxNkwTrT2q+LpLTG3AhOW74O/IMRy1cWftBx93SuLLKF19YsKkUMg== + dependencies: + get-value "^3.0.1" + set-value "^3.0.1" data-urls@^2.0.0: version "2.0.0" @@ -4287,10 +4288,10 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -faye-websocket@^0.11.3: - version "0.11.3" - resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.3.tgz#5c0e9a8968e8912c286639fde977a8b209f2508e" - integrity sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA== +faye-websocket@^0.11.4: + version "0.11.4" + resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.4.tgz#7f0d9275cfdd86a1c963dc8b65fcc451edcbb1da" + integrity sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g== dependencies: websocket-driver ">=0.5.1" @@ -4736,6 +4737,13 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= +get-value@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-3.0.1.tgz#5efd2a157f1d6a516d7524e124ac52d0a39ef5a8" + integrity sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA== + dependencies: + isobject "^3.0.1" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -8864,6 +8872,13 @@ set-value@^2.0.0, set-value@^2.0.1: is-plain-object "^2.0.3" split-string "^3.0.1" +set-value@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-3.0.2.tgz#74e8ecd023c33d0f77199d415409a40f21e61b90" + integrity sha512-npjkVoz+ank0zjlV9F47Fdbjfj/PfXyVhZvGALWsyIYU/qrMzpi6avjKW3/7KeSU2Df3I46BrN1xOI1+6vW0hA== + dependencies: + is-plain-object "^2.0.4" + setprototypeof@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" @@ -9648,12 +9663,12 @@ timeline-state-resolver-types@5.5.1: dependencies: tslib "^1.13.0" -timeline-state-resolver-types@5.9.0-nightly-release34-20210511-085833-30ad952a1.0: - version "5.9.0-nightly-release34-20210511-085833-30ad952a1.0" - resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-5.9.0-nightly-release34-20210511-085833-30ad952a1.0.tgz#6edada5ea4199b290645c9ced2a4051ed68985ea" - integrity sha512-3M9CptINGuerTEJGriuez4PqFqyD3Cf2ZZ9lI0cv4mnXOTLt3kwbp4a0uc2HWLBgzPsYPM9JEz2Y20XFSJC4TQ== +timeline-state-resolver-types@6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0: + version "6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0" + resolved "https://registry.yarnpkg.com/timeline-state-resolver-types/-/timeline-state-resolver-types-6.0.0-nightly-release35-20210608-165921-e8c0d0dbb.0.tgz#c0cbf83b95204cc1e4e10b176f120a41d1abca72" + integrity sha512-SqOfKD5orC4HcIX/pVEwfdL+B6LEsP+47McAqljUS7Ha64WC6OxSQfA9TL8I2TEHt07rj85ZbtkXZOBhxO+lFw== dependencies: - tslib "^1.13.0" + tslib "^2.1.0" tmp@^0.0.33: version "0.0.33" @@ -9961,11 +9976,6 @@ underscore@1.12.0, underscore@^1.12.0: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.0.tgz#4814940551fc80587cef7840d1ebb0f16453be97" integrity sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ== -underscore@1.13.1: - version "1.13.1" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.1.tgz#0c1c6bd2df54b6b69f2314066d65b6cde6fcf9d1" - integrity sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g== - underscore@^1.12.1: version "1.12.1" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.1.tgz#7bb8cc9b3d397e201cf8553336d262544ead829e"