From 166b275c360e793d9523567974a9dba3c43187d0 Mon Sep 17 00:00:00 2001 From: Wojciech Kozyra Date: Thu, 21 Nov 2024 17:34:06 +0100 Subject: [PATCH] wip --- .../core/src/compositorManager.ts | 9 +- ts/@live-compositor/core/src/index.ts | 3 +- .../core/src/offline_compositor.ts | 82 ++++++++++ .../core/src/offline_output.ts | 148 ++++++++++++++++++ ts/@live-compositor/node/src/index.ts | 11 +- .../node/src/manager/existingInstance.ts | 6 +- .../src/manager/locallySpawnedInstance.ts | 9 +- ts/examples/node-examples/src/combine_mp4.tsx | 17 +- 8 files changed, 270 insertions(+), 15 deletions(-) create mode 100644 ts/@live-compositor/core/src/offline_compositor.ts create mode 100644 ts/@live-compositor/core/src/offline_output.ts diff --git a/ts/@live-compositor/core/src/compositorManager.ts b/ts/@live-compositor/core/src/compositorManager.ts index ebdcdca73..e45aa5a06 100644 --- a/ts/@live-compositor/core/src/compositorManager.ts +++ b/ts/@live-compositor/core/src/compositorManager.ts @@ -1,7 +1,14 @@ import type { ApiRequest } from './api.js'; +export interface SetupInstanceOptions { + /** + * sets LIVE_COMPOSITOR_AHEAD_OF_TIME_PROCESSING_ENABLE environment variable. + */ + aheadOfTimeProcessing: boolean; +} + export interface CompositorManager { - setupInstance(): Promise; + setupInstance(opts: SetupInstanceOptions): Promise; sendRequest(request: ApiRequest): Promise; registerEventListener(cb: (event: unknown) => void): void; } diff --git a/ts/@live-compositor/core/src/index.ts b/ts/@live-compositor/core/src/index.ts index fb0b860af..f257c5ed5 100644 --- a/ts/@live-compositor/core/src/index.ts +++ b/ts/@live-compositor/core/src/index.ts @@ -1,5 +1,6 @@ export { ApiClient, ApiRequest } from './api.js'; export { LiveCompositor } from './compositor.js'; -export { CompositorManager } from './compositorManager.js'; +export { OfflineCompositor } from './offline_compositor.js'; +export { CompositorManager, SetupInstanceOptions } from './compositorManager.js'; export { RegisterInputRequest, RegisterInput } from './api/input.js'; export { RegisterOutputRequest, RegisterOutput } from './api/output.js'; diff --git a/ts/@live-compositor/core/src/offline_compositor.ts b/ts/@live-compositor/core/src/offline_compositor.ts new file mode 100644 index 000000000..ea63bfdc4 --- /dev/null +++ b/ts/@live-compositor/core/src/offline_compositor.ts @@ -0,0 +1,82 @@ +import type { Renderers } from 'live-compositor'; +import { _liveCompositorInternals } from 'live-compositor'; +import { ApiClient } from './api.js'; +import type { CompositorManager } from './compositorManager.js'; +import type { RegisterOutput } from './api/output.js'; +import { intoRegisterOutput } from './api/output.js'; +import type { RegisterInput } from './api/input.js'; +import { intoRegisterInput } from './api/input.js'; +import { onCompositorEvent } from './event.js'; +import { intoRegisterImage, intoRegisterWebRenderer } from './api/renderer.js'; +import OfflineOutput from './offline_output.js'; + +/** + * Offline rendering only supports one output, so we can just pick any value to use + * as an output ID. + */ +const OFFLINE_OUTPUT_ID = 'offline_output'; + +export type OfflineRenderOptions = { + output: RegisterOutput; + durationMs: number; +}; + +export class OfflineCompositor { + private manager: CompositorManager; + private api: ApiClient; + private store: _liveCompositorInternals.InstanceContextStore; + private outputs: Record = {}; + private render_started: boolean = false; + + public constructor(manager: CompositorManager) { + this.manager = manager; + this.api = new ApiClient(this.manager); + this.store = new _liveCompositorInternals.InstanceContextStore(); + } + + public async init(): Promise { + this.manager.registerEventListener((event: unknown) => onCompositorEvent(this.store, event)); + await this.manager.setupInstance({ aheadOfTimeProcessing: true }); + } + + public async render(request: RegisterOutput): Promise { + this.render_started = true; + const output = new OfflineOutput(OFFLINE_OUTPUT_ID, request, this.api, this.store); + + const apiRequest = intoRegisterOutput(request, output.scene()); + const result = await this.api.registerOutput(OFFLINE_OUTPUT_ID, apiRequest); + this.outputs[OFFLINE_OUTPUT_ID] = output; + await output.ready(); + // loop next_timestamp() + // - update props/store + // - check if finished rendering + // - + return result; + } + + public async registerInput(inputId: string, request: RegisterInput): Promise { + return this.store.runBlocking(async updateStore => { + const result = await this.api.registerInput(inputId, intoRegisterInput(request)); + updateStore({ type: 'add_input', input: { inputId } }); + return result; + }); + } + + public async registerShader( + shaderId: string, + request: Renderers.RegisterShader + ): Promise { + return this.api.registerShader(shaderId, request); + } + + public async registerImage(imageId: string, request: Renderers.RegisterImage): Promise { + return this.api.registerImage(imageId, intoRegisterImage(request)); + } + + public async registerWebRenderer( + instanceId: string, + request: Renderers.RegisterWebRenderer + ): Promise { + return this.api.registerWebRenderer(instanceId, intoRegisterWebRenderer(request)); + } +} diff --git a/ts/@live-compositor/core/src/offline_output.ts b/ts/@live-compositor/core/src/offline_output.ts new file mode 100644 index 000000000..a71ef809f --- /dev/null +++ b/ts/@live-compositor/core/src/offline_output.ts @@ -0,0 +1,148 @@ +import type { Outputs } from 'live-compositor'; +import { _liveCompositorInternals, View } from 'live-compositor'; +import type React from 'react'; +import { createElement, useSyncExternalStore } from 'react'; +import type { ApiClient, Api } from './api.js'; +import Renderer from './renderer.js'; +import type { RegisterOutput } from './api/output.js'; +import { intoAudioInputsConfiguration } from './api/output.js'; +import { throttle } from './utils.js'; + +type OutputContext = _liveCompositorInternals.OutputContext; +type InstanceContextStore = _liveCompositorInternals.InstanceContextStore; + +class OfflineOutput { + api: ApiClient; + outputId: string; + outputCtx: OutputContext; + outputShutdownStateStore: OutputShutdownStateStore; + + shouldUpdateWhenReady: boolean = false; + throttledUpdate: () => void; + videoRenderer?: Renderer; + initialAudioConfig?: Outputs.AudioInputsConfiguration; + + constructor( + outputId: string, + registerRequest: RegisterOutput, + api: ApiClient, + store: InstanceContextStore + ) { + this.api = api; + this.outputId = outputId; + this.outputShutdownStateStore = new OutputShutdownStateStore(); + this.shouldUpdateWhenReady = false; + this.throttledUpdate = () => { + this.shouldUpdateWhenReady = true; + }; + + const supportsAudio = 'audio' in registerRequest && !!registerRequest.audio; + if (supportsAudio) { + this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] }; + } + + const onUpdate = () => this.throttledUpdate(); + this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, { + supportsAudio, + offlineMode: true, + }); + + if (registerRequest.video) { + const rootElement = createElement(OutputRootComponent, { + instanceStore: store, + outputCtx: this.outputCtx, + outputRoot: registerRequest.video.root, + outputShutdownStateStore: this.outputShutdownStateStore, + }); + + this.videoRenderer = new Renderer({ + rootElement, + onUpdate, + idPrefix: `${outputId}-`, + }); + } + } + + public scene(): { video?: Api.Video; audio?: Api.Audio } { + const audio = this.outputCtx.audioContext.getAudioConfig() ?? this.initialAudioConfig; + return { + video: this.videoRenderer && { root: this.videoRenderer.scene() }, + audio: audio && intoAudioInputsConfiguration(audio), + }; + } + + public close(): void { + this.throttledUpdate = () => {}; + // close will switch a scene to just a , so we need replace `throttledUpdate` + // callback before it is called + this.outputShutdownStateStore.close(); + } + + public async ready() { + this.throttledUpdate = throttle(async () => { + await this.api.updateScene(this.outputId, this.scene()); + }, 30); + if (this.shouldUpdateWhenReady) { + this.throttledUpdate(); + } + } +} + +// External store to share shutdown information between React tree +// and external code that is managing it. +class OutputShutdownStateStore { + private shutdown: boolean = false; + private onChangeCallbacks: Set<() => void> = new Set(); + + public close() { + this.shutdown = true; + this.onChangeCallbacks.forEach(cb => cb()); + } + + // callback for useSyncExternalStore + public getSnapshot = (): boolean => { + return this.shutdown; + }; + + // callback for useSyncExternalStore + public subscribe = (onStoreChange: () => void): (() => void) => { + this.onChangeCallbacks.add(onStoreChange); + return () => { + this.onChangeCallbacks.delete(onStoreChange); + }; + }; +} + +function OutputRootComponent({ + outputRoot, + instanceStore, + outputCtx, + outputShutdownStateStore, +}: { + outputRoot: React.ReactElement; + instanceStore: InstanceContextStore; + outputCtx: OutputContext; + outputShutdownStateStore: OutputShutdownStateStore; +}) { + const shouldShutdown = useSyncExternalStore( + outputShutdownStateStore.subscribe, + outputShutdownStateStore.getSnapshot + ); + + if (shouldShutdown) { + // replace root with view to stop all the dynamic code + return createElement(View, {}); + } + + const reactCtx = { + instanceStore, + outputCtx, + }; + return createElement( + _liveCompositorInternals.LiveCompositorContext.Provider, + { value: reactCtx }, + outputRoot + ); +} + +export default OfflineOutput; diff --git a/ts/@live-compositor/node/src/index.ts b/ts/@live-compositor/node/src/index.ts index 60614db7f..38eb2f064 100644 --- a/ts/@live-compositor/node/src/index.ts +++ b/ts/@live-compositor/node/src/index.ts @@ -1,5 +1,8 @@ import type { CompositorManager } from '@live-compositor/core'; -import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core'; +import { + LiveCompositor as CoreLiveCompositor, + OfflineCompositor as CoreOfflineCompositor, +} from '@live-compositor/core'; import LocallySpawnedInstance from './manager/locallySpawnedInstance'; import ExistingInstance from './manager/existingInstance'; @@ -10,3 +13,9 @@ export default class LiveCompositor extends CoreLiveCompositor { super(manager ?? LocallySpawnedInstance.defaultManager()); } } + +export class OfflineCompositor extends CoreOfflineCompositor { + constructor(manager?: CompositorManager) { + super(manager ?? LocallySpawnedInstance.defaultManager()); + } +} diff --git a/ts/@live-compositor/node/src/manager/existingInstance.ts b/ts/@live-compositor/node/src/manager/existingInstance.ts index 3ea881de4..97aa3fb50 100644 --- a/ts/@live-compositor/node/src/manager/existingInstance.ts +++ b/ts/@live-compositor/node/src/manager/existingInstance.ts @@ -1,4 +1,4 @@ -import type { ApiRequest, CompositorManager } from '@live-compositor/core'; +import type { ApiRequest, CompositorManager, SetupInstanceOptions } from '@live-compositor/core'; import { sendRequest } from '../fetch'; import { retry, sleep } from '../utils'; @@ -28,7 +28,9 @@ class ExistingInstance implements CompositorManager { this.wsConnection = new WebSocketConnection(`${wsProtocol}://${this.ip}:${this.port}/ws`); } - public async setupInstance(): Promise { + public async setupInstance(_opts: SetupInstanceOptions): Promise { + // TODO: verify if options match + // https://github.com/software-mansion/live-compositor/issues/877 await retry(async () => { await sleep(500); return await this.sendRequest({ diff --git a/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts b/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts index 2d97f06e7..74f6806d0 100644 --- a/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts +++ b/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts @@ -4,7 +4,7 @@ import path from 'path'; import { v4 as uuidv4 } from 'uuid'; import * as fs from 'fs-extra'; import * as tar from 'tar'; -import type { ApiRequest, CompositorManager } from '@live-compositor/core'; +import type { ApiRequest, CompositorManager, SetupInstanceOptions } from '@live-compositor/core'; import { download, sendRequest } from '../fetch'; import { retry, sleep } from '../utils'; @@ -48,18 +48,21 @@ class LocallySpawnedInstance implements CompositorManager { }); } - public async setupInstance(): Promise { + public async setupInstance(opts: SetupInstanceOptions): Promise { const executablePath = this.executablePath ?? (await prepareExecutable(this.enableWebRenderer)); spawn(executablePath, [], { env: { - ...process.env, LIVE_COMPOSITOR_DOWNLOAD_DIR: path.join(this.workingdir, 'download'), LIVE_COMPOSITOR_API_PORT: this.port.toString(), LIVE_COMPOSITOR_WEB_RENDERER_ENABLE: this.enableWebRenderer ? 'true' : 'false', // silence scene updates logging LIVE_COMPOSITOR_LOGGER_LEVEL: 'info,wgpu_hal=warn,wgpu_core=warn,compositor_pipeline::pipeline=warn', + LIVE_COMPOSITOR_AHEAD_OF_TIME_PROCESSING_ENABLE: opts.aheadOfTimeProcessing + ? 'true' + : 'false', + ...process.env, }, }).catch(err => { console.error('LiveCompositor instance failed', err); diff --git a/ts/examples/node-examples/src/combine_mp4.tsx b/ts/examples/node-examples/src/combine_mp4.tsx index 1a304ff6d..4ceb52b3d 100644 --- a/ts/examples/node-examples/src/combine_mp4.tsx +++ b/ts/examples/node-examples/src/combine_mp4.tsx @@ -1,4 +1,4 @@ -import LiveCompositor from '@live-compositor/node'; +import LiveCompositor, { OfflineCompositor } from '@live-compositor/node'; import { View, Text, useInputStreams, Tiles, Rescaler, InputStream } from 'live-compositor'; import { downloadAllAssets, ffplayStartPlayerAsync, sleep } from './utils'; import path from 'path'; @@ -227,7 +227,7 @@ function ExampleAppWithBlockingUseEffect() { // sets the value early enough return ( - {text} + {text ?? ''} ); } @@ -245,8 +245,8 @@ function InputTile({ inputId }: { inputId: string }) { - - + + Input ID: {inputId} @@ -256,23 +256,27 @@ function InputTile({ inputId }: { inputId: string }) { async function run() { await downloadAllAssets(); - const compositor = new LiveCompositor(); + const compositor = new OfflineCompositor(); await compositor.init(); await compositor.registerInput('input_1', { type: 'mp4', serverPath: path.join(__dirname, '../.assets/BigBuckBunny.mp4'), + offsetMs: 0, + required: true, }); await compositor.registerInput('input_2', { type: 'mp4', serverPath: path.join(__dirname, '../.assets/ElephantsDream.mp4'), + offsetMs: 0, + required: true, }); void ffplayStartPlayerAsync('127.0.0.1', 8001); await sleep(2000); - await compositor.registerOutput('output_1', { + await compositor.render({ type: 'mp4', serverPath: path.join(__dirname, '../.assets/combing_mp4_output.mp4'), video: { @@ -287,6 +291,5 @@ async function run() { root: , }, }); - await compositor.start(); } void run();