Autoregister images (#909)

software-mansion · Jan 9, 2025 · 4178579 · 4178579
1 parent 47e6061
commit 4178579
Show file tree

Hide file tree

Showing 26 changed files with 317 additions and 93 deletions.
diff --git a/ts/@live-compositor/core/src/api.ts b/ts/@live-compositor/core/src/api.ts
@@ -2,6 +2,7 @@ import { Api } from 'live-compositor';
 import type { CompositorManager } from './compositorManager.js';
 import type { RegisterOutputRequest } from './api/output.js';
 import { inputRefIntoRawId, type InputRef, type RegisterInputRequest } from './api/input.js';
+import { imageRefIntoRawId, type ImageRef } from './api/image.js';
 
 export { Api };
 
@@ -31,21 +32,21 @@ export class ApiClient {
     });
   }
 
-  public async registerOutput(outptuId: string, request: RegisterOutputRequest): Promise<object> {
+  public async registerOutput(outputId: string, request: RegisterOutputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
-      route: `/api/output/${encodeURIComponent(outptuId)}/register`,
+      route: `/api/output/${encodeURIComponent(outputId)}/register`,
       body: request,
     });
   }
 
   public async unregisterOutput(
-    outptuId: string,
+    outputId: string,
     body: { schedule_time_ms?: number }
   ): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
-      route: `/api/output/${encodeURIComponent(outptuId)}/unregister`,
+      route: `/api/output/${encodeURIComponent(outputId)}/unregister`,
       body,
     });
   }
@@ -88,18 +89,18 @@ export class ApiClient {
     });
   }
 
-  public async registerImage(imageId: string, request: Api.ImageSpec): Promise<object> {
+  public async registerImage(imageRef: ImageRef, request: Api.ImageSpec): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
-      route: `/api/image/${encodeURIComponent(imageId)}/register`,
+      route: `/api/image/${encodeURIComponent(imageRefIntoRawId(imageRef))}/register`,
       body: request,
     });
   }
 
-  public async unregisterImage(imageId: string): Promise<object> {
+  public async unregisterImage(imageRef: ImageRef): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
-      route: `/api/image/${encodeURIComponent(imageId)}/unregister`,
+      route: `/api/image/${encodeURIComponent(imageRefIntoRawId(imageRef))}/unregister`,
       body: {},
     });
   }

diff --git a/ts/@live-compositor/core/src/api/image.ts b/ts/@live-compositor/core/src/api/image.ts
@@ -0,0 +1,8 @@
+import type { Api } from '../api.js';
+import { _liveCompositorInternals } from 'live-compositor';
+
+export type RegisterInputRequest = Api.RegisterInput;
+
+export type ImageRef = _liveCompositorInternals.ImageRef;
+export const imageRefIntoRawId = _liveCompositorInternals.imageRefIntoRawId;
+export const parseImageRef = _liveCompositorInternals.parseImageRef;
diff --git a/ts/@live-compositor/core/src/live/compositor.ts b/ts/@live-compositor/core/src/live/compositor.ts
@@ -12,6 +12,7 @@ import { intoRegisterImage, intoRegisterWebRenderer } from '../api/renderer.js';
 import { handleEvent } from './event.js';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
+import type { ImageRef } from '../api/image.js';
 
 export class LiveCompositor {
   private manager: CompositorManager;
@@ -109,12 +110,16 @@ export class LiveCompositor {
 
   public async registerImage(imageId: string, request: Renderers.RegisterImage): Promise<object> {
     this.logger.info({ imageId }, 'Register image');
-    return this.api.registerImage(imageId, intoRegisterImage(request));
+    const imageRef = { type: 'global', id: imageId } as const satisfies ImageRef;
+
+    return this.api.registerImage(imageRef, intoRegisterImage(request));
   }
 
   public async unregisterImage(imageId: string): Promise<object> {
     this.logger.info({ imageId }, 'Unregister image');
-    return this.api.unregisterImage(imageId);
+    const imageRef = { type: 'global', id: imageId } as const satisfies ImageRef;
+
+    return this.api.unregisterImage(imageRef);
   }
 
   public async registerWebRenderer(

diff --git a/ts/@live-compositor/core/src/live/event.ts b/ts/@live-compositor/core/src/live/event.ts
@@ -16,7 +16,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'ready' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'ready' },
@@ -28,7 +28,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'playing' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'playing' },
@@ -40,7 +40,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'finished' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, videoState: 'finished' },
@@ -52,7 +52,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'ready' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'ready' },
@@ -64,7 +64,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'playing' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'playing' },
@@ -76,7 +76,7 @@ export function handleEvent(
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'finished' },
       });
-    } else if (event.inputRef.type === 'output-local') {
+    } else if (event.inputRef.type === 'output-specific-input') {
       outputs[event.inputRef.outputId]?.inputStreamStore().dispatchUpdate({
         type: 'update_input',
         input: { inputId: event.inputRef.id, audioState: 'finished' },

diff --git a/ts/@live-compositor/core/src/live/output.ts b/ts/@live-compositor/core/src/live/output.ts
@@ -9,6 +9,7 @@ import { intoAudioInputsConfiguration } from '../api/output.js';
 import { ThrottledFunction } from '../utils.js';
 import { OutputRootComponent } from '../rootComponent.js';
 import type { Logger } from 'pino';
+import type { ImageRef } from '../api/image.js';
 
 type AudioContext = _liveCompositorInternals.AudioContext;
 type LiveTimeContext = _liveCompositorInternals.LiveTimeContext;
@@ -143,7 +144,7 @@ class OutputContext implements CompositorOutputContext {
   ): Promise<{ videoDurationMs?: number; audioDurationMs?: number }> {
     return await this.output.internalInputStreamStore.runBlocking(async updateStore => {
       const inputRef = {
-        type: 'output-local',
+        type: 'output-specific-input',
         outputId: this.outputId,
         id: inputId,
       } as const;
@@ -171,13 +172,32 @@ class OutputContext implements CompositorOutputContext {
   public async unregisterMp4Input(inputId: number): Promise<void> {
     await this.output.api.unregisterInput(
       {
-        type: 'output-local',
+        type: 'output-specific-input',
         outputId: this.outputId,
         id: inputId,
       },
       {}
     );
   }
+  public async registerImage(imageId: number, imageSpec: any) {
+    const imageRef = {
+      type: 'output-specific-image',
+      outputId: this.outputId,
+      id: imageId,
+    } as const satisfies ImageRef;
+
+    await this.output.api.registerImage(imageRef, {
+      url: imageSpec.url,
+      asset_type: imageSpec.assetType,
+    });
+  }
+  public async unregisterImage(imageId: number) {
+    await this.output.api.unregisterImage({
+      type: 'output-specific-image',
+      outputId: this.outputId,
+      id: imageId,
+    });
+  }
 }
 
 export default Output;
diff --git a/ts/@live-compositor/core/src/offline/compositor.ts b/ts/@live-compositor/core/src/offline/compositor.ts
@@ -11,6 +11,7 @@ import OfflineOutput from './output.js';
 import { CompositorEventType, parseEvent } from '../event.js';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
+import type { ImageRef } from '../api/image.js';
 
 /**
  * Offline rendering only supports one output, so we can just pick any value to use
@@ -126,7 +127,9 @@ export class OfflineCompositor {
   public async registerImage(imageId: string, request: Renderers.RegisterImage): Promise<object> {
     this.checkNotStarted();
     this.logger.info({ imageId }, 'Register image');
-    return this.api.registerImage(imageId, intoRegisterImage(request));
+    const imageRef = { type: 'global', id: imageId } as const satisfies ImageRef;
+
+    return this.api.registerImage(imageRef, intoRegisterImage(request));
   }
 
   private checkNotStarted() {

diff --git a/ts/@live-compositor/core/src/offline/output.ts b/ts/@live-compositor/core/src/offline/output.ts
@@ -10,6 +10,7 @@ import { sleep } from '../utils.js';
 import { OFFLINE_OUTPUT_ID } from './compositor.js';
 import { OutputRootComponent } from '../rootComponent.js';
 import type { Logger } from 'pino';
+import type { ImageRef } from '../api/image.js';
 
 type AudioContext = _liveCompositorInternals.AudioContext;
 type OfflineTimeContext = _liveCompositorInternals.OfflineTimeContext;
@@ -146,7 +147,7 @@ class OutputContext implements CompositorOutputContext {
     registerRequest: RegisterMp4Input
   ): Promise<{ videoDurationMs?: number; audioDurationMs?: number }> {
     const inputRef = {
-      type: 'output-local',
+      type: 'output-specific-input',
       outputId: this.outputId,
       id: inputId,
     } as const;
@@ -184,15 +185,33 @@ class OutputContext implements CompositorOutputContext {
   public async unregisterMp4Input(inputId: number): Promise<void> {
     await this.output.api.unregisterInput(
       {
-        type: 'output-local',
+        type: 'output-specific-input',
         outputId: this.outputId,
         id: inputId,
       },
       { schedule_time_ms: this.timeContext.timestampMs() }
     );
   }
-}
+  public async registerImage(imageId: number, imageSpec: any) {
+    const imageRef = {
+      type: 'output-specific-image',
+      outputId: this.outputId,
+      id: imageId,
+    } as const satisfies ImageRef;
 
+    await this.output.api.registerImage(imageRef, {
+      url: imageSpec.url,
+      asset_type: imageSpec.assetType,
+    });
+  }
+  public async unregisterImage(imageId: number) {
+    await this.output.api.unregisterImage({
+      type: 'output-specific-image',
+      outputId: this.outputId,
+      id: imageId,
+    });
+  }
+}
 async function waitForBlockingTasks(offlineContext: OfflineTimeContext): Promise<void> {
   while (offlineContext.isBlocked()) {
     await sleep(100);

diff --git a/ts/eslint.config.js b/ts/eslint.config.js
@@ -56,6 +56,7 @@ export default [
       '@typescript-eslint/no-floating-promises': ['error'],
       'no-constant-condition': [0],
       'no-unused-vars': 'off',
+      'curly': 'error',
       '@typescript-eslint/no-unused-vars': [
         'error',
         {

diff --git a/ts/examples/node-examples/src/dynamic-text.tsx b/ts/examples/node-examples/src/dynamic-text.tsx
@@ -1,5 +1,5 @@
 import LiveCompositor from '@live-compositor/node';
-import { View, Text } from 'live-compositor';
+import { View, Text, Image } from 'live-compositor';
 import { useEffect, useState } from 'react';
 import { ffplayStartPlayerAsync, sleep } from './utils';
 
@@ -47,6 +47,7 @@ function ExampleApp() {
       <PartialText text="Example partial text that transition in 1 second" transitionMs={1_000} />
       <PartialText text="Example partial text that transition in 2 second" transitionMs={2_000} />
       <PartialText text="Example partial text that transition in 5 second" transitionMs={5_000} />
+      <Image imageId="image_1" />
     </View>
   );
 }
@@ -58,6 +59,12 @@ async function run() {
   await ffplayStartPlayerAsync('127.0.0.1', 8001);
   await sleep(2000);
 
+  await compositor.registerImage('image_1', {
+    assetType: 'svg',
+    url: 'https://compositor.live/img/logo.svg',
+    resolution: { width: 300, height: 300 },
+  });
+
   await compositor.registerOutput('output_1', <ExampleApp />, {
     type: 'rtp_stream',
     port: 8001,

diff --git a/ts/examples/vite-browser-render/src/examples/mp4/decoder.ts b/ts/examples/vite-browser-render/src/examples/mp4/decoder.ts
@@ -58,7 +58,9 @@ export class MP4Decoder {
     console.log(`Using codec: ${videoTrack.codec}`);
 
     const trak = this.file.getTrackById(videoTrack.id);
-    if (!trak) return;
+    if (!trak) {
+      return;
+    }
     const description = getCodecDescription(trak);
     if (!description) {
       console.error('Codec description not found');