[ts-sdk] Make react sdk run on browser

ts/@live-compositor/browser-render/src/index.ts

@@ -1,2 +1,3 @@
 export { loadWasmModule } from './wasm';
 export * from './renderer';
+export * from './api';

ts/@live-compositor/browser-render/src/renderer.ts

@@ -8,11 +8,6 @@ export type RendererOptions = {
   streamFallbackTimeoutMs: number;
 };
 
-export type Framerate = {
-  num: number;
-  den: number;
-};
-
 export type FrameSet = {
   ptsMs: number;
   frames: { [id: string]: Frame };

ts/@live-compositor/core/src/api.ts

@@ -1,5 +1,7 @@
 import { Api } from 'live-compositor';
 import { CompositorManager } from './compositorManager';
+import { RegisterOutputRequest } from './api/output';
+import { RegisterInputRequest } from './api/input';
 
 export { Api };
 
@@ -24,7 +26,7 @@ export class ApiClient {
     });
   }
 
-  public async registerOutput(outptuId: string, request: Api.RegisterOutput): Promise<object> {
+  public async registerOutput(outptuId: string, request: RegisterOutputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/output/${encodeURIComponent(outptuId)}/register`,
@@ -40,7 +42,7 @@ export class ApiClient {
     });
   }
 
-  public async registerInput(inputId: string, request: Api.RegisterInput): Promise<object> {
+  public async registerInput(inputId: string, request: RegisterInputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/input/${encodeURIComponent(inputId)}/register`,

ts/@live-compositor/core/src/api/input.ts

@@ -1,17 +1,21 @@
 import { Api } from '../api';
 import { RegisterInput, Inputs } from 'live-compositor';
 
-export function intoRegisterInput(input: RegisterInput): Api.RegisterInput {
+export type RegisterInputRequest = Api.RegisterInput | { type: 'bytes' };
+
+export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
     return intoRtpRegisterInput(input);
+  } else if (input.type === 'bytes') {
+    return intoBytesRegisterInput();
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }
 }
 
-function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput {
+function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): RegisterInputRequest {
   return {
     type: 'mp4',
     url: input.url,
@@ -22,7 +26,7 @@ function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput
   };
 }
 
-function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): Api.RegisterInput {
+function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequest {
   return {
     type: 'rtp_stream',
     port: input.port,
@@ -34,6 +38,10 @@ function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): Api.RegisterInput
   };
 }
 
+function intoBytesRegisterInput(): RegisterInputRequest {
+  return { type: 'bytes' };
+}
+
 function intoInputAudio(audio: Inputs.InputRtpAudioOptions): Api.InputRtpAudioOptions {
   if (audio.decoder === 'opus') {
     return {

ts/@live-compositor/core/src/api/output.ts

@@ -1,22 +1,37 @@
-import { RegisterOutput, Api, Outputs } from 'live-compositor';
+import { RegisterOutput, Api, Outputs, OutputByteFormat } from 'live-compositor';
+
+export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput;
+
+export type RegisterBytesOutput = {
+  type: 'bytes';
+  video?: OutputBytesVideoOptions;
+};
+
+export type OutputBytesVideoOptions = {
+  format: OutputByteFormat;
+  resolution: Api.Resolution;
+  initial: Api.Video;
+};
 
 export function intoRegisterOutput(
   output: RegisterOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   if (output.type === 'rtp_stream') {
     return intoRegisterRtpOutput(output, initial);
   } else if (output.type === 'mp4') {
     return intoRegisterMp4Output(output, initial);
+  } else if (output.type === 'bytes') {
+    return intoRegisterBytesOutput(output, initial);
   } else {
-    throw new Error(`Unknown input type ${(output as any).type}`);
+    throw new Error(`Unknown output type ${(output as any).type}`);
   }
 }
 
 function intoRegisterRtpOutput(
   output: Outputs.RegisterRtpOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'rtp_stream',
     port: output.port,
@@ -30,7 +45,7 @@ function intoRegisterRtpOutput(
 function intoRegisterMp4Output(
   output: Outputs.RegisterMp4Output,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'mp4',
     path: output.serverPath,
@@ -39,6 +54,20 @@ function intoRegisterMp4Output(
   };
 }
 
+function intoRegisterBytesOutput(
+  output: Outputs.RegisterBytesOutput,
+  initial: { video?: Api.Video; _audio?: Api.Audio }
+): RegisterOutputRequest {
+  return {
+    type: 'bytes',
+    video: {
+      format: output.video.format,
+      resolution: output.video.resolution,
+      initial: initial.video!,
+    },
+  };
+}
+
 function intoOutputVideoOptions(
   video: Outputs.RtpVideoOptions | Outputs.Mp4VideoOptions,
   initial: Api.Video

ts/@live-compositor/core/src/index.ts

@@ -1,3 +1,5 @@
 export { ApiClient, ApiRequest } from './api';
 export { LiveCompositor } from './compositor';
 export { CompositorManager } from './compositorManager';
+export { RegisterInputRequest } from './api/input';
+export { RegisterOutputRequest, RegisterBytesOutput } from './api/output';

ts/@live-compositor/core/src/output.ts

@@ -33,12 +33,13 @@ class Output {
       this.shouldUpdateWhenReady = true;
     };
 
-    if (registerRequest.audio) {
-      this.initialAudioConfig = registerRequest.audio.initial ?? { inputs: [] };
+    const hasAudio = 'audio' in registerRequest && !!registerRequest.audio;
+    if (hasAudio) {
+      this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] };
     }
 
     const onUpdate = () => this.throttledUpdate();
-    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, !!registerRequest.audio);
+    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, hasAudio);
 
     if (registerRequest.video) {
       const rootElement = React.createElement(OutputRootComponent, {

ts/@live-compositor/core/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/node/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/web/.eslintignore

@@ -0,0 +1 @@
+dist

ts/@live-compositor/web/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "../../.eslintrc.base.json"
+  ]
+}

ts/@live-compositor/web/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@live-compositor/web",
+  "version": "0.1.0-rc.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "watch": "tsc --watch --preserveWatchOutput",
+    "build": "tsc",
+    "clean": "rimraf dist",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "author": "",
+  "license": "MIT",
+  "files": [
+    "/dist"
+  ],
+  "dependencies": {
+    "@live-compositor/browser-render": "0.1.0-rc.4",
+    "@live-compositor/core": "0.1.0-rc.5",
+    "live-compositor": "^0.1.0-rc.5",
+    "mp4box": "^0.5.2"
+  }
+}

ts/@live-compositor/web/src/compositor.ts

@@ -0,0 +1,89 @@
+import { Renderer } from '@live-compositor/browser-render';
+import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
+import WasmInstance from './manager/wasmInstance';
+import { Queue, StopQueueFn } from './queue';
+import { EventSender } from './eventSender';
+import { intoRegisterOutput, RegisterOutput } from './output/registerOutput';
+import { Output } from './output/output';
+import { intoRegisterInput, RegisterInput } from './input/registerInput';
+import { Input } from './input/input';
+import { RegisterImage } from './renderers';
+
+export type LiveCompositorOptions = {
+  framerate: Framerate;
+  streamFallbackTimeoutMs: number;
+};
+
+export type Framerate = {
+  num: number;
+  den: number;
+};
+
+export default class LiveCompositor {
+  private coreCompositor: CoreLiveCompositor;
+  private queue: Queue;
+  private renderer: Renderer;
+  private eventSender?: EventSender;
+
+  private constructor(renderer: Renderer, framerate: Framerate) {
+    this.coreCompositor = new CoreLiveCompositor(
+      new WasmInstance({
+        renderer: renderer,
+        onRegisterCallback: cb => {
+          this.eventSender = new EventSender(cb);
+        },
+      })
+    );
+    this.queue = new Queue(framerate, renderer);
+    this.renderer = renderer;
+  }
+
+  public static async create(options: LiveCompositorOptions): Promise<LiveCompositor> {
+    const renderer = await Renderer.create({
+      streamFallbackTimeoutMs: options.streamFallbackTimeoutMs,
+    });
+    const compositor = new LiveCompositor(renderer, options.framerate);
+    await compositor.coreCompositor.init();
+    return compositor;
+  }
+
+  public async registerOutput(outputId: string, request: RegisterOutput): Promise<void> {
+    await this.coreCompositor.registerOutput(outputId, intoRegisterOutput(request));
+    const output = Output.create(request);
+    this.queue.addOutput(outputId, output);
+  }
+
+  public async unregisterOutput(outputId: string): Promise<void> {
+    await this.coreCompositor.unregisterOutput(outputId);
+    this.queue.removeOutput(outputId);
+  }
+
+  public async registerInput(inputId: string, request: RegisterInput): Promise<void> {
+    await this.coreCompositor.registerInput(inputId, intoRegisterInput(request));
+
+    const input = Input.create(inputId, request, this.eventSender!);
+    this.queue.addInput(inputId, input);
+    input.start();
+  }
+
+  public async unregisterInput(inputId: string): Promise<void> {
+    await this.coreCompositor.unregisterInput(inputId);
+    this.queue.removeInput(inputId);
+  }
+
+  public async registerImage(imageId: string, request: RegisterImage): Promise<void> {
+    await this.coreCompositor.registerImage(imageId, request);
+  }
+
+  public async unregisterImage(imageId: string): Promise<void> {
+    await this.coreCompositor.unregisterImage(imageId);
+  }
+
+  public async registerFont(fontUrl: string): Promise<void> {
+    await this.renderer.registerFont(fontUrl);
+  }
+
+  public start(): StopQueueFn {
+    return this.queue.start();
+  }
+}

ts/@live-compositor/web/src/eventSender.ts

@@ -0,0 +1,24 @@
+import { InputId } from '@live-compositor/browser-render';
+import { CompositorEventType } from 'live-compositor';
+
+export class EventSender {
+  private eventCallback: (event: object) => void;
+
+  public constructor(eventCallback: (event: object) => void) {
+    this.eventCallback = eventCallback;
+  }
+
+  public sendEvent(event: ApiEvent) {
+    this.eventCallback(event);
+  }
+}
+
+export type ApiEvent =
+  | {
+      type: CompositorEventType.VIDEO_INPUT_DELIVERED;
+      input_id: InputId;
+    }
+  | {
+      type: CompositorEventType.VIDEO_INPUT_PLAYING;
+      input_id: InputId;
+    };