[ts-sdk] Make react sdk run on browser

Clean up

Fix `package-lock.json`

ts/@live-compositor/browser-render/src/index.ts

@@ -1,2 +1,3 @@
 export { loadWasmModule } from './wasm';
 export * from './renderer';
+export * from './api';

ts/@live-compositor/core/src/api.ts

@@ -1,5 +1,6 @@
 import { Api } from 'live-compositor';
 import { CompositorManager } from './compositorManager';
+import { RegisterOutputRequest } from './api/output';
 
 export { Api };
 
@@ -24,7 +25,7 @@ export class ApiClient {
     });
   }
 
-  public async registerOutput(outptuId: string, request: Api.RegisterOutput): Promise<object> {
+  public async registerOutput(outptuId: string, request: RegisterOutputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/output/${encodeURIComponent(outptuId)}/register`,

ts/@live-compositor/core/src/api/output.ts

@@ -1,22 +1,36 @@
-import { RegisterOutput, Api, Outputs } from 'live-compositor';
+import { RegisterOutput, Api, Outputs, OutputBytesFrameFormat } from 'live-compositor';
+
+export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput;
+
+export type RegisterBytesOutput = {
+  type: 'bytes';
+  video?: {
+    format: OutputBytesFrameFormat;
+    resolution: Api.Resolution;
+    initial: Api.Video;
+  };
+};
 
 export function intoRegisterOutput(
   output: RegisterOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
-  if (output.type === 'rtp_stream') {
-    return intoRegisterRtpOutput(output, initial);
-  } else if (output.type === 'mp4') {
-    return intoRegisterMp4Output(output, initial);
-  } else {
-    throw new Error(`Unknown input type ${(output as any).type}`);
+): RegisterOutputRequest {
+  switch (output.type) {
+    case 'rtp_stream':
+      return intoRegisterRtpOutput(output, initial);
+    case 'mp4':
+      return intoRegisterMp4Output(output, initial);
+    case 'bytes':
+      return intoRegisterBytesOutput(output, initial);
+    default:
+      throw new Error(`Unknown output type ${(output as any).type}`);
   }
 }
 
 function intoRegisterRtpOutput(
   output: Outputs.RegisterRtpOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'rtp_stream',
     port: output.port,
@@ -30,7 +44,7 @@ function intoRegisterRtpOutput(
 function intoRegisterMp4Output(
   output: Outputs.RegisterMp4Output,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'mp4',
     path: output.serverPath,
@@ -39,6 +53,20 @@ function intoRegisterMp4Output(
   };
 }
 
+function intoRegisterBytesOutput(
+  output: Outputs.RegisterBytesOutput,
+  initial: { video?: Api.Video; _audio?: Api.Audio }
+): RegisterOutputRequest {
+  return {
+    type: 'bytes',
+    video: {
+      format: output.video.format,
+      resolution: output.video.resolution,
+      initial: initial.video!,
+    },
+  };
+}
+
 function intoOutputVideoOptions(
   video: Outputs.OutputRtpVideoOptions | Outputs.OutputMp4VideoOptions,
   initial: Api.Video

ts/@live-compositor/core/src/index.ts

@@ -1,3 +1,4 @@
 export { ApiClient, ApiRequest } from './api';
 export { LiveCompositor, createLiveCompositor } from './compositor';
 export { CompositorManager } from './compositorManager';
+export { RegisterOutputRequest, RegisterBytesOutput } from './api/output';

ts/@live-compositor/core/src/output.ts

@@ -33,12 +33,13 @@ class Output {
       this.shouldUpdateWhenReady = true;
     };
 
-    if (registerRequest.audio) {
-      this.initialAudioConfig = registerRequest.audio.initial ?? { inputs: [] };
+    const hasAudio = 'audio' in registerRequest && !!registerRequest.audio;
+    if (hasAudio) {
+      this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] };
     }
 
     const onUpdate = () => this.throttledUpdate();
-    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, !!registerRequest.audio);
+    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, hasAudio);
 
     if (registerRequest.video) {
       const rootElement = React.createElement(OutputRootComponent, {

ts/@live-compositor/core/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/node/package.json

@@ -25,6 +25,7 @@
   },
   "dependencies": {
     "@live-compositor/core": "0.1.0-rc.0",
+    "@live-compositor/browser-render": "file:../browser-render",
     "fs-extra": "^11.2.0",
     "node-fetch": "^2.6.7",
     "tar": "^7.4.3",

ts/@live-compositor/node/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/web/.eslintignore

@@ -0,0 +1 @@
+dist

ts/@live-compositor/web/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "../../.eslintrc.base.json"
+  ]
+}

ts/@live-compositor/web/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@live-compositor/web",
+  "version": "0.1.0-rc.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "watch": "tsc --watch --preserveWatchOutput",
+    "build": "tsc",
+    "clean": "rimraf dist",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "author": "",
+  "license": "MIT",
+  "files": [
+    "/dist"
+  ],
+  "dependencies": {
+    "@live-compositor/browser-render": "file:../browser-render",
+    "@live-compositor/core": "0.1.0-rc.0",
+    "live-compositor": "^0.1.0-rc.0"
+  }
+}

ts/@live-compositor/web/src/index.ts

@@ -0,0 +1,63 @@
+import { LiveCompositor as CoreLiveCompositor, CompositorManager } from '@live-compositor/core';
+import WasmInstance from './manager/wasmInstance';
+import { Api, RegisterInput, RegisterOutput } from 'live-compositor';
+
+export { WasmInstance };
+
+export default class LiveCompositor extends CoreLiveCompositor {
+  private constructor(manager: CompositorManager) {
+    super(manager);
+  }
+
+  public static async create(manager: CompositorManager): Promise<LiveCompositor> {
+    const compositor = new LiveCompositor(manager);
+    await compositor['setupInstance']();
+    return compositor;
+  }
+
+  public override async registerOutput(outputId: string, request: RegisterOutput): Promise<object> {
+    if (request.type !== 'bytes') {
+      throw `Output type "${request.type}" is unsupported on web`;
+    }
+
+    return super.registerOutput(outputId, request);
+  }
+
+  public override async registerInput(inputId: string, request: RegisterInput): Promise<object> {
+    if (request.type !== 'bytes') {
+      throw `Input type "${request.type}" is unsupported on web`;
+    }
+
+    return super.registerInput(inputId, request);
+  }
+
+  public override async registerImage(imageId: string, request: Api.ImageSpec): Promise<object> {
+    if (request.path) {
+      throw "Image's `path` field is not supported on web";
+    }
+
+    return super.registerImage(imageId, request);
+  }
+
+  public override async registerShader(
+    _shaderId: string,
+    _request: Api.ShaderSpec
+  ): Promise<object> {
+    throw 'Shaders are unsupported';
+  }
+
+  public override async unregisterShader(_shaderId: string): Promise<object> {
+    throw 'Shaders are unsupported';
+  }
+
+  public override async registerWebRenderer(
+    _instanceId: string,
+    _request: Api.WebRendererSpec
+  ): Promise<object> {
+    throw 'Web renderers are unsupported';
+  }
+
+  public override async unregisterWebRenderer(_instanceId: string): Promise<object> {
+    throw 'Web renderers are unsupported';
+  }
+}

ts/@live-compositor/web/src/manager/wasmInstance.ts

@@ -0,0 +1,118 @@
+import { ApiRequest, CompositorManager, RegisterOutputRequest } from '@live-compositor/core';
+import { Renderer, Resolution, Component, ImageSpec } from '@live-compositor/browser-render';
+import { Api } from 'live-compositor';
+
+type Output = {
+  resolution: Resolution;
+};
+
+class WasmInstance implements CompositorManager {
+  private renderer: Renderer;
+  private outputs: Map<string, Output>;
+
+  public constructor(renderer: Renderer) {
+    this.renderer = renderer;
+    this.outputs = new Map();
+  }
+
+  public async setupInstance(): Promise<void> {}
+
+  public async sendRequest(request: ApiRequest): Promise<object> {
+    const paths = request.route.split('/');
+    if (paths.length < 3) {
+      return {};
+    }
+
+    switch (paths[2]) {
+      case 'input':
+        this.handleInputRequest(paths[3], paths[4]);
+        break;
+      case 'output':
+        this.handleOutputRequest(paths[3], paths[4], request.body);
+        break;
+      case 'image':
+        await this.handleImageRequest(paths[3], paths[4], request.body);
+        break;
+      case 'start':
+        // TODO(noituri):  Run queue
+        break;
+      case 'shader':
+        throw 'Shaders are not supported';
+      case 'web-renderer':
+        throw 'Web renderers are not supported';
+      default:
+        throw 'Unknown request: ' + paths[2];
+    }
+    return {};
+  }
+
+  public registerEventListener(_cb: (event: unknown) => void): void {}
+
+  private handleInputRequest(inputId: string, operation: string): void {
+    switch (operation) {
+      case 'register':
+        this.renderer.registerInput(inputId);
+        break;
+      case 'unregister':
+        this.renderer.unregisterInput(inputId);
+        break;
+      default:
+        throw 'Unknown operation: ' + operation;
+    }
+  }
+
+  private handleOutputRequest(outputId: string, operation: string, body?: object): void {
+    switch (operation) {
+      case 'register': {
+        const outputInfo = body! as RegisterOutputRequest;
+        if (outputInfo.video) {
+          const resolution = outputInfo.video.resolution;
+          this.outputs.set(outputId, { resolution: resolution });
+          this.renderer.updateScene(
+            outputId,
+            resolution,
+            outputInfo.video?.initial.root as Component
+          );
+        }
+        break;
+      }
+      case 'unregister': {
+        this.renderer.unregisterOutput(outputId);
+        break;
+      }
+      case 'update': {
+        const scene = body! as Api.UpdateOutputRequest;
+        if (!scene.video) {
+          return;
+        }
+        const output = this.outputs.get(outputId);
+        if (!output) {
+          throw `Unknown output "${outputId}"`;
+        }
+        this.renderer.updateScene(outputId, output.resolution, scene.video.root as Component);
+        break;
+      }
+      case 'request_keyframe':
+        // Do nothing
+        break;
+      default:
+        throw 'Unknown operation: ' + operation;
+    }
+  }
+
+  private async handleImageRequest(
+    imageId: string,
+    operation: string,
+    body?: object
+  ): Promise<void> {
+    switch (operation) {
+      case 'register':
+        await this.renderer.registerImage(imageId, body as ImageSpec);
+        break;
+      case 'unregister':
+        this.renderer.unregisterImage(imageId);
+    }
+  }
+}
+
+export default WasmInstance;

ts/@live-compositor/web/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
+  }
+}

ts/examples/vite-browser-render/.gitignore

@@ -11,6 +11,7 @@ node_modules
 dist
 dist-ssr
 *.local
+*.tsbuildinfo
 
 # Editor directories and files
 .vscode/*