[ts-sdk] Add hook for audio

software-mansion · Aug 30, 2024 · 3d5a601 · 3d5a601
1 parent f34caae
commit 3d5a601
Show file tree

Hide file tree

Showing 16 changed files with 411 additions and 106 deletions.
diff --git a/ts/@live-compositor/core/src/api/output.ts b/ts/@live-compositor/core/src/api/output.ts
@@ -2,40 +2,40 @@ import { RegisterOutput, Api, Outputs } from 'live-compositor';
 
 export function intoRegisterOutput(
   output: RegisterOutput,
-  initialVideo?: Api.Video
+  initial: { video?: Api.Video; audio?: Api.Audio }
 ): Api.RegisterOutput {
   if (output.type === 'rtp_stream') {
-    return intoRegisterRtpOutput(output, initialVideo);
+    return intoRegisterRtpOutput(output, initial);
   } else if (output.type === 'mp4') {
-    return intoRegisterMp4Output(output, initialVideo);
+    return intoRegisterMp4Output(output, initial);
   } else {
     throw new Error(`Unknown input type ${(output as any).type}`);
   }
 }
 
 function intoRegisterRtpOutput(
   output: Outputs.RegisterRtpOutput,
-  initialVideo?: Api.Video
+  initial: { video?: Api.Video; audio?: Api.Audio }
 ): Api.RegisterOutput {
   return {
     type: 'rtp_stream',
     port: output.port,
     ip: output.ip,
     transport_protocol: output.transportProtocol,
-    video: output.video && initialVideo && intoOutputVideoOptions(output.video, initialVideo),
-    audio: output.audio && intoOutputRtpAudioOptions(output.audio),
+    video: output.video && initial.video && intoOutputVideoOptions(output.video, initial.video),
+    audio: output.audio && initial.audio && intoOutputRtpAudioOptions(output.audio, initial.audio),
   };
 }
 
 function intoRegisterMp4Output(
   output: Outputs.RegisterMp4Output,
-  initialVideo?: Api.Video
+  initial: { video?: Api.Video; audio?: Api.Audio }
 ): Api.RegisterOutput {
   return {
     type: 'mp4',
     path: output.serverPath,
-    video: output.video && initialVideo && intoOutputVideoOptions(output.video, initialVideo),
-    audio: output.audio && intoOutputMp4AudioOptions(output.audio),
+    video: output.video && initial.video && intoOutputVideoOptions(output.video, initial.video),
+    audio: output.audio && initial.audio && intoOutputMp4AudioOptions(output.audio, initial.audio),
   };
 }
 
@@ -62,22 +62,24 @@ function intoVideoEncoderOptions(
 }
 
 function intoOutputRtpAudioOptions(
-  audio: Outputs.OutputRtpAudioOptions
+  audio: Outputs.OutputRtpAudioOptions,
+  initial: Api.Audio
 ): Api.OutputRtpAudioOptions {
   return {
     send_eos_when: audio.sendEosWhen && intoOutputEosCondition(audio.sendEosWhen),
     encoder: intoRtpAudioEncoderOptions(audio.encoder),
-    initial: intoAudioInputsConfiguration(audio.initial),
+    initial,
   };
 }
 
 function intoOutputMp4AudioOptions(
-  audio: Outputs.OutputMp4AudioOptions
+  audio: Outputs.OutputMp4AudioOptions,
+  initial: Api.Audio
 ): Api.OutputMp4AudioOptions {
   return {
     send_eos_when: audio.sendEosWhen && intoOutputEosCondition(audio.sendEosWhen),
     encoder: intoMp4AudioEncoderOptions(audio.encoder),
-    initial: intoAudioInputsConfiguration(audio.initial),
+    initial,
   };
 }
 

diff --git a/ts/@live-compositor/core/src/compositor.ts b/ts/@live-compositor/core/src/compositor.ts
@@ -1,4 +1,4 @@
-import { ContextStore, RegisterInput, RegisterOutput } from 'live-compositor';
+import { HookContext, RegisterInput, RegisterOutput } from 'live-compositor';
 import { ApiClient, Api } from './api';
 import Output from './output';
 import { CompositorManager } from './compositorManager';
@@ -15,12 +15,13 @@ export async function createLiveCompositor(manager: CompositorManager): Promise<
 export class LiveCompositor {
   private manager: CompositorManager;
   private api: ApiClient;
-  private store: ContextStore = new ContextStore();
+  private store: HookContext.InstanceContextStore;
   private outputs: Record<string, Output> = {};
 
   public constructor(manager: CompositorManager) {
     this.manager = manager;
     this.api = new ApiClient(this.manager);
+    this.store = new HookContext.InstanceContextStore();
   }
 
   private async setupInstance() {
@@ -30,14 +31,19 @@ export class LiveCompositor {
 
   public async registerOutput(outputId: string, request: RegisterOutput): Promise<object> {
     const output = new Output(outputId, request, this.api, this.store);
-    const initialVideo = output.scene();
 
-    const apiRequest = intoRegisterOutput(request, initialVideo);
+    const apiRequest = intoRegisterOutput(request, output.scene());
     const result = await this.api.registerOutput(outputId, apiRequest);
     this.outputs[outputId] = output;
     return result;
   }
 
+  public async unregisterOutput(outputId: string): Promise<object> {
+    this.outputs[outputId].close();
+    delete this.outputs[outputId];
+    return this.api.unregisterOutput(outputId);
+  }
+
   public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
     const result = await this.api.registerInput(inputId, intoRegisterInput(request));
     this.store.addInput({ inputId });

diff --git a/ts/@live-compositor/core/src/event.ts b/ts/@live-compositor/core/src/event.ts
@@ -1,6 +1,6 @@
-import { CompositorEvent, CompositorEventType, ContextStore } from 'live-compositor';
+import { CompositorEvent, CompositorEventType, HookContext } from 'live-compositor';
 
-export function onCompositorEvent(store: ContextStore, rawEvent: unknown) {
+export function onCompositorEvent(store: HookContext.InstanceContextStore, rawEvent: unknown) {
   const event = parseEvent(rawEvent);
   if (!event) {
     return;

diff --git a/ts/@live-compositor/core/src/output.ts b/ts/@live-compositor/core/src/output.ts
@@ -1,4 +1,4 @@
-import { ContextStore, LiveCompositorContext, RegisterOutput } from 'live-compositor';
+import { HookContext, RegisterOutput } from 'live-compositor';
 import { ApiClient, Api } from './api';
 import Renderer from './renderer';
 import { intoAudioInputsConfiguration } from './api/output';
@@ -7,43 +7,60 @@ import React from 'react';
 class Output {
   api: ApiClient;
   outputId: string;
+  outputCtx: HookContext.OutputContext;
   initialized: boolean = false;
 
   throttledUpdate: () => void;
   videoRenderer?: Renderer;
 
-  constructor(outputId: string, output: RegisterOutput, api: ApiClient, store: ContextStore) {
+  constructor(
+    outputId: string,
+    output: RegisterOutput,
+    api: ApiClient,
+    store: HookContext.InstanceContextStore
+  ) {
     this.api = api;
     this.outputId = outputId;
 
-    let audioOptions: Api.Audio | undefined;
+    const onUpdate = () => this.throttledUpdate?.();
+    this.outputCtx = new HookContext.OutputContext(onUpdate, output.audio?.initial);
+
     if (output.video) {
-      this.videoRenderer = new Renderer(
-        React.createElement(LiveCompositorContext.Provider, { value: store }, output.video.root),
-        () => this.onRendererUpdate(),
-        `${outputId}-`
+      const reactCtx = {
+        instanceStore: store,
+        outputCtx: this.outputCtx,
+      };
+      const rootElement = React.createElement(
+        HookContext.LiveCompositorContext.Provider,
+        { value: reactCtx },
+        output.video.root
       );
+
+      this.videoRenderer = new Renderer({
+        rootElement,
+        onUpdate,
+        idPrefix: `${outputId}-`,
+      });
     }
-    if (output.audio) {
-      audioOptions = intoAudioInputsConfiguration(output.audio.initial);
-    }
+
     this.throttledUpdate = throttle(async () => {
-      await api.updateScene(this.outputId, {
-        video: this.videoRenderer && { root: this.videoRenderer.scene() },
-        audio: audioOptions,
-      });
+      await api.updateScene(this.outputId, this.scene());
     }, 30);
   }
 
-  public scene(): Api.Video | undefined {
-    return this.videoRenderer && { root: this.videoRenderer.scene() };
+  public scene(): { video?: Api.Video; audio?: Api.Audio } {
+    const audio = this.outputCtx.getAudioConfig();
+    return {
+      video: this.videoRenderer && { root: this.videoRenderer.scene() },
+      audio: audio && intoAudioInputsConfiguration(audio),
+    };
   }
 
-  private onRendererUpdate() {
-    if (!this.throttledUpdate || !this.videoRenderer) {
-      return;
+  public close(): void {
+    if (this.videoRenderer) {
+      delete this.videoRenderer;
     }
-    this.throttledUpdate();
+    this.throttledUpdate = () => {};
   }
 }
 

diff --git a/ts/@live-compositor/core/src/renderer.ts b/ts/@live-compositor/core/src/renderer.ts
@@ -210,11 +210,17 @@ const HostConfig: Reconciler.HostConfig<
 
 const CompositorRenderer = Reconciler(HostConfig);
 
+type RendererOptions = {
+  rootElement: React.ReactElement;
+  onUpdate: () => void;
+  idPrefix: string;
+};
+
 class Renderer {
   root: any;
   onUpdateFn: () => void;
 
-  constructor(element: React.ReactElement, onUpdate: () => void, idPrefix: string) {
+  constructor({ rootElement: element, onUpdate, idPrefix }: RendererOptions) {
     const root = CompositorRenderer.createContainer(
       this, // container tag
       LegacyRoot,

diff --git a/ts/examples/src/audio.tsx b/ts/examples/src/audio.tsx
@@ -0,0 +1,95 @@
+import LiveCompositor from '@live-compositor/node';
+import { Text, InputStream, Tiles, Rescaler, View } from 'live-compositor';
+import { downloadAllAssets, gstReceiveTcpStream, sleep } from './utils';
+import path from 'path';
+import { useState, useEffect } from 'react';
+
+function ExampleApp() {
+  const [streamWithAudio, setStream] = useState('input_1');
+  useEffect(() => {
+    const timeout = setTimeout(() => {
+      setStream(streamWithAudio === 'input_1' ? 'input_2' : 'input_1');
+    }, 5000);
+    return () => clearTimeout(timeout);
+  }, [streamWithAudio]);
+
+  return (
+    <Tiles transition={{ durationMs: 200 }}>
+      <InputTile inputId="input_1" mute={streamWithAudio === 'input_1'} />
+      <InputTile inputId="input_2" mute={streamWithAudio === 'input_2'} />
+    </Tiles>
+  );
+}
+
+function InputTile({ inputId, mute }: { inputId: string; mute: boolean }) {
+  const [volume, setVolume] = useState(1.0);
+
+  useEffect(() => {
+    const timeout = setTimeout(() => {
+      if (volume < 0.2) {
+        setVolume(1.0);
+      } else {
+        setVolume(volume - 0.1);
+      }
+    }, 1000);
+    return () => clearTimeout(timeout);
+  }, [volume]);
+
+  return (
+    <View>
+      <Rescaler>
+        <InputStream inputId={inputId} volume={volume} mute={mute} />
+      </Rescaler>
+      <View bottom={10} left={10} height={40}>
+        <Text fontSize={40}>
+          Input ID: {inputId}, volume: {volume.toFixed(2)} {mute ? 'muted' : 'live'}
+        </Text>
+      </View>
+    </View>
+  );
+}
+
+async function run() {
+  await downloadAllAssets();
+  const compositor = await LiveCompositor.create();
+
+  await sleep(2000);
+
+  await compositor.registerOutput('output_1', {
+    type: 'rtp_stream',
+    port: 8001,
+    transportProtocol: 'tcp_server',
+    video: {
+      encoder: {
+        type: 'ffmpeg_h264',
+        preset: 'ultrafast',
+      },
+      resolution: {
+        width: 1920,
+        height: 1080,
+      },
+      root: <ExampleApp />,
+    },
+    audio: {
+      encoder: {
+        type: 'opus',
+        channels: 'stereo',
+      },
+      initial: { inputs: [] },
+    },
+  });
+  gstReceiveTcpStream('127.0.0.1', 8001);
+
+  await compositor.registerInput('input_1', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.assets/BigBuckBunny.mp4'),
+  });
+
+  await compositor.registerInput('input_2', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.assets/ElephantsDream.mp4'),
+  });
+
+  await compositor.start();
+}
+run();
diff --git a/ts/examples/src/dynamic-inputs.tsx b/ts/examples/src/dynamic-inputs.tsx
@@ -6,12 +6,22 @@ import path from 'path';
 function ExampleApp() {
   const inputs = useInputStreams();
   return (
-    <Tiles transition={{ durationMs: 2000 }}>
-      {inputs
-        .filter(input => input.videoState === 'playing')
-        .map(input => (
+    <Tiles transition={{ durationMs: 200 }}>
+      {Object.values(inputs).map(input =>
+        !input.videoState ? (
+          <Text key={input.inputId} fontSize={40}>
+            Waiting for stream {input.inputId} to connect
+          </Text>
+        ) : input.videoState === 'playing' ? (
           <InputTile key={input.inputId} inputId={input.inputId} />
-        ))}
+        ) : input.videoState === 'finished' ? (
+          <Text key={input.inputId} fontSize={40}>
+            Stream {input.inputId} finished
+          </Text>
+        ) : (
+          'Fallback'
+        )
+      )}
     </Tiles>
   );
 }

diff --git a/ts/examples/src/utils.ts b/ts/examples/src/utils.ts
@@ -28,6 +28,21 @@ export async function ffplayStartPlayerAsync(
   return { spawn_promise: promise };
 }
 
+export async function gstReceiveTcpStream(
+  ip: string,
+  port: number
+): Promise<{ spawn_promise: SpawnPromise }> {
+  const tcpReceiver = `tcpclientsrc host=${ip} port=${port} ! "application/x-rtp-stream" ! rtpstreamdepay ! queue ! demux.`;
+  const videoPipe =
+    'demux.src_96 ! "application/x-rtp,media=video,clock-rate=90000,encoding-name=H264" ! queue ! rtph264depay ! decodebin ! videoconvert ! autovideosink';
+  const audioPipe =
+    'demux.src_97 ! "application/x-rtp,media=audio,clock-rate=48000,encoding-name=OPUS" ! queue ! rtpopusdepay ! decodebin ! audioconvert ! autoaudiosink ';
+  const gstCmd = `gst-launch-1.0 -v rtpptdemux name=demux ${tcpReceiver} ${videoPipe} ${audioPipe}`;
+
+  const promise = spawn('bash', ['-c', gstCmd]);
+  return { spawn_promise: promise };
+}
+
 export function ffmpegSendVideoFromMp4(port: number, mp4Path: string): SpawnPromise {
   return spawn('ffmpeg', [
     '-stream_loop',