Skip to content

Commit

Permalink
[ts-sdk] Add hook for audio
Browse files Browse the repository at this point in the history
  • Loading branch information
wkozyra95 committed Sep 2, 2024
1 parent f34caae commit e1122ab
Show file tree
Hide file tree
Showing 18 changed files with 525 additions and 157 deletions.
28 changes: 15 additions & 13 deletions ts/@live-compositor/core/src/api/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,40 @@ import { RegisterOutput, Api, Outputs } from 'live-compositor';

export function intoRegisterOutput(
output: RegisterOutput,
initialVideo?: Api.Video
initial: { video?: Api.Video; audio?: Api.Audio }
): Api.RegisterOutput {
if (output.type === 'rtp_stream') {
return intoRegisterRtpOutput(output, initialVideo);
return intoRegisterRtpOutput(output, initial);
} else if (output.type === 'mp4') {
return intoRegisterMp4Output(output, initialVideo);
return intoRegisterMp4Output(output, initial);
} else {
throw new Error(`Unknown input type ${(output as any).type}`);
}
}

function intoRegisterRtpOutput(
output: Outputs.RegisterRtpOutput,
initialVideo?: Api.Video
initial: { video?: Api.Video; audio?: Api.Audio }
): Api.RegisterOutput {
return {
type: 'rtp_stream',
port: output.port,
ip: output.ip,
transport_protocol: output.transportProtocol,
video: output.video && initialVideo && intoOutputVideoOptions(output.video, initialVideo),
audio: output.audio && intoOutputRtpAudioOptions(output.audio),
video: output.video && initial.video && intoOutputVideoOptions(output.video, initial.video),
audio: output.audio && initial.audio && intoOutputRtpAudioOptions(output.audio, initial.audio),
};
}

function intoRegisterMp4Output(
output: Outputs.RegisterMp4Output,
initialVideo?: Api.Video
initial: { video?: Api.Video; audio?: Api.Audio }
): Api.RegisterOutput {
return {
type: 'mp4',
path: output.serverPath,
video: output.video && initialVideo && intoOutputVideoOptions(output.video, initialVideo),
audio: output.audio && intoOutputMp4AudioOptions(output.audio),
video: output.video && initial.video && intoOutputVideoOptions(output.video, initial.video),
audio: output.audio && initial.audio && intoOutputMp4AudioOptions(output.audio, initial.audio),
};
}

Expand All @@ -62,22 +62,24 @@ function intoVideoEncoderOptions(
}

function intoOutputRtpAudioOptions(
audio: Outputs.OutputRtpAudioOptions
audio: Outputs.OutputRtpAudioOptions,
initial: Api.Audio
): Api.OutputRtpAudioOptions {
return {
send_eos_when: audio.sendEosWhen && intoOutputEosCondition(audio.sendEosWhen),
encoder: intoRtpAudioEncoderOptions(audio.encoder),
initial: intoAudioInputsConfiguration(audio.initial),
initial,
};
}

function intoOutputMp4AudioOptions(
audio: Outputs.OutputMp4AudioOptions
audio: Outputs.OutputMp4AudioOptions,
initial: Api.Audio
): Api.OutputMp4AudioOptions {
return {
send_eos_when: audio.sendEosWhen && intoOutputEosCondition(audio.sendEosWhen),
encoder: intoMp4AudioEncoderOptions(audio.encoder),
initial: intoAudioInputsConfiguration(audio.initial),
initial,
};
}

Expand Down
14 changes: 10 additions & 4 deletions ts/@live-compositor/core/src/compositor.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { ContextStore, RegisterInput, RegisterOutput } from 'live-compositor';
import { _liveCompositorInternals, RegisterInput, RegisterOutput } from 'live-compositor';
import { ApiClient, Api } from './api';
import Output from './output';
import { CompositorManager } from './compositorManager';
Expand All @@ -15,12 +15,13 @@ export async function createLiveCompositor(manager: CompositorManager): Promise<
export class LiveCompositor {
private manager: CompositorManager;
private api: ApiClient;
private store: ContextStore = new ContextStore();
private store: _liveCompositorInternals.InstanceContextStore;
private outputs: Record<string, Output> = {};

public constructor(manager: CompositorManager) {
this.manager = manager;
this.api = new ApiClient(this.manager);
this.store = new _liveCompositorInternals.InstanceContextStore();
}

private async setupInstance() {
Expand All @@ -30,14 +31,19 @@ export class LiveCompositor {

public async registerOutput(outputId: string, request: RegisterOutput): Promise<object> {
const output = new Output(outputId, request, this.api, this.store);
const initialVideo = output.scene();

const apiRequest = intoRegisterOutput(request, initialVideo);
const apiRequest = intoRegisterOutput(request, output.scene());
const result = await this.api.registerOutput(outputId, apiRequest);
this.outputs[outputId] = output;
return result;
}

public async unregisterOutput(outputId: string): Promise<object> {
this.outputs[outputId].close();
delete this.outputs[outputId];
return this.api.unregisterOutput(outputId);
}

public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
const result = await this.api.registerInput(inputId, intoRegisterInput(request));
this.store.addInput({ inputId });
Expand Down
6 changes: 4 additions & 2 deletions ts/@live-compositor/core/src/event.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { CompositorEvent, CompositorEventType, ContextStore } from 'live-compositor';
import { _liveCompositorInternals, CompositorEvent, CompositorEventType } from 'live-compositor';

export function onCompositorEvent(store: ContextStore, rawEvent: unknown) {
type InstanceContextStore = _liveCompositorInternals.InstanceContextStore;

export function onCompositorEvent(store: InstanceContextStore, rawEvent: unknown) {
const event = parseEvent(rawEvent);
if (!event) {
return;
Expand Down
155 changes: 97 additions & 58 deletions ts/@live-compositor/core/src/output.ts
Original file line number Diff line number Diff line change
@@ -1,91 +1,130 @@
import { ContextStore, LiveCompositorContext, RegisterOutput } from 'live-compositor';
import { _liveCompositorInternals, RegisterOutput, View } from 'live-compositor';
import React, { useSyncExternalStore } from 'react';
import { ApiClient, Api } from './api';
import Renderer from './renderer';
import { intoAudioInputsConfiguration } from './api/output';
import React from 'react';
import { throttle } from './utils';

type OutputContext = _liveCompositorInternals.OutputContext;
type InstanceContextStore = _liveCompositorInternals.InstanceContextStore;

class Output {
api: ApiClient;
outputId: string;
outputCtx: OutputContext;
outputShutdownStateStore: OutputShutdownStateStore;
initialized: boolean = false;

throttledUpdate: () => void;
videoRenderer?: Renderer;

constructor(outputId: string, output: RegisterOutput, api: ApiClient, store: ContextStore) {
constructor(
outputId: string,
registerRequest: RegisterOutput,
api: ApiClient,
store: InstanceContextStore
) {
this.api = api;
this.outputId = outputId;
this.outputShutdownStateStore = new OutputShutdownStateStore();

let audioOptions: Api.Audio | undefined;
if (output.video) {
this.videoRenderer = new Renderer(
React.createElement(LiveCompositorContext.Provider, { value: store }, output.video.root),
() => this.onRendererUpdate(),
`${outputId}-`
);
}
if (output.audio) {
audioOptions = intoAudioInputsConfiguration(output.audio.initial);
const onUpdate = () => this.throttledUpdate?.();
this.outputCtx = new _liveCompositorInternals.OutputContext(
onUpdate,
registerRequest.audio?.initial
);

if (registerRequest.video) {
const rootElement = React.createElement(OutputRootComponent, {
instanceStore: store,
outputCtx: this.outputCtx,
outputRoot: registerRequest.video.root,
outputShutdownStateStore: this.outputShutdownStateStore,
});

this.videoRenderer = new Renderer({
rootElement,
onUpdate,
idPrefix: `${outputId}-`,
});
}

this.throttledUpdate = throttle(async () => {
await api.updateScene(this.outputId, {
video: this.videoRenderer && { root: this.videoRenderer.scene() },
audio: audioOptions,
});
await api.updateScene(this.outputId, this.scene());
}, 30);
}

public scene(): Api.Video | undefined {
return this.videoRenderer && { root: this.videoRenderer.scene() };
public scene(): { video?: Api.Video; audio?: Api.Audio } {
const audio = this.outputCtx.getAudioConfig();
return {
video: this.videoRenderer && { root: this.videoRenderer.scene() },
audio: audio && intoAudioInputsConfiguration(audio),
};
}

private onRendererUpdate() {
if (!this.throttledUpdate || !this.videoRenderer) {
return;
}
this.throttledUpdate();
public close(): void {
this.throttledUpdate = () => {};
// close will switch a scene to just a <View />, so we need replace `throttledUpdate`
// callback before it is called
this.outputShutdownStateStore.close();
}
}

function throttle(fn: () => Promise<void>, timeoutMs: number): () => void {
let shouldCall: boolean = false;
let running: boolean = false;

const start = async () => {
while (shouldCall) {
const start = Date.now();
shouldCall = false;

try {
await fn();
} catch (error) {
console.log(error);
}

const timeoutLeft = start + timeoutMs - Date.now();
if (timeoutLeft > 0) {
await sleep(timeoutLeft);
}
running = false;
}
// External store to share shutdown information between React tree
// and external code that is managing it.
class OutputShutdownStateStore {
private shutdown: boolean = false;
private onChangeCallbacks: Set<() => void> = new Set();

public close() {
this.shutdown = true;
this.onChangeCallbacks.forEach(cb => cb());
}

// callback for useSyncExternalStore
public getSnapshot = (): boolean => {
return this.shutdown;
};

return () => {
shouldCall = true;
if (running) {
return;
}
running = true;
void start();
// callback for useSyncExternalStore
public subscribe = (onStoreChange: () => void): (() => void) => {
this.onChangeCallbacks.add(onStoreChange);
return () => {
this.onChangeCallbacks.delete(onStoreChange);
};
};
}

async function sleep(timeout_ms: number): Promise<void> {
await new Promise<void>(res => {
setTimeout(() => {
res();
}, timeout_ms);
});
function OutputRootComponent({
outputRoot,
instanceStore,
outputCtx,
outputShutdownStateStore,
}: {
outputRoot: React.ReactElement;
instanceStore: InstanceContextStore;
outputCtx: OutputContext;
outputShutdownStateStore: OutputShutdownStateStore;
}) {
const shouldShutdown = useSyncExternalStore(
outputShutdownStateStore.subscribe,
outputShutdownStateStore.getSnapshot
);

if (shouldShutdown) {
// replace root with view to stop all the dynamic code
return React.createElement(View, {});
}

const reactCtx = {
instanceStore,
outputCtx,
};
return React.createElement(
_liveCompositorInternals.LiveCompositorContext.Provider,
{ value: reactCtx },
outputRoot
);
}

export default Output;
30 changes: 18 additions & 12 deletions ts/@live-compositor/core/src/renderer.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import Reconciler from 'react-reconciler';
import { DefaultEventPriority, LegacyRoot } from 'react-reconciler/constants';
import { Api } from './api';
import { SceneBuilder, SceneComponent } from 'live-compositor';
import { _liveCompositorInternals } from 'live-compositor';
import React from 'react';

type SceneBuilder<P> = _liveCompositorInternals.SceneBuilder<P>;
type SceneComponent = _liveCompositorInternals.SceneComponent;

export class LiveCompositorHostComponent {
public props: object;
Expand Down Expand Up @@ -210,12 +214,20 @@ const HostConfig: Reconciler.HostConfig<

const CompositorRenderer = Reconciler(HostConfig);

type RendererOptions = {
rootElement: React.ReactElement;
onUpdate: () => void;
idPrefix: string;
};

class Renderer {
root: any;
onUpdateFn: () => void;
private root: any;
public readonly onUpdate: () => void;

constructor(element: React.ReactElement, onUpdate: () => void, idPrefix: string) {
const root = CompositorRenderer.createContainer(
constructor({ rootElement, onUpdate, idPrefix }: RendererOptions) {
this.onUpdate = onUpdate;

this.root = CompositorRenderer.createContainer(
this, // container tag
LegacyRoot,
null, // hydrationCallbacks
Expand All @@ -225,10 +237,8 @@ class Renderer {
console.error, // onRecoverableError
null // transitionCallbacks
);
this.root = root;
this.onUpdateFn = onUpdate;

CompositorRenderer.updateContainer(element, root, null, () => {});
CompositorRenderer.updateContainer(rootElement, this.root, null, () => {});
}

public scene(): Api.Component {
Expand All @@ -238,10 +248,6 @@ class Renderer {
const rootComponent = this.root.pendingChildren[0] ?? rootHostComponent(this.root.current);
return rootComponent.scene();
}

public onUpdate() {
this.onUpdateFn();
}
}

function rootHostComponent(root: any): LiveCompositorHostComponent {
Expand Down
Loading

0 comments on commit e1122ab

Please sign in to comment.