From 6686a34901be2045d64a747af7e17d957cd834bb Mon Sep 17 00:00:00 2001 From: Martin <901824+martinstark@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:20:08 +0200 Subject: [PATCH] feat: handle multiple audio streams --- .../production-line/production-line.tsx | 12 +--- .../production-line/use-audio-element.ts | 51 ----------------- .../production-line/use-rtc-connection.ts | 55 +++++++++++++------ 3 files changed, 42 insertions(+), 76 deletions(-) delete mode 100644 src/components/production-line/use-audio-element.ts diff --git a/src/components/production-line/production-line.tsx b/src/components/production-line/production-line.tsx index f6620d0a..31019d86 100644 --- a/src/components/production-line/production-line.tsx +++ b/src/components/production-line/production-line.tsx @@ -6,7 +6,6 @@ import { useAudioInput } from "./use-audio-input.ts"; import { useRtcConnection } from "./use-rtc-connection.ts"; import { useEstablishSession } from "./use-establish-session.ts"; import { ActionButton } from "../landing-page/form-elements.tsx"; -import { useAudioElement } from "./use-audio-element.ts"; import { UserList } from "./user-list.tsx"; import { API } from "../../api/api.ts"; import { noop } from "../../helpers.ts"; @@ -23,10 +22,6 @@ export const ProductionLine: FC = () => { { name: string; sessionid: string }[] | null >(null); - const { playbackState, audioElement } = useAudioElement({ - audioContainerRef, - }); - const inputAudioStream = useAudioInput({ inputId: joinProductionOptions?.audioinput ?? null, }); @@ -35,12 +30,11 @@ export const ProductionLine: FC = () => { joinProductionOptions, }); - const { connectionState } = useRtcConnection({ + const { connectionState, audioElements } = useRtcConnection({ inputAudioStream, sdpOffer, joinProductionOptions, sessionId, - audioElement, }); // Participant list, TODO extract hook to separate file @@ -91,8 +85,8 @@ export const ProductionLine: FC = () => { Production View - {playbackState && ( - Audio Playback State: {playbackState} + {audioElements.length && ( + Incoming Audio Channels: {audioElements.length} )} {connectionState && ( RTC Connection State: {connectionState} diff --git a/src/components/production-line/use-audio-element.ts b/src/components/production-line/use-audio-element.ts deleted file mode 100644 index 6a880b99..00000000 --- a/src/components/production-line/use-audio-element.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { RefObject, useEffect, useState } from "react"; -import { - FilteredMediaEvent, - getMediaEventFilter, -} from "@eyevinn/media-event-filter"; - -type TUseAudioElementOptions = { - audioContainerRef: RefObject; -}; - -export const useAudioElement = ({ - audioContainerRef, -}: TUseAudioElementOptions) => { - const [audioElement] = useState(() => new Audio()); - const [playbackState, setPlaybackState] = useState( - null - ); - - // TODO extract audio element hooks to separate file - // Set up audio element - useEffect(() => { - audioElement.controls = true; - audioElement.autoplay = true; - audioElement.onerror = console.error; - - const { teardown } = getMediaEventFilter({ - mediaElement: audioElement, - callback: (ev) => { - if (ev === FilteredMediaEvent.TIME_UPDATE) return; - - setPlaybackState(ev); - }, - }); - - return teardown; - }, [audioElement]); - - // Attach audio element to DOM - useEffect(() => { - if (!audioContainerRef) return; - - if (audioContainerRef.current) { - audioContainerRef.current.appendChild(audioElement); - } - }, [audioContainerRef, audioElement]); - - return { - playbackState, - audioElement, - }; -}; diff --git a/src/components/production-line/use-rtc-connection.ts b/src/components/production-line/use-rtc-connection.ts index 3e783373..04fc2438 100644 --- a/src/components/production-line/use-rtc-connection.ts +++ b/src/components/production-line/use-rtc-connection.ts @@ -1,4 +1,4 @@ -import { Dispatch, useEffect, useState } from "react"; +import { Dispatch, SetStateAction, useEffect, useState } from "react"; import { noop } from "../../helpers"; import { API } from "../../api/api.ts"; import { TJoinProductionOptions } from "./types.ts"; @@ -11,7 +11,6 @@ type TRtcConnectionOptions = { sdpOffer: string | null; joinProductionOptions: TJoinProductionOptions | null; sessionId: string | null; - audioElement: HTMLAudioElement; }; type TEstablishConnection = { @@ -19,8 +18,8 @@ type TEstablishConnection = { sdpOffer: string; joinProductionOptions: TJoinProductionOptions; sessionId: string; - audioElement: HTMLAudioElement; dispatch: Dispatch; + setAudioElements: Dispatch>; }; type TAttachAudioStream = { @@ -41,20 +40,30 @@ const establishConnection = ({ sdpOffer, joinProductionOptions, sessionId, - audioElement, dispatch, + setAudioElements, }: TEstablishConnection): { teardown: () => void } => { const onRtcTrack = ({ streams }: RTCTrackEvent) => { - // We can count on there being only a single stream for now. - // Needs updating if a video stream is also added. + // We can count on there being only a single stream per event for now. const selectedStream = streams[0]; - if (selectedStream) { - if (selectedStream.getAudioTracks().length !== 0) { - // Add incoming stream to output audio element - // eslint-disable-next-line no-param-reassign - audioElement.srcObject = selectedStream; - } + if (selectedStream && selectedStream.getAudioTracks().length !== 0) { + const audioElement = new Audio(); + + audioElement.controls = false; + audioElement.autoplay = true; + audioElement.onerror = () => { + dispatch({ + type: "ERROR", + payload: new Error( + `Audio Error: ${audioElement.error?.code} - ${audioElement.error?.message}` + ), + }); + }; + + audioElement.srcObject = selectedStream; + + setAudioElements((prevArray) => [audioElement, ...prevArray]); } else { // TODO handle error case of 0 available streams } @@ -139,7 +148,6 @@ export const useRtcConnection = ({ sdpOffer, joinProductionOptions, sessionId, - audioElement, }: TRtcConnectionOptions) => { const [rtcPeerConnection] = useState( () => new RTCPeerConnection() @@ -147,6 +155,20 @@ export const useRtcConnection = ({ const [, dispatch] = useGlobalState(); const [connectionState, setConnectionState] = useState(null); + const [audioElements, setAudioElements] = useState([]); + + // Teardown + useEffect( + () => () => { + audioElements.forEach((el) => { + console.log("Tearing down audio element"); + el.pause(); + // eslint-disable-next-line no-param-reassign + el.srcObject = null; + }); + }, + [audioElements] + ); useEffect(() => { if ( @@ -158,6 +180,8 @@ export const useRtcConnection = ({ return noop; } + console.log("Setting up RTC Peer Connection"); + const onConnectionStateChange = () => { setConnectionState(rtcPeerConnection.connectionState); }; @@ -181,8 +205,8 @@ export const useRtcConnection = ({ sdpOffer, joinProductionOptions, sessionId, - audioElement, dispatch, + setAudioElements, }); return () => { @@ -201,7 +225,6 @@ export const useRtcConnection = ({ sessionId, joinProductionOptions, rtcPeerConnection, - audioElement, dispatch, ]); @@ -268,5 +291,5 @@ export const useRtcConnection = ({ }; }, [rtcPeerConnection]); - return { connectionState }; + return { connectionState, audioElements }; };