From 6686a34901be2045d64a747af7e17d957cd834bb Mon Sep 17 00:00:00 2001
From: Martin <901824+martinstark@users.noreply.github.com>
Date: Wed, 10 Apr 2024 11:20:08 +0200
Subject: [PATCH] feat: handle multiple audio streams
---
.../production-line/production-line.tsx | 12 +---
.../production-line/use-audio-element.ts | 51 -----------------
.../production-line/use-rtc-connection.ts | 55 +++++++++++++------
3 files changed, 42 insertions(+), 76 deletions(-)
delete mode 100644 src/components/production-line/use-audio-element.ts
diff --git a/src/components/production-line/production-line.tsx b/src/components/production-line/production-line.tsx
index f6620d0a..31019d86 100644
--- a/src/components/production-line/production-line.tsx
+++ b/src/components/production-line/production-line.tsx
@@ -6,7 +6,6 @@ import { useAudioInput } from "./use-audio-input.ts";
import { useRtcConnection } from "./use-rtc-connection.ts";
import { useEstablishSession } from "./use-establish-session.ts";
import { ActionButton } from "../landing-page/form-elements.tsx";
-import { useAudioElement } from "./use-audio-element.ts";
import { UserList } from "./user-list.tsx";
import { API } from "../../api/api.ts";
import { noop } from "../../helpers.ts";
@@ -23,10 +22,6 @@ export const ProductionLine: FC = () => {
{ name: string; sessionid: string }[] | null
>(null);
- const { playbackState, audioElement } = useAudioElement({
- audioContainerRef,
- });
-
const inputAudioStream = useAudioInput({
inputId: joinProductionOptions?.audioinput ?? null,
});
@@ -35,12 +30,11 @@ export const ProductionLine: FC = () => {
joinProductionOptions,
});
- const { connectionState } = useRtcConnection({
+ const { connectionState, audioElements } = useRtcConnection({
inputAudioStream,
sdpOffer,
joinProductionOptions,
sessionId,
- audioElement,
});
// Participant list, TODO extract hook to separate file
@@ -91,8 +85,8 @@ export const ProductionLine: FC = () => {
Production View
- {playbackState && (
- Audio Playback State: {playbackState}
+ {audioElements.length && (
+ Incoming Audio Channels: {audioElements.length}
)}
{connectionState && (
RTC Connection State: {connectionState}
diff --git a/src/components/production-line/use-audio-element.ts b/src/components/production-line/use-audio-element.ts
deleted file mode 100644
index 6a880b99..00000000
--- a/src/components/production-line/use-audio-element.ts
+++ /dev/null
@@ -1,51 +0,0 @@
-import { RefObject, useEffect, useState } from "react";
-import {
- FilteredMediaEvent,
- getMediaEventFilter,
-} from "@eyevinn/media-event-filter";
-
-type TUseAudioElementOptions = {
- audioContainerRef: RefObject;
-};
-
-export const useAudioElement = ({
- audioContainerRef,
-}: TUseAudioElementOptions) => {
- const [audioElement] = useState(() => new Audio());
- const [playbackState, setPlaybackState] = useState(
- null
- );
-
- // TODO extract audio element hooks to separate file
- // Set up audio element
- useEffect(() => {
- audioElement.controls = true;
- audioElement.autoplay = true;
- audioElement.onerror = console.error;
-
- const { teardown } = getMediaEventFilter({
- mediaElement: audioElement,
- callback: (ev) => {
- if (ev === FilteredMediaEvent.TIME_UPDATE) return;
-
- setPlaybackState(ev);
- },
- });
-
- return teardown;
- }, [audioElement]);
-
- // Attach audio element to DOM
- useEffect(() => {
- if (!audioContainerRef) return;
-
- if (audioContainerRef.current) {
- audioContainerRef.current.appendChild(audioElement);
- }
- }, [audioContainerRef, audioElement]);
-
- return {
- playbackState,
- audioElement,
- };
-};
diff --git a/src/components/production-line/use-rtc-connection.ts b/src/components/production-line/use-rtc-connection.ts
index 3e783373..04fc2438 100644
--- a/src/components/production-line/use-rtc-connection.ts
+++ b/src/components/production-line/use-rtc-connection.ts
@@ -1,4 +1,4 @@
-import { Dispatch, useEffect, useState } from "react";
+import { Dispatch, SetStateAction, useEffect, useState } from "react";
import { noop } from "../../helpers";
import { API } from "../../api/api.ts";
import { TJoinProductionOptions } from "./types.ts";
@@ -11,7 +11,6 @@ type TRtcConnectionOptions = {
sdpOffer: string | null;
joinProductionOptions: TJoinProductionOptions | null;
sessionId: string | null;
- audioElement: HTMLAudioElement;
};
type TEstablishConnection = {
@@ -19,8 +18,8 @@ type TEstablishConnection = {
sdpOffer: string;
joinProductionOptions: TJoinProductionOptions;
sessionId: string;
- audioElement: HTMLAudioElement;
dispatch: Dispatch;
+ setAudioElements: Dispatch>;
};
type TAttachAudioStream = {
@@ -41,20 +40,30 @@ const establishConnection = ({
sdpOffer,
joinProductionOptions,
sessionId,
- audioElement,
dispatch,
+ setAudioElements,
}: TEstablishConnection): { teardown: () => void } => {
const onRtcTrack = ({ streams }: RTCTrackEvent) => {
- // We can count on there being only a single stream for now.
- // Needs updating if a video stream is also added.
+ // We can count on there being only a single stream per event for now.
const selectedStream = streams[0];
- if (selectedStream) {
- if (selectedStream.getAudioTracks().length !== 0) {
- // Add incoming stream to output audio element
- // eslint-disable-next-line no-param-reassign
- audioElement.srcObject = selectedStream;
- }
+ if (selectedStream && selectedStream.getAudioTracks().length !== 0) {
+ const audioElement = new Audio();
+
+ audioElement.controls = false;
+ audioElement.autoplay = true;
+ audioElement.onerror = () => {
+ dispatch({
+ type: "ERROR",
+ payload: new Error(
+ `Audio Error: ${audioElement.error?.code} - ${audioElement.error?.message}`
+ ),
+ });
+ };
+
+ audioElement.srcObject = selectedStream;
+
+ setAudioElements((prevArray) => [audioElement, ...prevArray]);
} else {
// TODO handle error case of 0 available streams
}
@@ -139,7 +148,6 @@ export const useRtcConnection = ({
sdpOffer,
joinProductionOptions,
sessionId,
- audioElement,
}: TRtcConnectionOptions) => {
const [rtcPeerConnection] = useState(
() => new RTCPeerConnection()
@@ -147,6 +155,20 @@ export const useRtcConnection = ({
const [, dispatch] = useGlobalState();
const [connectionState, setConnectionState] =
useState(null);
+ const [audioElements, setAudioElements] = useState([]);
+
+ // Teardown
+ useEffect(
+ () => () => {
+ audioElements.forEach((el) => {
+ console.log("Tearing down audio element");
+ el.pause();
+ // eslint-disable-next-line no-param-reassign
+ el.srcObject = null;
+ });
+ },
+ [audioElements]
+ );
useEffect(() => {
if (
@@ -158,6 +180,8 @@ export const useRtcConnection = ({
return noop;
}
+ console.log("Setting up RTC Peer Connection");
+
const onConnectionStateChange = () => {
setConnectionState(rtcPeerConnection.connectionState);
};
@@ -181,8 +205,8 @@ export const useRtcConnection = ({
sdpOffer,
joinProductionOptions,
sessionId,
- audioElement,
dispatch,
+ setAudioElements,
});
return () => {
@@ -201,7 +225,6 @@ export const useRtcConnection = ({
sessionId,
joinProductionOptions,
rtcPeerConnection,
- audioElement,
dispatch,
]);
@@ -268,5 +291,5 @@ export const useRtcConnection = ({
};
}, [rtcPeerConnection]);
- return { connectionState };
+ return { connectionState, audioElements };
};