diff --git a/src/components/ha-web-rtc-player.ts b/src/components/ha-web-rtc-player.ts index e6276aa5f82f..a2ae49e47299 100644 --- a/src/components/ha-web-rtc-player.ts +++ b/src/components/ha-web-rtc-player.ts @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +import { UnsubscribeFunc } from "home-assistant-js-websocket"; import { css, CSSResultGroup, @@ -11,9 +12,12 @@ import { customElement, property, query, state } from "lit/decorators"; import { ifDefined } from "lit/directives/if-defined"; import { fireEvent } from "../common/dom/fire_event"; import { + addWebRtcCandidate, fetchWebRtcClientConfiguration, - handleWebRtcOffer, WebRtcAnswer, + WebRTCClientConfiguration, + webRtcOffer, + WebRtcOfferEvent, } from "../data/camera"; import type { HomeAssistant } from "../types"; import "./ha-alert"; @@ -27,7 +31,7 @@ import "./ha-alert"; class HaWebRtcPlayer extends LitElement { @property({ attribute: false }) public hass!: HomeAssistant; - @property() public entityid!: string; + @property() public entityid?: string; @property({ type: Boolean, attribute: "controls" }) public controls = false; @@ -45,12 +49,20 @@ class HaWebRtcPlayer extends LitElement { @state() private _error?: string; - @query("#remote-stream", true) private _videoEl!: HTMLVideoElement; + @query("#remote-stream") private _videoEl!: HTMLVideoElement; + + private _clientConfig?: WebRTCClientConfiguration; private _peerConnection?: RTCPeerConnection; private _remoteStream?: MediaStream; + private _unsub?: Promise; + + private _sessionId?: string; + + private _candidatesList: string[] = []; + protected override render(): TemplateResult { if (this._error) { return html`${this._error}`; @@ -70,7 +82,7 @@ class HaWebRtcPlayer extends LitElement { public override connectedCallback() { super.connectedCallback(); - if (this.hasUpdated) { + if (this.hasUpdated && this.entityid) { this._startWebRtc(); } } @@ -80,7 +92,8 @@ class HaWebRtcPlayer extends LitElement { this._cleanUp(); } - protected override updated(changedProperties: PropertyValues) { + protected override willUpdate(changedProperties: PropertyValues) { + super.willUpdate(changedProperties); if (!changedProperties.has("entityid")) { return; } @@ -88,28 +101,68 @@ class HaWebRtcPlayer extends LitElement { } private async _startWebRtc(): Promise { + this._cleanUp(); + + if (!this.hass || !this.entityid) { + return; + } + console.time("WebRTC"); this._error = undefined; console.timeLog("WebRTC", "start clientConfig"); - const clientConfig = await fetchWebRtcClientConfiguration( + this._clientConfig = await fetchWebRtcClientConfiguration( this.hass, this.entityid ); - console.timeLog("WebRTC", "end clientConfig", clientConfig); + console.timeLog("WebRTC", "end clientConfig", this._clientConfig); - const peerConnection = new RTCPeerConnection(clientConfig.configuration); + this._peerConnection = new RTCPeerConnection( + this._clientConfig.configuration + ); - if (clientConfig.dataChannel) { + if (this._clientConfig.dataChannel) { // Some cameras (such as nest) require a data channel to establish a stream // however, not used by any integrations. - peerConnection.createDataChannel(clientConfig.dataChannel); + this._peerConnection.createDataChannel(this._clientConfig.dataChannel); + } + + this._peerConnection.onnegotiationneeded = this._startNegotiation; + + this._peerConnection.onicecandidate = this._handleIceCandidate; + this._peerConnection.oniceconnectionstatechange = + this._iceConnectionStateChanged; + + // just for debugging + this._peerConnection.onsignalingstatechange = (ev) => { + switch ((ev.target as RTCPeerConnection).signalingState) { + case "stable": + console.timeLog("WebRTC", "ICE negotiation complete"); + break; + default: + console.timeLog( + "WebRTC", + "Signaling state changed", + (ev.target as RTCPeerConnection).signalingState + ); + } + }; + + // Setup callbacks to render remote stream once media tracks are discovered. + this._remoteStream = new MediaStream(); + this._peerConnection.ontrack = this._addTrack; + + this._peerConnection.addTransceiver("audio", { direction: "recvonly" }); + this._peerConnection.addTransceiver("video", { direction: "recvonly" }); + } + + private _startNegotiation = async () => { + if (!this._peerConnection) { + return; } - peerConnection.addTransceiver("audio", { direction: "recvonly" }); - peerConnection.addTransceiver("video", { direction: "recvonly" }); const offerOptions: RTCOfferOptions = { offerToReceiveAudio: true, @@ -119,98 +172,218 @@ class HaWebRtcPlayer extends LitElement { console.timeLog("WebRTC", "start createOffer", offerOptions); const offer: RTCSessionDescriptionInit = - await peerConnection.createOffer(offerOptions); + await this._peerConnection.createOffer(offerOptions); + + if (!this._peerConnection) { + return; + } console.timeLog("WebRTC", "end createOffer", offer); console.timeLog("WebRTC", "start setLocalDescription"); - await peerConnection.setLocalDescription(offer); + await this._peerConnection.setLocalDescription(offer); + + if (!this._peerConnection || !this.entityid) { + return; + } console.timeLog("WebRTC", "end setLocalDescription"); - console.timeLog("WebRTC", "start iceResolver"); - - let candidates = ""; // Build an Offer SDP string with ice candidates - const iceResolver = new Promise((resolve) => { - peerConnection.addEventListener("icecandidate", (event) => { - if (!event.candidate?.candidate) { - resolve(); // Gathering complete - return; - } - console.timeLog("WebRTC", "iceResolver candidate", event.candidate); - candidates += `a=${event.candidate.candidate}\r\n`; + let candidates = ""; + + if (this._clientConfig?.getCandidatesUpfront) { + await new Promise((resolve) => { + this._peerConnection!.onicegatheringstatechange = (ev: Event) => { + const iceGatheringState = (ev.target as RTCPeerConnection) + .iceGatheringState; + if (iceGatheringState === "complete") { + this._peerConnection!.onicegatheringstatechange = null; + resolve(); + } + + console.timeLog( + "WebRTC", + "Ice gathering state changed", + iceGatheringState + ); + }; }); - }); - await iceResolver; - console.timeLog("WebRTC", "end iceResolver", candidates); + if (!this._peerConnection || !this.entityid) { + return; + } + } + + while (this._candidatesList.length) { + const candidate = this._candidatesList.pop(); + if (candidate) { + candidates += `a=${candidate}\r\n`; + } + } const offer_sdp = offer.sdp! + candidates; - let webRtcAnswer: WebRtcAnswer; + console.timeLog("WebRTC", "start webRtcOffer", offer_sdp); + try { - console.timeLog("WebRTC", "start WebRTCOffer", offer_sdp); - webRtcAnswer = await handleWebRtcOffer( - this.hass, - this.entityid, - offer_sdp + this._unsub = webRtcOffer(this.hass, this.entityid, offer_sdp, (event) => + this._handleOfferEvent(event) ); - console.timeLog("WebRTC", "end webRtcOffer", webRtcAnswer); } catch (err: any) { this._error = "Failed to start WebRTC stream: " + err.message; - peerConnection.close(); + this._cleanUp(); + } + }; + + private _iceConnectionStateChanged = () => { + console.timeLog( + "WebRTC", + "ice connection state change", + this._peerConnection?.iceConnectionState + ); + if (this._peerConnection?.iceConnectionState === "failed") { + this._peerConnection.restartIce(); + } + }; + + private async _handleOfferEvent(event: WebRtcOfferEvent) { + if (!this.entityid) { return; } + if (event.type === "session") { + this._sessionId = event.session_id; + this._candidatesList.forEach((candidate) => + addWebRtcCandidate( + this.hass, + this.entityid!, + event.session_id, + candidate + ) + ); + this._candidatesList = []; + } + if (event.type === "answer") { + console.timeLog("WebRTC", "answer", event.answer); - // Setup callbacks to render remote stream once media tracks are discovered. - const remoteStream = new MediaStream(); - peerConnection.addEventListener("track", (event) => { - console.timeLog("WebRTC", "track", event); - remoteStream.addTrack(event.track); - this._videoEl.srcObject = remoteStream; - }); - this._remoteStream = remoteStream; + this._handleAnswer(event); + } + if (event.type === "candidate") { + console.timeLog("WebRTC", "remote ice candidate", event.candidate); + + try { + await this._peerConnection?.addIceCandidate( + new RTCIceCandidate({ candidate: event.candidate, sdpMid: "0" }) + ); + } catch (err: any) { + console.error(err); + } + } + if (event.type === "error") { + this._error = "Failed to start WebRTC stream: " + event.message; + this._cleanUp(); + } + } + + private _handleIceCandidate = (event: RTCPeerConnectionIceEvent) => { + if (!this.entityid || !event.candidate?.candidate) { + return; + } + + console.timeLog( + "WebRTC", + "local ice candidate", + event.candidate?.candidate + ); + + if (this._sessionId) { + addWebRtcCandidate( + this.hass, + this.entityid, + this._sessionId, + event.candidate?.candidate + ); + } else { + this._candidatesList.push(event.candidate?.candidate); + } + }; + + private _addTrack = async (event: RTCTrackEvent) => { + if (!this._remoteStream) { + return; + } + this._remoteStream.addTrack(event.track); + if (!this.hasUpdated) { + await this.updateComplete; + } + this._videoEl.srcObject = this._remoteStream; + }; + + private async _handleAnswer(event: WebRtcAnswer) { + if ( + !this._peerConnection?.signalingState || + ["stable", "closed"].includes(this._peerConnection.signalingState) + ) { + return; + } // Initiate the stream with the remote device const remoteDesc = new RTCSessionDescription({ type: "answer", - sdp: webRtcAnswer.answer, + sdp: event.answer, }); try { console.timeLog("WebRTC", "start setRemoteDescription", remoteDesc); - await peerConnection.setRemoteDescription(remoteDesc); - console.timeLog("WebRTC", "end setRemoteDescription"); + await this._peerConnection.setRemoteDescription(remoteDesc); } catch (err: any) { this._error = "Failed to connect WebRTC stream: " + err.message; - peerConnection.close(); - return; + this._cleanUp(); } - this._peerConnection = peerConnection; + console.timeLog("WebRTC", "end setRemoteDescription"); } private _cleanUp() { + console.timeLog("WebRTC", "stopped"); + console.timeEnd("WebRTC"); + if (this._remoteStream) { this._remoteStream.getTracks().forEach((track) => { track.stop(); }); + this._remoteStream = undefined; } - if (this._videoEl) { - this._videoEl.removeAttribute("src"); - this._videoEl.load(); + const videoEl = this._videoEl; + if (videoEl) { + videoEl.removeAttribute("src"); + videoEl.load(); } if (this._peerConnection) { this._peerConnection.close(); + + this._peerConnection.onnegotiationneeded = null; + this._peerConnection.onicecandidate = null; + this._peerConnection.oniceconnectionstatechange = null; + this._peerConnection.onicegatheringstatechange = null; + this._peerConnection.ontrack = null; + + // just for debugging + this._peerConnection.onsignalingstatechange = null; + this._peerConnection = undefined; } + this._unsub?.then((unsub) => unsub()); + this._unsub = undefined; + this._sessionId = undefined; + this._candidatesList = []; } private _loadedData() { - console.timeLog("WebRTC", "loadedData"); - console.timeEnd("WebRTC"); // @ts-ignore fireEvent(this, "load"); + + console.timeLog("WebRTC", "loadedData"); + console.timeEnd("WebRTC"); } static get styles(): CSSResultGroup { diff --git a/src/data/camera.ts b/src/data/camera.ts index 0bf40f9cb524..84df934a74ac 100644 --- a/src/data/camera.ts +++ b/src/data/camera.ts @@ -39,10 +39,37 @@ export interface Stream { url: string; } +export type WebRtcOfferEvent = + | WebRtcId + | WebRtcAnswer + | WebRtcCandidate + | WebRtcError; + +export interface WebRtcId { + type: "session"; + session_id: string; +} + export interface WebRtcAnswer { + type: "answer"; answer: string; } +export interface WebRtcCandidate { + type: "candidate"; + candidate: string; +} + +export interface WebRtcError { + type: "error"; + code: string; + message: string; +} + +export interface WebRtcOfferResponse { + id: string; +} + export const cameraUrlWithWidthHeight = ( base_url: string, width: number, @@ -94,15 +121,29 @@ export const fetchStreamUrl = async ( return stream; }; -export const handleWebRtcOffer = ( +export const webRtcOffer = ( hass: HomeAssistant, - entityId: string, - offer: string + entity_id: string, + offer: string, + callback: (event: WebRtcOfferEvent) => void ) => - hass.callWS({ - type: "camera/web_rtc_offer", - entity_id: entityId, - offer: offer, + hass.connection.subscribeMessage(callback, { + type: "camera/webrtc/offer", + entity_id, + offer, + }); + +export const addWebRtcCandidate = ( + hass: HomeAssistant, + entity_id: string, + session_id: string, + candidate: string +) => + hass.callWS({ + type: "camera/webrtc/candidate", + entity_id, + session_id, + candidate, }); export const fetchCameraPrefs = (hass: HomeAssistant, entityId: string) => @@ -137,6 +178,7 @@ export const getEntityIdFromCameraMediaSource = (mediaContentId: string) => export interface WebRTCClientConfiguration { configuration: RTCConfiguration; dataChannel?: string; + getCandidatesUpfront: boolean; } export const fetchWebRtcClientConfiguration = async (