Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Try both HLS and webRTC and pick best stream #22585

Merged
merged 3 commits into from
Oct 29, 2024
Merged

Try both HLS and webRTC and pick best stream #22585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c1100cd
Try both HLS and webRTC and pick best stream
bramkragten Oct 29, 2024
605910c
move support checks
bramkragten Oct 29, 2024
29a2d0a
Improve fallback to mjpeg
bramkragten Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 104 additions & 60 deletions src/components/ha-camera-stream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,22 @@ import {
CSSResultGroup,
html,
LitElement,
PropertyValues,
nothing,
PropertyValues,
} from "lit";
import { customElement, property, state } from "lit/decorators";
import { isComponentLoaded } from "../common/config/is_component_loaded";
import { computeStateName } from "../common/entity/compute_state_name";
import { supportsFeature } from "../common/entity/supports-feature";
import {
CameraEntity,
CAMERA_SUPPORT_STREAM,
CameraCapabilities,
CameraEntity,
computeMJPEGStreamUrl,
fetchStreamUrl,
fetchCameraCapabilities,
fetchThumbnailUrlWithCache,
STREAM_TYPE_HLS,
STREAM_TYPE_WEB_RTC,
StreamType,
} from "../data/camera";
import { HomeAssistant } from "../types";
import "./ha-hls-player";
Expand All @@ -41,13 +42,15 @@ export class HaCameraStream extends LitElement {
// Video background image before its loaded
@state() private _posterUrl?: string;

// We keep track if we should force MJPEG if there was a failure
// to get the HLS stream url. This is reset if we change entities.
@state() private _forceMJPEG?: string;
@state() private _connected = false;

@state() private _url?: string;
@state() private _capabilities?: CameraCapabilities;

@state() private _connected = false;
@state() private _streamType?: StreamType;

@state() private _hlsStreams?: { hasAudio: boolean; hasVideo: boolean };

@state() private _webRtcStreams?: { hasAudio: boolean; hasVideo: boolean };

public willUpdate(changedProps: PropertyValues): void {
if (
Expand All @@ -57,12 +60,8 @@ export class HaCameraStream extends LitElement {
(changedProps.get("stateObj") as CameraEntity | undefined)?.entity_id !==
this.stateObj.entity_id
) {
this._getCapabilities();
this._getPosterUrl();
if (this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_HLS) {
this._forceMJPEG = undefined;
this._url = undefined;
this._getStreamUrl();
}
}
}

Expand All @@ -87,54 +86,79 @@ export class HaCameraStream extends LitElement {
: this._connected
? computeMJPEGStreamUrl(this.stateObj)
: ""}
.alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`}
alt=${`Preview of the ${computeStateName(this.stateObj)} camera.`}
/>`;
}
if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_HLS) {
return this._url
? html`<ha-hls-player
autoplay
playsinline
.allowExoPlayer=${this.allowExoPlayer}
.muted=${this.muted}
.controls=${this.controls}
.hass=${this.hass}
.url=${this._url}
.posterUrl=${this._posterUrl}
></ha-hls-player>`
: nothing;
return html`${this._streamType === STREAM_TYPE_HLS ||
(!this._streamType &&
this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_HLS))
? html`<ha-hls-player
autoplay
playsinline
.allowExoPlayer=${this.allowExoPlayer}
.muted=${this.muted}
.controls=${this.controls}
.hass=${this.hass}
.entityid=${this.stateObj.entity_id}
.posterUrl=${this._posterUrl}
@streams=${this._handleHlsStreams}
class=${!this._streamType && this._webRtcStreams ? "hidden" : ""}
></ha-hls-player>`
: nothing}
${this._streamType === STREAM_TYPE_WEB_RTC ||
(!this._streamType &&
this._capabilities?.frontend_stream_types.includes(STREAM_TYPE_WEB_RTC))
? html`<ha-web-rtc-player
autoplay
playsinline
.muted=${this.muted}
.controls=${this.controls}
.hass=${this.hass}
.entityid=${this.stateObj.entity_id}
.posterUrl=${this._posterUrl}
@streams=${this._handleWebRtcStreams}
class=${this._streamType !== STREAM_TYPE_WEB_RTC &&
!this._webRtcStreams
? "hidden"
: ""}
></ha-web-rtc-player>`
: nothing}`;
}

private async _getCapabilities() {
this._capabilities = undefined;
this._hlsStreams = undefined;
this._webRtcStreams = undefined;
if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) {
return;
}
if (this.stateObj.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC) {
return html`<ha-web-rtc-player
autoplay
playsinline
.muted=${this.muted}
.controls=${this.controls}
.hass=${this.hass}
.entityid=${this.stateObj.entity_id}
.posterUrl=${this._posterUrl}
></ha-web-rtc-player>`;
this._capabilities = await fetchCameraCapabilities(
this.hass!,
this.stateObj!.entity_id
);
if (this._capabilities.frontend_stream_types.length === 1) {
this._streamType = this._capabilities.frontend_stream_types[0];
}
return nothing;
}

private get _shouldRenderMJPEG() {
if (this._forceMJPEG === this.stateObj!.entity_id) {
// Fallback when unable to fetch stream url
return true;
}
if (!supportsFeature(this.stateObj!, CAMERA_SUPPORT_STREAM)) {
// Steaming is not supported by the camera so fallback to MJPEG stream
return true;
}
if (
this.stateObj!.attributes.frontend_stream_type === STREAM_TYPE_WEB_RTC
this._capabilities &&
(!this._capabilities.frontend_stream_types.includes(STREAM_TYPE_HLS) ||
this._hlsStreams?.hasVideo === false) &&
(!this._capabilities.frontend_stream_types.includes(
STREAM_TYPE_WEB_RTC
) ||
this._webRtcStreams?.hasVideo === false)
) {
// Browser support required for WebRTC
return typeof RTCPeerConnection === "undefined";
// No video in HLS stream and no video in WebRTC stream
return true;
Comment on lines -133 to +159
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bramkragten I think this change might have caused a regression in the HLS player. When the _setRetryableError is called from the ha-hls-player with:

  private _setRetryableError(errorMessage: string) {
    this._error = errorMessage;
    this._errorIsFatal = false;
    fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
  }

This code path (or the updated path from #22674) will cause the camera stream to switch to the mjpeg stream without retrying. I have a situation where an intermittent bufferAppendError being raised from an HLS stream, and instead of retrying it immediately falls back to the mjpeg stream. Previously I think that fallback would only have happened if fetchStreamUrl failed.

What do you think?

}
// Server side stream component required for HLS
return !isComponentLoaded(this.hass!, "stream");
return false;
}

private async _getPosterUrl(): Promise<void> {
Expand All @@ -151,20 +175,28 @@ export class HaCameraStream extends LitElement {
}
}

private async _getStreamUrl(): Promise<void> {
try {
const { url } = await fetchStreamUrl(
this.hass!,
this.stateObj!.entity_id
);
private _handleHlsStreams(ev: CustomEvent) {
this._hlsStreams = ev.detail;
this._pickStreamType();
}

this._url = url;
} catch (err: any) {
// Fails if we were unable to get a stream
// eslint-disable-next-line
console.error(err);
private _handleWebRtcStreams(ev: CustomEvent) {
this._webRtcStreams = ev.detail;
this._pickStreamType();
}

this._forceMJPEG = this.stateObj!.entity_id;
private _pickStreamType() {
if (!this._hlsStreams || !this._webRtcStreams) {
return;
}
if (
this._hlsStreams.hasVideo &&
this._hlsStreams.hasAudio &&
!this._webRtcStreams.hasAudio
) {
this._streamType = STREAM_TYPE_HLS;
} else if (this._webRtcStreams.hasVideo) {
this._streamType = STREAM_TYPE_WEB_RTC;
}
}

Expand All @@ -178,6 +210,10 @@ export class HaCameraStream extends LitElement {
img {
width: 100%;
}

.hidden {
display: none;
}
`;
}
}
Expand All @@ -186,4 +222,12 @@ declare global {
interface HTMLElementTagNameMap {
"ha-camera-stream": HaCameraStream;
}
interface HASSDomEvents {
load: undefined;
streams: {
hasAudio: boolean;
hasVideo: boolean;
codecs?: string[];
};
}
}
71 changes: 59 additions & 12 deletions src/components/ha-hls-player.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { fireEvent } from "../common/dom/fire_event";
import { nextRender } from "../common/util/render-status";
import type { HomeAssistant } from "../types";
import "./ha-alert";
import { fetchStreamUrl } from "../data/camera";
import { isComponentLoaded } from "../common/config/is_component_loaded";

type HlsLite = Omit<
HlsType,
Expand All @@ -22,9 +24,9 @@ type HlsLite = Omit<
class HaHLSPlayer extends LitElement {
@property({ attribute: false }) public hass!: HomeAssistant;

@property() public url!: string;
@property() public entityid?: string;

@property() public posterUrl!: string;
@property({ attribute: "poster-url" }) public posterUrl?: string;

@property({ type: Boolean, attribute: "controls" })
public controls = false;
Expand All @@ -48,6 +50,8 @@ class HaHLSPlayer extends LitElement {

@state() private _errorIsFatal = false;

@state() private _url!: string;

private _hlsPolyfillInstance?: HlsLite;

private _exoPlayer = false;
Expand Down Expand Up @@ -95,19 +99,44 @@ class HaHLSPlayer extends LitElement {
protected updated(changedProps: PropertyValues) {
super.updated(changedProps);

const urlChanged = changedProps.has("url");
const entityChanged = changedProps.has("entityid");

if (!urlChanged) {
if (!entityChanged) {
return;
}
this._getStreamUrl();
}

private async _getStreamUrl(): Promise<void> {
this._cleanUp();
this._resetError();
this._startHls();

if (!isComponentLoaded(this.hass!, "stream")) {
this._setFatalError("Streaming component is not loaded.");
return;
}

if (!this.entityid) {
return;
}
try {
const { url } = await fetchStreamUrl(this.hass!, this.entityid);

this._url = url;
this._cleanUp();
this._resetError();
this._startHls();
} catch (err: any) {
// Fails if we were unable to get a stream
// eslint-disable-next-line
console.error(err);

fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
}
}

private async _startHls(): Promise<void> {
const masterPlaylistPromise = fetch(this.url);
const masterPlaylistPromise = fetch(this._url);

const Hls: typeof HlsType = (await import("hls.js/dist/hls.light.mjs"))
.default;
Expand Down Expand Up @@ -138,10 +167,10 @@ class HaHLSPlayer extends LitElement {
return;
}

// Parse playlist assuming it is a master playlist. Match group 1 is whether hevc, match group 2 is regular playlist url
// Parse playlist assuming it is a master playlist. Match group 1 and 2 are codec, match group 3 is regular playlist url
// See https://tools.ietf.org/html/rfc8216 for HLS spec details
const playlistRegexp =
/#EXT-X-STREAM-INF:.*?(?:CODECS=".*?(hev1|hvc1)?\..*?".*?)?(?:\n|\r\n)(.+)/g;
/#EXT-X-STREAM-INF:.*?(?:CODECS=".*?([^.]*)?\..*?,([^.]*)?\..*?".*?)?(?:\n|\r\n)(.+)/g;
const match = playlistRegexp.exec(masterPlaylist);
const matchTwice = playlistRegexp.exec(masterPlaylist);

Expand All @@ -150,13 +179,20 @@ class HaHLSPlayer extends LitElement {
let playlist_url: string;
if (match !== null && matchTwice === null) {
// Only send the regular playlist url if we match exactly once
playlist_url = new URL(match[2], this.url).href;
playlist_url = new URL(match[3], this._url).href;
} else {
playlist_url = this.url;
playlist_url = this._url;
}

const codecs = match ? `${match[1]},${match[2]}` : undefined;

this._reportStreams(codecs);

// If codec is HEVC and ExoPlayer is supported, use ExoPlayer.
if (useExoPlayer && match !== null && match[1] !== undefined) {
if (
useExoPlayer &&
(codecs?.includes("hevc") || codecs?.includes("hev1"))
) {
this._renderHLSExoPlayer(playlist_url);
} else if (Hls.isSupported()) {
this._renderHLSPolyfill(this._videoEl, Hls, playlist_url);
Expand Down Expand Up @@ -313,15 +349,26 @@ class HaHLSPlayer extends LitElement {
private _setFatalError(errorMessage: string) {
this._error = errorMessage;
this._errorIsFatal = true;
fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
}

private _setRetryableError(errorMessage: string) {
this._error = errorMessage;
this._errorIsFatal = false;
fireEvent(this, "streams", { hasAudio: false, hasVideo: false });
}

private _reportStreams(codecs?: string) {
const codec = codecs?.split(",");
fireEvent(this, "streams", {
hasAudio: codec?.includes("mp4a") ?? false,
hasVideo: codec?.includes("mp4a")
? codec?.length > 1
: Boolean(codec?.length),
});
}

private _loadedData() {
// @ts-ignore
fireEvent(this, "load");
}

Expand Down
Loading
Loading