feat: Add experimental rewrite of go2rtc live provider (MSE/WebRTC/MP4/MJPEG) (#2580)

- Closes #2556
- Closes #2450

**Key intended features:**
 - go2rtc compatible
- 100% test coverage to significantly improve ability to test, maintain
and work around browser weirdnesses (e.g. Safari).
 - Written from the ground up in the style of the rest of the project.

**To use:**
 - Change `live_provider` from `go2rtc` to `go2rtc-experimental`.
This commit is contained in:
Dermot Duffy
2026-07-14 14:22:41 -07:00
committed by GitHub
parent 5662e48c22
commit c02c692f68
125 changed files with 9608 additions and 807 deletions
+8 -4
View File
@@ -18,7 +18,10 @@ import type { HassStateDifference, HomeAssistant } from '../ha/types';
import { localize } from '../localize/localize';
import type { CapabilitiesRaw, CapabilityKey, Endpoint } from '../types';
import { arrayify } from '../utils/basic';
import { liveProviderSupports2WayAudio } from '../utils/live-provider';
import {
isGo2RTCLiveProvider,
liveProviderSupports2WayAudio,
} from '../utils/live-provider';
import { Capabilities } from './capabilities';
import type { CameraManagerEngine } from './engine';
import { CameraNoIDError } from './error';
@@ -345,9 +348,10 @@ export class Camera {
...resolveProxyConfig(this._config.proxy),
live:
this._config.proxy.live === 'auto'
? // Live is proxied if the live provider is go2rtc and if a go2rtc
// URL is manually set.
this._config.live_provider === 'go2rtc' && !!this._config.go2rtc?.url
? // Live is proxied if the live provider streams from go2rtc and a
// go2rtc URL is manually set.
isGo2RTCLiveProvider(this._config.live_provider) &&
!!this._config.go2rtc?.url
: this._config.proxy.live,
media: this._config.proxy.media === 'auto' ? false : this._config.proxy.media,
};
+4 -1
View File
@@ -6,6 +6,9 @@ export class PauseAction extends AdvancedCameraCardAction<GeneralActionConfig> {
public async execute(api: CardActionsAPI): Promise<void> {
await super.execute(api);
await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.pause();
await api
.getMediaLoadedInfoManager()
.get()
?.mediaPlayerController?.playback?.pause();
}
}
+1 -1
View File
@@ -6,6 +6,6 @@ export class PlayAction extends AdvancedCameraCardAction<GeneralActionConfig> {
public async execute(api: CardActionsAPI): Promise<void> {
await super.execute(api);
await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.play();
await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.playback?.play();
}
}
+5 -4
View File
@@ -1,6 +1,7 @@
import { createNotificationFromText } from '../../components-lib/notification/factory';
import type { ConditionStateChange } from '../../condition-trigger/conditions/types';
import { localize } from '../../localize/localize';
import { Generation } from '../../utils/concurrency/generation';
import { Timer } from '../../utils/timer';
import { getStreamCameraID } from '../../view/substream';
import type { View } from '../../view/view';
@@ -22,7 +23,7 @@ export class CallManager {
// resumed tail leaks audio onto the shared lock from an instance the user
// can no longer see or control, and may install state into a fresh
// lifecycle from a request that belongs to the previous one.
private _initEpoch = 0;
private _initGeneration = new Generation();
constructor(api: CardCallAPI) {
this._api = api;
@@ -103,7 +104,7 @@ export class CallManager {
return false;
}
const initEpoch = this._initEpoch;
const initGeneration = this._initGeneration.current();
const microphoneConnected = await this._connectMicrophone();
// If the init/uninit lifecycle advanced while the microphone connect was
// in flight, this request belongs to a previous lifecycle -- the view we
@@ -111,7 +112,7 @@ export class CallManager {
// clean teardown path for state we'd install here. Bail before touching
// `_call`, the ringtone lock, or surfacing a notification onto a torn-down
// NotificationManager.
if (initEpoch !== this._initEpoch) {
if (!this._initGeneration.isCurrent(initGeneration)) {
return false;
}
if (!microphoneConnected) {
@@ -241,7 +242,7 @@ export class CallManager {
//
// Safe to re-initialize afterwards via `initialize()`.
public uninitialize(): void {
this._initEpoch++;
this._initGeneration.invalidate();
this._ringtone.stop();
this._unansweredTimer.stop();
if (this._call) {
@@ -21,7 +21,10 @@ export type MediaUnavailableIssueReason =
| 'entity_unavailable'
| 'not_loading'
| 'playback_error'
| 'stalled';
| 'server_error'
| 'stalled'
| 'two_way_audio_error'
| 'unsupported';
declare module 'issue' {
interface IssueTriggerContext {
@@ -50,10 +53,22 @@ export const MEDIA_UNAVAILABLE_REASONS: Record<
localizationKey: 'issues.media_unavailable.reasons.playback_error',
icon: 'mdi:alert-circle',
},
server_error: {
localizationKey: 'issues.media_unavailable.reasons.server_error',
icon: 'mdi:server-network-off',
},
stalled: {
localizationKey: 'issues.media_unavailable.reasons.stalled',
icon: 'mdi:motion-pause',
},
two_way_audio_error: {
localizationKey: 'issues.media_unavailable.reasons.two_way_audio_error',
icon: 'mdi:microphone-off',
},
unsupported: {
localizationKey: 'issues.media_unavailable.reasons.unsupported',
icon: 'mdi:video-off-outline',
},
};
export class MediaUnavailableIssue implements Issue {
@@ -1,3 +1,4 @@
import type { MediaUnavailableIssueReason } from '../../../../card-controller/issues/issues/media-unavailable';
import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller';
const LIVE_ERROR_EVENT = 'advanced-camera-card:live:error';
@@ -38,12 +39,19 @@ export class ProviderErrorDetector implements LivenessDetector {
return this._verdict;
}
private _handler = (ev: Event): void => {
private _handler = (
ev: CustomEvent<MediaUnavailableIssueReason | undefined>,
): void => {
ev.stopPropagation();
if (this._verdict.state !== 'not_live') {
// Authoritative: an explicit provider error overrides even direct frame
// evidence. No placeholder -- the provider renders its own error.
this._verdict = { state: 'not_live', authority: 'hard', reason: 'playback_error' };
// evidence. No placeholder -- the provider renders its own error. The
// provider may name the cause; otherwise it is a generic playback error.
this._verdict = {
state: 'not_live',
authority: 'hard',
reason: ev.detail ?? 'playback_error',
};
this._onChange();
}
};
@@ -41,10 +41,19 @@ export type LivenessVerdict =
renderPlaceholder?: boolean;
};
// The reconnecting placeholder the wrapper renders in place of a frozen
// provider, carrying the cause so it can show a cause-specific message.
interface LivenessPlaceholder {
// The wrapper-facing projection of the internal LivenessVerdict: it keeps the
// detector-internal `authority` from leaking out and collapses `unknown` into
// "no failure", so a consumer sees only a confirmed failure (with its cause) or
// nothing. Modeled as the failure rather than a `live` flag because a stream
// that is merely still connecting is not a failure yet is not playing either, so
// a positive `live` boolean would misleadingly read as "media is playing".
interface StreamFailure {
reason: MediaUnavailableIssueReason;
// Whether the wrapper should replace the provider with a reconnecting
// placeholder (a silent freeze, e.g. an unavailable camera). False when the
// provider renders its own error and should stay mounted.
renderPlaceholder: boolean;
}
export interface LivenessDetector {
@@ -116,17 +125,15 @@ export class StreamLivenessController implements ReactiveController {
}
public isLive(): boolean {
return this._getVerdict().state !== 'not_live';
return !this.getFailure();
}
// The reconnecting placeholder to render in place of the (frozen) provider,
// carrying the cause so the wrapper can show a cause-specific message. Null
// when the provider should stay mounted (live, or a provider error that
// renders its own error).
public getPlaceholder(): LivenessPlaceholder | null {
// The stream's confirmed liveness failure, or null when there is none (the
// stream is live, or still connecting).
public getFailure(): StreamFailure | null {
const verdict = this._getVerdict();
return verdict.state === 'not_live' && verdict.renderPlaceholder
? { reason: verdict.reason }
return verdict.state === 'not_live'
? { reason: verdict.reason, renderPlaceholder: !!verdict.renderPlaceholder }
: null;
}
@@ -0,0 +1,13 @@
# go2rtc-experimental provider
![Architecture of the go2rtc-experimental provider](architecture.drawio.svg)
The SVG is also its own editable source: it embeds the draw.io diagram, so open
`architecture.drawio.svg` directly in draw.io / diagrams.net to change it.
## Maintaining the diagram
`architecture.drawio.svg` is a dual file: the **rendered SVG** (shown above /
in GitHub) plus the **editable mxGraph XML** embedded in its `content="..."`
attribute. Keep them in sync -- never commit an SVG whose embedded XML changed
but whose rendered picture did not, or the image above goes stale.
@@ -0,0 +1,148 @@
// Wraps the browser's two incompatible MediaSource "flavors" behind one
// interface, so mse.ts stays flavor-agnostic (and mockable in tests):
// - classic MediaSource (Chrome/Firefox/desktop): attach via an object URL
// on `video.src`.
// - ManagedMediaSource (Safari/iOS 17+; iOS had no MediaSource at all before
// then): attach via `video.srcObject`, and needs `disableRemotePlayback`.
import type { UnsubscribeCallback } from '../../../../../types';
declare global {
interface Window {
// Safari 17+ managed variant of MediaSource; API-compatible for the
// subset used here.
ManagedMediaSource?: typeof MediaSource;
}
}
// The unified MediaSource surface mse.ts programs against; the flavor-specific
// attach/detach lives in the implementations below.
export interface MediaSourceInterface {
attach(video: HTMLVideoElement): void;
detach(video: HTMLVideoElement): void;
// Fires once the source is ready to accept SourceBuffers; nothing can be
// appended before it.
subscribeToSourceOpen(callback: () => void): UnsubscribeCallback;
// Creates the SourceBuffer that media chunks are appended to, configured for
// the given codec MIME string.
addSourceBuffer(codecs: string): SourceBuffer;
// Declares the seekable live window to the browser, which cannot infer it for
// an open-ended live source.
setLiveSeekableRange(startSeconds: number, endSeconds: number): void;
// Whether the source is still attached and accepting SourceBuffer operations.
isOpen(): boolean;
// Whether a codec MIME string is playable.
isTypeSupported(mimeType: string): boolean;
}
// Creates a wrapped MediaSource, or null when the browser supports no variant.
export type MediaSourceFactory = () => MediaSourceInterface | null;
abstract class MediaSourceInstanceBase implements MediaSourceInterface {
protected _mediaSource: MediaSource;
constructor(mediaSource: MediaSource) {
this._mediaSource = mediaSource;
}
public abstract attach(video: HTMLVideoElement): void;
public abstract detach(video: HTMLVideoElement): void;
public abstract isTypeSupported(mimeType: string): boolean;
public subscribeToSourceOpen(callback: () => void): UnsubscribeCallback {
this._mediaSource.addEventListener('sourceopen', callback);
return () => this._mediaSource.removeEventListener('sourceopen', callback);
}
public addSourceBuffer(codecs: string): SourceBuffer {
return this._mediaSource.addSourceBuffer(codecs);
}
public setLiveSeekableRange(startSeconds: number, endSeconds: number): void {
this._mediaSource.setLiveSeekableRange(startSeconds, endSeconds);
}
public isOpen(): boolean {
return this._mediaSource.readyState === 'open';
}
}
class ManagedMediaSourceInstance extends MediaSourceInstanceBase {
private _mediaSourceConstructor: typeof MediaSource;
constructor(mediaSourceConstructor: typeof MediaSource) {
super(new mediaSourceConstructor());
this._mediaSourceConstructor = mediaSourceConstructor;
}
public attach(video: HTMLVideoElement): void {
// ManagedMediaSource does not deliver data while remote playback (e.g.
// AirPlay) is possible.
video.disableRemotePlayback = true;
video.srcObject = this._mediaSource;
}
public detach(video: HTMLVideoElement): void {
video.srcObject = null;
}
public isTypeSupported(mimeType: string): boolean {
return this._mediaSourceConstructor.isTypeSupported(mimeType);
}
}
class ClassicMediaSourceInstance extends MediaSourceInstanceBase {
private _objectURL: string | null = null;
constructor() {
super(new MediaSource());
}
public attach(video: HTMLVideoElement): void {
this._objectURL = URL.createObjectURL(this._mediaSource);
// The object URL is only needed until the browser has opened the media
// source.
this._mediaSource.addEventListener('sourceopen', () => this._revokeObjectURL(), {
once: true,
});
video.src = this._objectURL;
video.srcObject = null;
}
public detach(video: HTMLVideoElement): void {
video.src = '';
this._revokeObjectURL();
}
public isTypeSupported(mimeType: string): boolean {
return MediaSource.isTypeSupported(mimeType);
}
private _revokeObjectURL(): void {
if (this._objectURL) {
URL.revokeObjectURL(this._objectURL);
this._objectURL = null;
}
}
}
export const createBrowserMediaSource: MediaSourceFactory = () => {
// Safari exposes both flavors, so check Managed first and prefer it there: it
// lets the browser throttle buffering to save battery and memory and hands
// off cleanly to AirPlay, which classic MediaSource does not. Classic is the
// fallback for other browsers.
const managedMediaSource = window.ManagedMediaSource;
if (managedMediaSource) {
return new ManagedMediaSourceInstance(managedMediaSource);
}
if ('MediaSource' in window) {
return new ClassicMediaSourceInstance();
}
return null;
};
@@ -0,0 +1,14 @@
export type PeerConnectionFactory = (config: RTCConfiguration) => RTCPeerConnection;
export const GO2RTC_PEER_CONNECTION_CONFIG: RTCConfiguration = {
bundlePolicy: 'max-bundle',
iceServers: [
{
// Two public STUN servers so connectivity survives one being unreachable.
urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'],
},
],
};
export const createBrowserPeerConnection: PeerConnectionFactory = (config) =>
new RTCPeerConnection(config);
File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 924 KiB

@@ -0,0 +1,144 @@
import type { LitElement } from 'lit';
import { Generation } from '../../../../utils/concurrency/generation';
import { LatestValueRunner } from '../../../../utils/concurrency/latest-value-runner';
import { ImageMediaPlayerController } from '../../../media-player/image';
import { OffscreenImage } from './offscreen-image';
import type { ImageSurface } from './session-controller';
// Liveness: while frames are expected, a gap beyond the window is a stall.
// Omitted -> the surface reports no liveness. `stallWindowSeconds` defaults to
// the standard frame-stall window.
interface ImageSurfaceLivenessOptions {
isFrameExpected: () => boolean;
stallWindowSeconds?: number;
}
interface ImageSurfaceOptions {
livenessOptions?: ImageSurfaceLivenessOptions;
// Factory for the detached image loader used to decode a frame.
createImage?: () => HTMLImageElement;
}
// Presents a go2rtc MJPEG/MP4 stream as a sequence of frames on an <img>: each
// frame is decoded off-DOM, then shown via an object URL (the previous one
// revoked), and an ImageMediaPlayerController exposes it to the card as a media
// player. The renderer owns the <img> element; this owns what is shown on it.
export class ImageSurfaceController implements ImageSurface {
private _getImageCallback: () => HTMLImageElement | null;
private _mediaPlayerController: ImageMediaPlayerController;
// An off-screen <img> used to decode each frame before it is shown on the
// visible element (reused: only one frame decodes at a time).
private _decoder: OffscreenImage;
// Current frame's object URL.
private _currentObjectURL: string | null = null;
// Invalidated by reset() so a frame whose decode was still in flight when the
// surface was switched or torn down does not paint a stale frame afterwards.
private _generation = new Generation();
// Frames can arrive faster than the browser decodes one. Present at most one
// at a time and keep only the newest frame while busy, so the display
// converges on the latest without unbounded decode work.
private _frameRunner = new LatestValueRunner<Blob>((frame) =>
this._presentFrame(frame),
);
constructor(
host: LitElement,
getImageCallback: () => HTMLImageElement | null,
options?: ImageSurfaceOptions,
) {
this._getImageCallback = getImageCallback;
this._decoder = new OffscreenImage(options?.createImage);
this._mediaPlayerController = new ImageMediaPlayerController(
host,
getImageCallback,
{
livenessOptions: options?.livenessOptions,
},
);
}
public getElement(): HTMLImageElement | null {
return this._getImageCallback();
}
public getMediaPlayer(): ImageMediaPlayerController {
return this._mediaPlayerController;
}
// Show a new frame. A newer frame supersedes any still waiting to show; the
// returned promise resolves once *a* frame has been presented.
public showFrame(blob: Blob): Promise<void> {
return this._frameRunner.submit(blob);
}
private async _presentFrame(blob: Blob): Promise<void> {
const image = this._getImageCallback();
// Drop frames for a detached element (e.g. a surface superseded
// mid-teardown) so a retired surface never paints.
if (!image || !image.isConnected) {
return;
}
const generation = this._generation.current();
const url = URL.createObjectURL(blob);
// Decode the frame on the off-screen loader before showing it. Assigning an
// undecoded object URL to the visible <img> makes WebKit repaint the
// element empty until the decode completes, flashing the media background
// between frames; decoding first lets the swap paint from cache (as it's
// the same url) with no empty state.
const decoder = this._decoder.get();
decoder.src = url;
try {
await decoder.decode();
} catch {
// An undecodable frame is not fatal: drop it and keep the current one.
URL.revokeObjectURL(url);
return;
}
// The surface may have been switched (reset) or detached during the decode;
// painting now would show a stale frame on a retired surface.
if (!this._generation.isCurrent(generation) || !image.isConnected) {
URL.revokeObjectURL(url);
return;
}
image.src = url;
const previousURL = this._currentObjectURL;
this._currentObjectURL = url;
if (previousURL) {
URL.revokeObjectURL(previousURL);
}
}
// Drop the current frame and its object URL (e.g. on a surface switch or
// disconnect).
public reset(): void {
// Invalidate any frame still decoding so it cannot paint onto a surface
// that has since been switched or disconnected.
this._generation.invalidate();
// Drop any frame still waiting to show, for the same reason.
this._frameRunner.clear();
if (this._currentObjectURL) {
URL.revokeObjectURL(this._currentObjectURL);
this._currentObjectURL = null;
}
this._decoder.clear();
const image = this._getImageCallback();
if (image) {
image.removeAttribute('src');
}
}
}
@@ -0,0 +1,26 @@
type ImageElementFactory = () => HTMLImageElement;
// A held, injectable off-screen <img> for decoding a frame separate from the
// visible display element. Created lazily on first `get()` and reused until
// `clear()` detaches its source and forgets it.
export class OffscreenImage {
private _create: ImageElementFactory;
private _image: HTMLImageElement | null = null;
constructor(create?: ImageElementFactory) {
this._create = create ?? (() => new Image());
}
// Return the held image, creating it on first use.
public get(): HTMLImageElement {
return (this._image ??= this._create());
}
// Detach any source and forget the image. Safe to call when none is held.
public clear(): void {
if (this._image) {
this._image.removeAttribute('src');
this._image = null;
}
}
}
@@ -0,0 +1,28 @@
export type VideoElementFactory = () => HTMLVideoElement;
// A held, injectable off-screen <video> for decoding a stream separate from the
// real display element. Created lazily on first `get()` and reused until
// `clear()` detaches its media and forgets it.
export class OffscreenVideo {
private _create: VideoElementFactory;
private _video: HTMLVideoElement | null = null;
constructor(create?: VideoElementFactory) {
this._create = create ?? (() => document.createElement('video'));
}
// Return the held video, creating it on first use.
public get(): HTMLVideoElement {
return (this._video ??= this._create());
}
// Detach any media (whether attached via `src` or `srcObject`) and forget the
// video. Safe to call when none is held.
public clear(): void {
if (this._video) {
this._video.removeAttribute('src');
this._video.srcObject = null;
this._video = null;
}
}
}
@@ -0,0 +1,721 @@
import { isEqual } from 'lodash-es';
import { GO2RTC_MODES, type Go2RTCMode } from '../../../../config/schema/cameras';
import type { CardWideConfig } from '../../../../config/schema/types';
import type {
MediaPlayerController,
UntargetedMediaLoadedInfo,
} from '../../../../types';
import {
addAudioTracksMuteStateListener,
type AudioTracksMuteStateCleanup,
} from '../../../../utils/audio';
import {
hideMediaControlsTemporarily,
MEDIA_LOAD_CONTROLS_HIDE_SECONDS,
} from '../../../../utils/controls';
import { log } from '../../../../utils/debug';
import { createMediaLoadedInfo } from '../../../../utils/media-info';
import { RetryTimer } from '../../../../utils/retry-timer';
import { convertToWebSocketURL } from '../../../../utils/websocket-url';
import type { MediaSourceFactory } from './adapters/media-source';
import type { PeerConnectionFactory } from './adapters/peer-connection';
import { OffscreenVideo, type VideoElementFactory } from './offscreen-video';
import { SignalingChannel, type WebSocketFactory } from './signaling';
import {
createBinarySource,
createWebRTCSource,
type BinarySource,
type BinarySourceFactory,
type WebRTCSourceFactory,
} from './sources/factory';
import type { MediaStreamFactory, WebRTCStreamSource } from './sources/webrtc';
import type {
Lane,
StreamSource,
StreamSourceContext,
StreamSourceFailureReason,
SurfaceKind,
VideoStreamTarget,
} from './types';
import { getPreferredSource } from './utils/source-priority';
// Preference order for modes that stream binary media over the WebSocket; the
// protocol permits only one such mode per connection, tried in this order with
// fallback (real video first with MSE, then the MP4/MJPEG image fallbacks).
// WebRTC is not here: it carries no WebSocket binary and runs in parallel with
// the chosen binary mode. The configured modes select membership, not order.
const BINARY_MODE_PRECEDENCE: readonly Go2RTCMode[] = ['mse', 'mp4', 'mjpeg'];
// A dropped connection is retried in place a few times, quickly, so a transient
// blip (e.g. a brief network drop on an MSE stream) heals with no full player
// remount (e.g. without the card IssueManager). After a short run of failures
// the stream is treated as down: the failure is reported upward, handing
// recovery to the card's managed retry (which shows a reconnecting indicator,
// backs off, and can give up) instead of looping here silently. The attempt cap
// keeps a persistently-failing stream from hammering the server.
const RECONNECT_INTERVAL_SECONDS = 2;
const RECONNECT_MAX_ATTEMPTS = 3;
// The two render surfaces the session commits onto. The component owns the
// elements and the media-player controllers; the session drives which one is
// live and reports it back via surfaceCommittedCallback.
export interface VideoSurface {
getElement(): HTMLVideoElement | null;
getMediaPlayer(): MediaPlayerController;
}
export interface ImageSurface {
getElement(): HTMLImageElement | null;
getMediaPlayer(): MediaPlayerController;
// Frames are handed over as Blobs through this callback rather than the
// session setting `img.src` itself. The implementation (ImageSurfaceController)
// creates each frame's object URL and revokes the previous one; the session
// only ever holds Blobs, so it cannot leak URLs. The returned promise resolves
// once the frame has decoded (real dimensions are then available).
showFrame(frame: Blob): Promise<void>;
reset(): void;
}
export interface SessionSurfaces {
video: VideoSurface;
image: ImageSurface;
}
interface Go2RTCSessionCallbacks {
getControls: () => boolean;
getCardWideConfig: () => CardWideConfig | null;
mediaLoadedCallback: (info: UntargetedMediaLoadedInfo) => void;
// The lane that just committed is live on this surface (the component shows
// it and hides the other).
surfaceCommittedCallback: (surface: SurfaceKind) => void;
// The session has exhausted its own quick reconnects and cannot recover the
// stream; a higher level should take over (e.g. the card's media-load retry).
// The reason is the most recent source failure, or null when there is none
// (e.g. the socket dropped with no source having reported a cause).
errorCallback: (reason: StreamSourceFailureReason | null) => void;
}
// Injectable platform and factory seams for tests. Every field defaults to
// the real browser implementation; production passes none of them.
interface Go2RTCSessionOptions {
createBinarySource?: BinarySourceFactory;
createWebRTCSource?: WebRTCSourceFactory;
createWebSocket?: WebSocketFactory;
createPeerConnection?: PeerConnectionFactory;
createMediaStream?: MediaStreamFactory;
createVideoElement?: VideoElementFactory;
createMediaSource?: MediaSourceFactory;
userAgent?: string;
}
// One WebSocket connection attempt: its channel and URL, plus a reference to
// the render surfaces to draw onto (the surfaces outlive any single attempt).
// Handlers capture this rather than reading session fields so a stale
// connection's events cannot act on a newer connection.
interface ConnectionContext {
channel: SignalingChannel;
url: string;
surfaces: SessionSurfaces;
}
// Coordinates one go2rtc streaming session. A "binary" lane (MSE, MP4/MJPEG)
// plays on the real video with sequential fallback; a parallel WebRTC lane
// decodes into an off-screen video element and, if its stream scores
// higher, takes over the real video. The session owns the signaling channel, the
// bounded fast reconnects, and reporting loaded media.
export class Go2RTCSessionController {
private _callbacks: Go2RTCSessionCallbacks;
private _options: Go2RTCSessionOptions | null;
private _channel: SignalingChannel | null = null;
private _url: string | null = null;
private _surfaces: SessionSurfaces | null = null;
private _modes: readonly Go2RTCMode[] = GO2RTC_MODES;
private _microphoneStream: MediaStream | null = null;
// Binary lane: the active source paired with the surface it renders on (mse
// -> video, MP4/MJPEG -> image). Kept as one unit because the factory returns
// them together and they are read together when the source commits. Null when
// no binary source is running.
private _binary: BinarySource | null = null;
// Queue of fallback modes not yet tried on this connection, in precedence
// order. Starting a binary source consumes the head; teardown empties the
// queue so no further fallback is attempted.
private _binaryModes: Go2RTCMode[] = [];
// WebRTC lane.
private _webRTCSource: WebRTCStreamSource | null = null;
// Holds the off-screen video element the WebRTC stream decodes into while the
// WebRTC lane races a binary lane.
private _offscreenVideo: OffscreenVideo;
// The committed lane: the one whose media is live (on the video or image
// surface) and has reported as loaded; null before anything loads or once the
// lanes are torn down. Failure handling reads it to distinguish a committed
// (live) stream dying from a losing racer being dropped.
private _committedLane: Lane | null = null;
// The surface currently showing committed media; null before anything commits
// or once torn down. On a surface switch (e.g. MSE falls back to MP4, or a
// WebRTC win over an MJPEG image) the outgoing surface is reset.
private _committedSurface: SurfaceKind | null = null;
// The source whose media is committed; null before anything commits or once
// torn down. A source can re-report loaded (e.g. a <video> re-fires
// loadeddata on a mid-stream resolution change), so this distinguishes a
// fresh commit from a repeated load.
private _committedSource: StreamSource | null = null;
// Unsubscriber for the listener on the committed WebRTC stream's audio
// tracks. Should a WebRTC track's mute state change after load, the listener
// sends the card a fresh media-loaded info so its audio capabilities reflect
// the stream's current audio, not the state at load time. Null when no WebRTC
// stream is committed; invoked and cleared on teardown and before re-arming
// so a retired stream's listener can never fire.
private _audioTracksMuteStateUnsubscribeCallback: AudioTracksMuteStateCleanup = null;
// The most recent source failure on this connection, handed to the error
// callback when the session finally gives up so the card can name the cause.
// Null before any failure and after a healthy commit.
private _lastFailureReason: StreamSourceFailureReason | null = null;
private _retryTimer = new RetryTimer(RECONNECT_INTERVAL_SECONDS);
constructor(callbacks: Go2RTCSessionCallbacks, options?: Go2RTCSessionOptions) {
this._callbacks = callbacks;
this._options = options ?? null;
this._offscreenVideo = new OffscreenVideo(this._options?.createVideoElement);
}
// ===========================================================================
// Public API.
// ===========================================================================
// The `surfaces` object identifies the session target: the same object means
// "keep the established session" (so callers may invoke this on every render),
// a new object means "reset and reconnect". Callers must therefore hold one
// stable surfaces object for as long as the target is unchanged, and hand over
// a fresh one (or `reset()` first) if they ever remount the underlying
// elements.
public connect(url: string, surfaces: SessionSurfaces, modes?: Go2RTCMode[]): void {
const normalizedModes: readonly Go2RTCMode[] = modes?.length ? modes : GO2RTC_MODES;
if (
this._url === url &&
this._surfaces === surfaces &&
isEqual(this._modes, normalizedModes)
) {
return;
}
this.reset();
this._url = url;
this._surfaces = surfaces;
this._modes = normalizedModes;
this._connectChannel(url, surfaces);
}
public reset(): void {
this._retryTimer.reset();
this._lastFailureReason = null;
this._teardownLanes();
this._channel?.close();
this._channel = null;
const video = this._surfaces?.video.getElement();
if (video) {
video.srcObject = null;
video.src = '';
}
this._surfaces?.image.reset();
this._url = null;
this._surfaces = null;
}
public setMicrophoneStream(stream: MediaStream | null): void {
this._microphoneStream = stream;
this._webRTCSource?.setMicrophoneStream(stream).catch(() => {});
}
// ===========================================================================
// Session teardown.
// ===========================================================================
private _teardownLanes(): void {
this._teardownBinaryLane();
this._teardownWebRTCLane();
this._committedLane = null;
this._committedSurface = null;
this._committedSource = null;
}
// ===========================================================================
// Signaling channel lifecycle.
// ===========================================================================
private _connectChannel(url: string, surfaces: SessionSurfaces): void {
// The open/close callbacks reference `channel`, the very constant this
// `new` expression is being assigned to. That is safe because the callbacks
// fire on asynchronous WebSocket events, which cannot happen before the
// assignment completes.
const channel: SignalingChannel = new SignalingChannel(
convertToWebSocketURL(url),
{
openCallback: () => this._handleOpen({ channel, url, surfaces }),
disconnectCallback: () => this._handleClose({ channel, url, surfaces }),
},
{ createWebSocket: this._options?.createWebSocket },
);
this._channel = channel;
channel.connect();
}
private _handleOpen(context: ConnectionContext): void {
this._binaryModes = BINARY_MODE_PRECEDENCE.filter((mode) =>
this._modes.includes(mode),
);
// While a binary lane races, WebRTC decodes into a separate off-screen video
// element (see OffscreenVideo) where its stream can be evaluated without
// contending for the real <video> that an MSE binary lane plays on. With no
// binary lane the real video element is free, so WebRTC attaches to it
// directly; that also avoids decoding into an element outside the document,
// which has been implicated in Firefox WebRTC failures. See
// https://github.com/dermotduffy/advanced-camera-card/issues/2222
//
// Every configured mode is a binary mode or webrtc, and empty modes default
// to all, so at least one lane always starts.
//
// Start WebRTC before the binary lane: starting a binary source can fail
// synchronously (e.g. an unsupported codec drains the whole mode queue),
// which would trigger a premature reconnect while no lane is live yet and
// leave a stale WebRTC source running on a closed channel. Starting WebRTC
// first means that reconnect check always sees a live lane.
const hasBinaryModes = this._binaryModes.length > 0;
if (this._modes.includes('webrtc')) {
this._startWebRTCSource(context, hasBinaryModes);
}
if (hasBinaryModes) {
this._startNextBinarySource(context);
}
}
private _handleClose(context: ConnectionContext): void {
this._teardownLanes();
this._channel = null;
this._reconnectOrEscalateError(context);
}
// ===========================================================================
// Binary lane.
// ===========================================================================
private _startNextBinarySource(context: ConnectionContext): void {
const mode = this._binaryModes.shift();
if (!mode) {
this._maybeReconnectIfLanesDead(context);
return;
}
// No element to render onto (the surface was detached); abandon the lane.
const video = context.surfaces.video.getElement();
if (!video) {
return;
}
// The callbacks capture the source's own identity so a retired source
// (stopped, replaced or reset) cannot act on the session. The variable is
// declared before the factory call so callbacks fired during construction
// see null and are ignored.
let source: StreamSource | null = null;
const binarySource = (this._options?.createBinarySource ?? createBinarySource)(
mode,
{
video: { kind: 'video', video },
image: {
kind: 'image',
showFrame: (frame) => context.surfaces.image.showFrame(frame),
},
},
context.channel,
{
loadedCallback: () => {
if (source) {
this._handleBinaryLoaded(context, source);
}
},
failedCallback: (reason: StreamSourceFailureReason) => {
if (source) {
this._lastFailureReason = reason;
this._logSourceFailure('binary', reason, mode);
this._handleBinaryFailed(context, source);
}
},
},
{
createMediaSource: this._options?.createMediaSource,
userAgent: this._options?.userAgent,
},
);
if (!binarySource) {
this._startNextBinarySource(context);
return;
}
source = binarySource.source;
this._binary = binarySource;
source.start();
}
private _handleBinaryLoaded(context: ConnectionContext, source: StreamSource): void {
const binary = this._binary;
if (!binary || source !== binary.source) {
return;
}
this._committedLane = 'binary';
this._reportCommittedLoad(context, binary.surface, source);
}
private _handleBinaryFailed(context: ConnectionContext, source: StreamSource): void {
if (source !== this._binary?.source) {
return;
}
source.stop();
this._binary = null;
if (this._committedLane === 'binary') {
this._committedLane = null;
}
this._startNextBinarySource(context);
}
private _teardownBinaryLane(): void {
// Clear the current-source identity before stopping, so any callback fired
// synchronously by stop() fails its `!== this._binary?.source` guard.
const source = this._binary?.source;
this._binary = null;
this._binaryModes = [];
source?.stop();
}
// ===========================================================================
// WebRTC lane.
// ===========================================================================
private _startWebRTCSource(
context: ConnectionContext,
useOffscreenVideo: boolean,
): void {
// While racing a binary lane, decode into the off-screen element; otherwise
// decode straight onto the real video (absent it, there is nothing to do).
let decodeVideo: HTMLVideoElement;
if (useOffscreenVideo) {
decodeVideo = this._offscreenVideo.get();
} else {
const video = context.surfaces.video.getElement();
if (!video) {
return;
}
decodeVideo = video;
}
let source: WebRTCStreamSource | null = null;
const sourceContext: StreamSourceContext<VideoStreamTarget> = {
target: { kind: 'video', video: decodeVideo },
channel: context.channel,
callbacks: {
loadedCallback: () => {
if (source) {
this._handleWebRTCLoaded(context, source, useOffscreenVideo, decodeVideo);
}
},
failedCallback: (reason) => {
if (source) {
this._lastFailureReason = reason;
this._logSourceFailure('webrtc', reason);
this._handleWebRTCFailed(context, source);
}
},
},
};
source = (this._options?.createWebRTCSource ?? createWebRTCSource)(sourceContext, {
microphoneStream: this._microphoneStream,
createPeerConnection: this._options?.createPeerConnection,
createMediaStream: this._options?.createMediaStream,
});
this._webRTCSource = source;
source.start();
}
private _handleWebRTCLoaded(
context: ConnectionContext,
source: WebRTCStreamSource,
useOffscreenVideo: boolean,
decodeVideo: HTMLVideoElement,
): void {
if (source !== this._webRTCSource) {
return;
}
if (!useOffscreenVideo) {
// There is nothing in the 'binary' lane, this controller is WebRTC-only:
// the stream already decoded on the real video element.
this._commitWebRTC(context, source, decodeVideo);
return;
}
// There is video on both the 'binary' and 'webrtc' lane -- choose a winner.
const binary = this._binary?.source ?? null;
const winner = binary
? getPreferredSource(source.getStreamProfile(), binary.getStreamProfile())
: 'webrtc';
if (winner === 'binary') {
// The binary source scored higher: keep it on the real video element and
// tear down the WebRTC lane.
this._teardownWebRTCLane();
return;
}
// WebRTC wins. Tear down the binary lane first: stopping the binary source
// clears the real video element it was playing on, so the WebRTC stream is
// attached to that element (below) only after, never before.
this._teardownBinaryLane();
const stream = source.getMediaStream();
const video = context.surfaces.video.getElement();
if (video && stream) {
video.srcObject = stream;
}
// Report the media-loaded frame size from the off-screen video element
// (decodeVideo): it has been decoding this stream throughout the
// lane-choice so it already knows the size, whereas the real video element
// was only handed the stream on the line above and has not decoded a frame
// yet (so it would report 0x0).
this._commitWebRTC(context, source, decodeVideo);
this._offscreenVideo.clear();
}
private _handleWebRTCFailed(
context: ConnectionContext,
source: WebRTCStreamSource,
): void {
if (source !== this._webRTCSource) {
return;
}
const wasCommitted = this._committedLane === 'webrtc';
this._teardownWebRTCLane();
if (wasCommitted) {
// WebRTC was the committed (live) lane, so losing it leaves the view with
// nothing playing: clear the real video element and force a full
// reconnect (which restarts every configured mode, letting a binary lane
// take over) or escalate via the error callback.
this._committedLane = null;
const video = context.surfaces.video.getElement();
if (video) {
video.srcObject = null;
}
this._closeChannelAndReconnect(context);
} else {
// WebRTC lost a race it was still contesting; the binary lane carries on
// if present, otherwise reconnect.
this._maybeReconnectIfLanesDead(context);
}
}
private _commitWebRTC(
context: ConnectionContext,
source: WebRTCStreamSource,
dimensionsVideo: HTMLVideoElement,
): void {
this._committedLane = 'webrtc';
// WebRTC now carries the media over the peer connection, so the socket is
// done: signaling (offer/answer/ICE) is complete and any binary lane's
// local consumer is torn down. go2rtc has no "stop" message, though --
// tearing a lane down only detaches our side, so the server keeps muxing
// that mode's binary frames down the socket (onto a now-discarded consumer)
// until the socket itself closes. Closing it is what actually ends that
// server stream.
this._channel?.close();
this._channel = null;
this._setupAudioMuteRedispatch(context, source);
this._reportCommittedLoad(context, 'video', source, dimensionsVideo);
}
private _setupAudioMuteRedispatch(
context: ConnectionContext,
source: WebRTCStreamSource,
): void {
this._audioTracksMuteStateUnsubscribeCallback?.();
// The listener is removed on teardown, so a stale source can never fire it.
// Audio mute/unmute re-dispatches read the real video, which holds the
// stream (and its dimensions) by the time any such change occurs.
this._audioTracksMuteStateUnsubscribeCallback = addAudioTracksMuteStateListener(
source.getPeerConnection(),
() => this._dispatchMediaLoaded(context, 'video', source),
);
}
private _teardownWebRTCLane(): void {
// Clear the current-source identity and the audio listener before stopping,
// so any callback fired synchronously by stop() fails its guard or is gone.
const source = this._webRTCSource;
this._webRTCSource = null;
this._audioTracksMuteStateUnsubscribeCallback?.();
this._audioTracksMuteStateUnsubscribeCallback = null;
this._offscreenVideo.clear();
source?.stop();
}
// A source in one of the lanes failed.
private _logSourceFailure(
lane: Lane,
reason: StreamSourceFailureReason,
mode?: Go2RTCMode,
): void {
log(this._callbacks.getCardWideConfig(), 'go2rtc-experimental source failed', {
lane,
...(mode ? { mode } : {}),
reason,
});
}
// ===========================================================================
// Reconnect & escalation.
// ===========================================================================
private _maybeReconnectIfLanesDead(context: ConnectionContext): void {
if (!this._binary && !this._webRTCSource && !this._binaryModes.length) {
this._closeChannelAndReconnect(context);
}
}
// Abandon this connection's channel and start the reconnect-or-escalate flow.
private _closeChannelAndReconnect(context: ConnectionContext): void {
context.channel.close();
this._channel = null;
this._reconnectOrEscalateError(context);
}
private _reconnectOrEscalateError(context: ConnectionContext): void {
if (this._retryTimer.getAttempts() >= RECONNECT_MAX_ATTEMPTS) {
this._callbacks.errorCallback(this._lastFailureReason);
return;
}
this._retryTimer.schedule(() => this._connectChannel(context.url, context.surfaces));
}
// ===========================================================================
// Media-loaded reporting.
// ===========================================================================
// A lane just committed with live media: the connection has proven healthy, so
// reset the fast-reconnect budget, hand the newly live surface to the
// component (resetting the one being left behind), briefly hide the video
// controls so they do not flash over the freshly loaded media, then report the
// load with the committed surface's controller.
private _reportCommittedLoad(
context: ConnectionContext,
surface: SurfaceKind,
source: StreamSource,
dimensionsVideo?: HTMLVideoElement,
): void {
if (source === this._committedSource) {
// The same source re-reporting (e.g. a mid-stream resolution change
// re-fires the video's loadeddata): the surface and controls are already
// set up, so only refresh the reported media dimensions. Re-running the
// control hide would leak its loadstart listener on each repeat.
this._dispatchMediaLoaded(context, surface, source, dimensionsVideo);
return;
}
this._committedSource = source;
this._retryTimer.reset();
this._lastFailureReason = null;
if (this._committedSurface && this._committedSurface !== surface) {
this._resetSurface(context, this._committedSurface);
}
this._committedSurface = surface;
this._callbacks.surfaceCommittedCallback(surface);
const video = context.surfaces.video.getElement();
if (surface === 'video' && this._callbacks.getControls() && video) {
hideMediaControlsTemporarily(video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
}
this._dispatchMediaLoaded(context, surface, source, dimensionsVideo);
}
private _resetSurface(context: ConnectionContext, surface: SurfaceKind): void {
if (surface === 'image') {
context.surfaces.image.reset();
return;
}
const video = context.surfaces.video.getElement();
if (video) {
video.srcObject = null;
video.src = '';
}
}
private _dispatchMediaLoaded(
context: ConnectionContext,
surface: SurfaceKind,
source: StreamSource,
// Element to read the frame size from; used only for the video surface and
// defaults to the video-surface element. Differs when that element was just
// handed a stream and has not decoded a frame yet (WebRTC committed from the
// off-screen video), so its size must come from that off-screen element.
dimensionsVideo?: HTMLVideoElement,
): void {
const targetSurface =
surface === 'video' ? context.surfaces.video : context.surfaces.image;
const dimensionsElement =
surface === 'video'
? dimensionsVideo ?? context.surfaces.video.getElement()
: context.surfaces.image.getElement();
if (!dimensionsElement) {
return;
}
const info = createMediaLoadedInfo(dimensionsElement, {
mediaPlayerController: targetSurface.getMediaPlayer(),
capabilities: source.getCapabilities(),
technology: source.getTechnology(),
});
if (info) {
this._callbacks.mediaLoadedCallback(info);
}
}
}
@@ -0,0 +1,149 @@
import type { UnsubscribeCallback } from '../../../../types';
import {
go2RTCMessageSchema,
type BinaryCallback,
type Go2RTCMessage,
type MessageCallback,
} from './types';
export type WebSocketFactory = (url: string) => WebSocket;
interface SignalingChannelCallbacks {
openCallback?: () => void;
// Fires when the socket disconnects (i.e. not intentional `close()`).
disconnectCallback?: () => void;
}
// Injectable test seam. Defaults to the real `WebSocket`; production passes
// nothing here.
interface SignalingChannelOptions {
createWebSocket?: WebSocketFactory;
}
// Owns the go2rtc WebSocket: JSON text frames are parsed and fanned out to
// message subscribers, binary frames go to the single binary consumer. No
// callbacks of any kind are delivered after `close()`.
export class SignalingChannel {
private _url: string;
private _ws: WebSocket | null = null;
private _open = false;
private _callbacks: SignalingChannelCallbacks;
private _createWebSocket: WebSocketFactory;
private _binaryCallback: BinaryCallback | null = null;
private _messageCallbacks = new Set<MessageCallback>();
constructor(
url: string,
callbacks: SignalingChannelCallbacks,
options?: SignalingChannelOptions,
) {
this._url = url;
this._callbacks = callbacks;
this._createWebSocket =
options?.createWebSocket ?? ((wsURL: string) => new WebSocket(wsURL));
}
public connect(): void {
if (this._ws) {
return;
}
const ws = this._createWebSocket(this._url);
this._ws = ws;
ws.binaryType = 'arraybuffer';
// Each listener ignores events once this websocket is no longer current
// (e.g. events already queued when `close()` was called).
ws.addEventListener('open', () => {
if (this._ws !== ws) {
return;
}
this._open = true;
this._callbacks.openCallback?.();
});
ws.addEventListener('close', () => {
if (this._ws !== ws) {
return;
}
this._ws = null;
this._open = false;
// This will not be called when we intentionally call close(), as the
// guard above will have nulled out this._ws.
this._callbacks.disconnectCallback?.();
});
ws.addEventListener('message', (ev) => {
if (this._ws !== ws) {
return;
}
this._handleMessage(ev.data);
});
}
public close(): void {
const ws = this._ws;
this._ws = null;
this._open = false;
ws?.close();
}
public isOpen(): boolean {
return this._open;
}
public send(message: Go2RTCMessage): void {
if (this._ws && this._open) {
this._ws.send(JSON.stringify(message));
}
}
public subscribeToMessages(callback: MessageCallback): UnsubscribeCallback {
this._messageCallbacks.add(callback);
return () => {
this._messageCallbacks.delete(callback);
};
}
public setBinaryCallback(callback: BinaryCallback | null): void {
this._binaryCallback = callback;
}
private _handleMessage(data: unknown): void {
if (data instanceof ArrayBuffer) {
this._binaryCallback?.(data);
return;
}
if (typeof data !== 'string') {
return;
}
let parsed: unknown;
try {
parsed = JSON.parse(data);
} catch {
return;
}
const result = go2RTCMessageSchema.safeParse(parsed);
if (!result.success) {
return;
}
// Iterate over a copy: a callback may subscribe or unsubscribe during
// dispatch.
for (const callback of [...this._messageCallbacks]) {
callback(result.data);
}
}
}
@@ -0,0 +1,102 @@
import type { Go2RTCMode } from '../../../../../config/schema/cameras';
import type { MediaSourceFactory } from '../adapters/media-source';
import type { PeerConnectionFactory } from '../adapters/peer-connection';
import type {
ImageStreamTarget,
StreamSource,
StreamSourceCallbacks,
StreamSourceChannel,
StreamSourceContext,
SurfaceKind,
VideoStreamTarget,
} from '../types';
import { MJPEGStreamSource } from './mjpeg';
import { MP4StreamSource } from './mp4';
import { MSEStreamSource } from './mse';
import { WebRTCStreamSource, type MediaStreamFactory } from './webrtc';
// ===========================================================================
// Binary lane
// ===========================================================================
interface CreateBinarySourceOptions {
createMediaSource?: MediaSourceFactory;
userAgent?: string;
}
// The available render targets, one of which each mode is wired to. The factory
// is the single place that maps a mode to its surface, so it reports back which
// one it chose.
export interface BinaryStreamTargets {
video: VideoStreamTarget;
image: ImageStreamTarget;
}
export interface BinarySource {
source: StreamSource;
surface: SurfaceKind;
}
// Builds a source for a mode that streams binary media over the WebSocket (only
// one such mode runs per connection). WebRTC is created separately via
// `createWebRTCSource` because it runs in parallel and has its own interface.
export type BinarySourceFactory = (
mode: Go2RTCMode,
targets: BinaryStreamTargets,
channel: StreamSourceChannel,
callbacks: StreamSourceCallbacks,
options?: CreateBinarySourceOptions,
) => BinarySource | null;
// Returns null for modes with no binary source (i.e. `webrtc`, handled
// separately via `createWebRTCSource`).
export const createBinarySource: BinarySourceFactory = (
mode: Go2RTCMode,
targets: BinaryStreamTargets,
channel: StreamSourceChannel,
callbacks: StreamSourceCallbacks,
options?: CreateBinarySourceOptions,
): BinarySource | null => {
switch (mode) {
case 'mse':
return {
source: new MSEStreamSource(
{ target: targets.video, channel, callbacks },
options,
),
surface: 'video',
};
case 'mp4':
return {
source: new MP4StreamSource({ target: targets.image, channel, callbacks }),
surface: 'image',
};
case 'mjpeg':
return {
source: new MJPEGStreamSource({ target: targets.image, channel, callbacks }),
surface: 'image',
};
default:
return null;
}
};
// ===========================================================================
// WebRTC lane
// ===========================================================================
export interface CreateWebRTCSourceOptions {
createPeerConnection?: PeerConnectionFactory;
createMediaStream?: MediaStreamFactory;
microphoneStream?: MediaStream | null;
}
export type WebRTCSourceFactory = (
context: StreamSourceContext<VideoStreamTarget>,
options?: CreateWebRTCSourceOptions,
) => WebRTCStreamSource;
export const createWebRTCSource: WebRTCSourceFactory = (
context: StreamSourceContext<VideoStreamTarget>,
options?: CreateWebRTCSourceOptions,
): WebRTCStreamSource => new WebRTCStreamSource(context, options);
@@ -0,0 +1,112 @@
import type { Go2RTCMode } from '../../../../../config/schema/cameras';
import type {
MediaLoadedCapabilities,
MediaTechnology,
UnsubscribeCallback,
} from '../../../../../types';
import { Timer } from '../../../../../utils/timer';
import type {
Go2RTCMessage,
ImageStreamTarget,
StreamProfile,
StreamSource,
StreamSourceContext,
} from '../types';
import { isServerErrorForMode } from '../utils/messages';
// Fail if no frame arrives within this window. The channel is open and the mode
// was requested, but the server may send neither a frame nor an error, so
// without this the source would hang silently instead of failing over. Mirrors
// the MSE negotiation and WebRTC connect timeouts.
const FIRST_FRAME_TIMEOUT_SECONDS = 5;
// Base for the modes that present a stream as a sequence of still images fed to
// the image surface (MJPEG, MP4): both receive binary frames and turn each into
// a single-image Blob, differing only in the request message and the per-frame
// conversion. These are last-resort fallbacks: no audio, no "real" playback.
export abstract class ImageFrameStreamSource implements StreamSource {
protected _context: StreamSourceContext<ImageStreamTarget>;
private _loaded = false;
private _stopped = false;
private _firstFrameTimer = new Timer();
private _unsubscribe: UnsubscribeCallback | null = null;
constructor(context: StreamSourceContext<ImageStreamTarget>) {
this._context = context;
}
protected abstract _mode: Go2RTCMode;
protected abstract _getRequestMessage(): Go2RTCMessage;
protected abstract _handleFrame(data: ArrayBuffer): void;
public start(): void {
// Arm before wiring up the channel: a synchronous channel could deliver the
// first frame during send(), which must be able to stop an already-armed
// timer rather than have start() arm one that never stops.
this._firstFrameTimer.start(FIRST_FRAME_TIMEOUT_SECONDS, () =>
this._context.callbacks.failedCallback('connect_timeout'),
);
this._unsubscribe = this._context.channel.subscribeToMessages((message) => {
if (isServerErrorForMode(message, this._mode)) {
this._context.callbacks.failedCallback('server_error');
}
});
this._context.channel.setBinaryCallback((data) => this._handleFrame(data));
this._context.channel.send(this._getRequestMessage());
}
public stop(): void {
this._stopped = true;
this._firstFrameTimer.stop();
this._unsubscribe?.();
this._unsubscribe = null;
this._context.channel.setBinaryCallback(null);
this._teardown();
}
public getCapabilities(): MediaLoadedCapabilities {
return { supportsPause: false };
}
public getTechnology(): MediaTechnology[] {
return [this._mode];
}
public getStreamProfile(): StreamProfile {
return { hasVideo: true, hasH265Video: false, hasAudio: false, hasAACAudio: false };
}
// Show a rendered frame and, on the first one, report the stream loaded.
protected async _showFrame(frame: Blob): Promise<void> {
// MP4 turns each frame into a Blob asynchronously (canvas.toBlob), so a
// frame can arrive here after stop(). Drop it so a stopped source never
// writes to the shared image surface or reports a stale load.
if (this._stopped) {
return;
}
const decoded = this._context.target.showFrame(frame);
if (this._loaded) {
return;
}
this._loaded = true;
this._firstFrameTimer.stop();
// Report loaded only once the frame has decoded: the media-loaded info is
// built by reading the <img> dimensions, and an <img> whose src was just
// set still reports 0x0 (which is rejected as invalid), so reporting
// synchronously would leave the media stuck "not loading".
await decoded;
if (!this._stopped) {
this._context.callbacks.loadedCallback();
}
}
protected _teardown(): void {}
}
@@ -0,0 +1,16 @@
import type { Go2RTCMessage } from '../types';
import { ImageFrameStreamSource } from './image-frame';
// Each binary frame is a complete JPEG, shown directly as an image frame.
export class MJPEGStreamSource extends ImageFrameStreamSource {
protected _mode = 'mjpeg' as const;
protected _getRequestMessage(): Go2RTCMessage {
return { type: 'mjpeg' };
}
protected _handleFrame(data: ArrayBuffer): void {
/* istanbul ignore next: This never rejects; the catch satisfies the no-floating-promises -- @preserve */
this._showFrame(new Blob([data], { type: 'image/jpeg' })).catch(() => {});
}
}
@@ -0,0 +1,98 @@
import { OffscreenVideo, type VideoElementFactory } from '../offscreen-video';
import type { Go2RTCMessage, ImageStreamTarget, StreamSourceContext } from '../types';
import { arrayBufferToBase64 } from '../utils/base64';
import {
convertToCodecString,
GO2RTC_CODECS,
selectSupportedCodecs,
} from '../utils/codecs';
import { ImageFrameStreamSource } from './image-frame';
type CanvasElementFactory = () => HTMLCanvasElement;
interface MP4StreamSourceOptions {
createVideoElement?: VideoElementFactory;
createCanvasElement?: CanvasElementFactory;
}
// Each binary frame is a standalone MP4 holding one keyframe. Unlike JPEG it
// cannot be shown directly, so it is decoded in an off-screen video and drawn
// to a canvas to produce the image.
export class MP4StreamSource extends ImageFrameStreamSource {
protected _mode = 'mp4' as const;
private _decoder: OffscreenVideo;
private _createCanvasElement: CanvasElementFactory;
private _canvas: HTMLCanvasElement | null = null;
constructor(
context: StreamSourceContext<ImageStreamTarget>,
options?: MP4StreamSourceOptions,
) {
super(context);
const createVideo =
options?.createVideoElement ?? (() => document.createElement('video'));
this._decoder = new OffscreenVideo(() => {
const video = createVideo();
video.autoplay = true;
video.playsInline = true;
video.muted = true;
video.addEventListener('loadeddata', () => this._drawFrame(video));
return video;
});
this._createCanvasElement =
options?.createCanvasElement ?? (() => document.createElement('canvas'));
}
protected _getRequestMessage(): Go2RTCMessage {
// Codec support is probed on the decoder video, since that is what plays
// the incoming MP4 frames; the image surface onto which this is rendered
// has has no video element of its own.
const decoder = this._decoder.get();
return {
type: 'mp4',
value: convertToCodecString(
selectSupportedCodecs(
GO2RTC_CODECS,
{ audio: false, video: true },
(mimeType) => !!decoder.canPlayType(mimeType),
),
),
};
}
protected _handleFrame(data: ArrayBuffer): void {
this._decoder.get().src = 'data:video/mp4;base64,' + arrayBufferToBase64(data);
}
protected _teardown(): void {
this._decoder.clear();
this._canvas = null;
}
private _drawFrame(decoder: HTMLVideoElement): void {
const canvas = (this._canvas ??= this._createCanvasElement());
// Reassigning a canvas dimension reallocates its backing buffer, so skip it
// when the video size is unchanged.
if (canvas.width !== decoder.videoWidth || canvas.height !== decoder.videoHeight) {
canvas.width = decoder.videoWidth;
canvas.height = decoder.videoHeight;
}
const context = canvas.getContext('2d');
if (!context) {
return;
}
context.drawImage(decoder, 0, 0, canvas.width, canvas.height);
canvas.toBlob((frame) => {
if (frame) {
/* istanbul ignore next: This never rejects; the catch satisfies the no-floating-promises -- @preserve */
this._showFrame(frame).catch(() => {});
}
}, 'image/jpeg');
}
}
@@ -0,0 +1,347 @@
import type {
MediaLoadedCapabilities,
MediaTechnology,
UnsubscribeCallback,
} from '../../../../../types';
import { hasAudio } from '../../../../../utils/audio';
import { Timer } from '../../../../../utils/timer';
import {
createBrowserMediaSource,
type MediaSourceFactory,
type MediaSourceInterface,
} from '../adapters/media-source';
import type {
Go2RTCMessage,
StreamProfile,
StreamSource,
StreamSourceContext,
VideoStreamTarget,
} from '../types';
import { BoundedBufferQueue } from '../utils/bounded-buffer-queue';
import {
convertToCodecString,
getCodecsForUserAgent,
selectSupportedCodecs,
} from '../utils/codecs';
import { LiveEdgeTracker } from '../utils/live-edge-tracker';
import type { LiveEdgeAction } from '../utils/live-edge-tracker/types';
import { isServerErrorForMode } from '../utils/messages';
import { isWebKitUserAgent } from '../utils/user-agent';
// ===========================================================================
// MSE Tuning
// ===========================================================================
// These are judgement-based budgets (time, buffered seconds, staged bytes)
// balancing latency and resilience, not derived values. The retained-buffer
// window and the staged-bytes cap follow go2rtc's reference web client; the
// negotiation timeout has no counterpart there and is added by this
// implementation.
// Seconds of media retained behind the live edge; older media is trimmed. This
// bounds memory even across a long pause (the buffer never grows past this
// window), so a paused stream can hold its frame indefinitely.
const RETAINED_BUFFER_SECONDS = 15;
// On resume, if the playhead is more than this far behind the live edge, jump
// forward to rejoin live rather than replaying the buffered-behind media.
const LIVE_EDGE_RESUME_LAG_SECONDS = 1;
// Where a live-edge jump lands, as a gap behind the buffered end (a small
// cushion so playback does not immediately starve at the very edge).
const LIVE_EDGE_JUMP_OFFSET_SECONDS = 0.75;
// Bound on media staged while the SourceBuffer is busy; a stream that outruns
// this is unrecoverable without a reconnect.
const MAX_PENDING_BUFFER_BYTES = 2 * 1024 * 1024;
// How long to wait for the server's codec reply before failing the source.
const MSE_NEGOTIATION_TIMEOUT_SECONDS = 5;
// ===========================================================================
// MSEStreamSource
// ===========================================================================
interface MSEStreamSourceOptions {
createMediaSource?: MediaSourceFactory;
userAgent?: string;
}
export class MSEStreamSource implements StreamSource {
private _context: StreamSourceContext<VideoStreamTarget>;
private _createMediaSource: MediaSourceFactory;
private _mediaSource: MediaSourceInterface | null = null;
private _sourceBuffer: SourceBuffer | null = null;
private _pendingBuffer = new BoundedBufferQueue(MAX_PENDING_BUFFER_BYTES);
// The codec string the server settled on during negotiation; null until then.
private _codecs: string | null = null;
// The codecs offered to the server, derived once from the user agent.
private _codecCandidates: readonly string[];
private _negotiationTimer = new Timer();
private _liveEdge: LiveEdgeTracker;
private _unsubscribeCallbacks: UnsubscribeCallback[] = [];
private _loadedHandler = (): void => {
this._context.callbacks.loadedCallback();
};
private _errorHandler = (): void => {
this._context.callbacks.failedCallback('media_error');
};
// On resume, rejoin the very live edge.
private _playHandler = (): void => {
const sourceBuffer = this._sourceBuffer;
if (!sourceBuffer?.buffered.length) {
return;
}
const video = this._context.target.video;
const end = sourceBuffer.buffered.end(sourceBuffer.buffered.length - 1);
if (end - video.currentTime > LIVE_EDGE_RESUME_LAG_SECONDS) {
video.currentTime = end - LIVE_EDGE_JUMP_OFFSET_SECONDS;
}
};
constructor(
context: StreamSourceContext<VideoStreamTarget>,
options?: MSEStreamSourceOptions,
) {
this._context = context;
this._createMediaSource = options?.createMediaSource ?? createBrowserMediaSource;
const userAgent = options?.userAgent ?? navigator.userAgent;
this._liveEdge = new LiveEdgeTracker({ webkit: isWebKitUserAgent(userAgent) });
this._codecCandidates = getCodecsForUserAgent(userAgent);
}
public start(): void {
const mediaSource = this._createMediaSource();
if (!mediaSource) {
this._context.callbacks.failedCallback('unsupported');
return;
}
this._mediaSource = mediaSource;
this._unsubscribeCallbacks.push(
mediaSource.subscribeToSourceOpen(() => this._negotiate(mediaSource)),
this._context.channel.subscribeToMessages((message) =>
this._handleMessage(mediaSource, message),
),
);
this._context.target.video.addEventListener('loadeddata', this._loadedHandler);
this._context.target.video.addEventListener('error', this._errorHandler);
this._context.target.video.addEventListener('play', this._playHandler);
mediaSource.attach(this._context.target.video);
}
public stop(): void {
this._negotiationTimer.stop();
this._unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe());
this._unsubscribeCallbacks = [];
this._context.target.video.removeEventListener('loadeddata', this._loadedHandler);
this._context.target.video.removeEventListener('error', this._errorHandler);
this._context.target.video.removeEventListener('play', this._playHandler);
this._context.channel.setBinaryCallback(null);
this._mediaSource?.detach(this._context.target.video);
this._mediaSource = null;
this._sourceBuffer = null;
this._pendingBuffer.clear();
}
public getCapabilities(): MediaLoadedCapabilities {
return {
supportsPause: true,
hasAudio: hasAudio(this._context.target.video, { mseCodecs: this._codecs }),
has2WayAudio: false,
};
}
public getTechnology(): MediaTechnology[] {
return ['mse'];
}
public getStreamProfile(): StreamProfile {
const codecs = this._codecs ?? '';
return {
hasVideo: codecs.includes('avc1') || codecs.includes('hvc1'),
hasH265Video: codecs.includes('hvc1'),
hasAudio:
codecs.includes('mp4a') || codecs.includes('opus') || codecs.includes('flac'),
hasAACAudio: codecs.includes('mp4a'),
};
}
private _negotiate(mediaSource: MediaSourceInterface): void {
const codecs = selectSupportedCodecs(
this._codecCandidates,
{ audio: true, video: true },
(mimeType) => mediaSource.isTypeSupported(mimeType),
);
this._context.channel.send({ type: 'mse', value: convertToCodecString(codecs) });
this._negotiationTimer.start(MSE_NEGOTIATION_TIMEOUT_SECONDS, () =>
this._context.callbacks.failedCallback('negotiation_timeout'),
);
}
private _handleMessage(
mediaSource: MediaSourceInterface,
message: Go2RTCMessage,
): void {
if (isServerErrorForMode(message, 'mse')) {
this._negotiationTimer.stop();
this._context.callbacks.failedCallback('server_error');
return;
}
if (
message.type !== 'mse' ||
typeof message.value !== 'string' ||
this._sourceBuffer
) {
return;
}
this._negotiationTimer.stop();
// go2rtc answers the codec offer with an `mse` text message whose value is
// the codec string for addSourceBuffer. It arrives once, before any
// SourceBuffer exists (the guard above returns for every later message), so
// this value is the negotiated codecs.
this._codecs = message.value;
let sourceBuffer: SourceBuffer;
try {
sourceBuffer = mediaSource.addSourceBuffer(message.value);
} catch {
this._context.callbacks.failedCallback('media_error');
return;
}
// Segments mode: order media by its fMP4 timestamps rather than by
// arrival order.
sourceBuffer.mode = 'segments';
const updateEndListener = (): void =>
this._handleUpdateEnd(mediaSource, sourceBuffer);
sourceBuffer.addEventListener('updateend', updateEndListener);
this._unsubscribeCallbacks.push(() =>
sourceBuffer.removeEventListener('updateend', updateEndListener),
);
this._sourceBuffer = sourceBuffer;
this._context.channel.setBinaryCallback((data) =>
this._handleData(sourceBuffer, data),
);
}
private _handleData(sourceBuffer: SourceBuffer, data: ArrayBuffer): void {
if (sourceBuffer.updating || !this._pendingBuffer.isEmpty) {
if (!this._pendingBuffer.push(data)) {
this._context.callbacks.failedCallback('buffer_overflow');
}
return;
}
this._append(sourceBuffer, data);
}
private _append(sourceBuffer: SourceBuffer, data: ArrayBuffer): void {
try {
sourceBuffer.appendBuffer(data);
} catch {
// Append failures during teardown races are recoverable by later appends;
// fatal conditions surface through the video element's error event
// instead.
}
}
private _handleUpdateEnd(
mediaSource: MediaSourceInterface,
sourceBuffer: SourceBuffer,
): void {
if (sourceBuffer.updating) {
return;
}
// A queued updateend can fire after a reconnect or teardown has closed the
// MediaSource (detached from the video).
if (!mediaSource.isOpen()) {
return;
}
const pending = this._pendingBuffer.shift();
if (pending) {
this._append(sourceBuffer, pending);
return;
}
if (!sourceBuffer.buffered.length) {
return;
}
const video = this._context.target.video;
const end = sourceBuffer.buffered.end(sourceBuffer.buffered.length - 1);
this._trimSourceBuffer(mediaSource, sourceBuffer, end);
// While paused the user is holding a frame: leave the playhead alone (the
// trim above still bounds memory) and do not sample the growing lag, which
// would otherwise bias the catch-up rate once playback resumes. Resuming
// jumps back to the live edge (see the play handler).
if (video.paused) {
return;
}
// Keep playback tracking the live edge as new media arrives.
this._applyLiveEdgeAction(
video,
this._liveEdge.next({
bufferedEndSeconds: end,
currentTimeSeconds: video.currentTime,
playbackRate: video.playbackRate,
now: new Date(),
}),
);
}
private _applyLiveEdgeAction(video: HTMLVideoElement, action: LiveEdgeAction): void {
if (action.action === 'seek') {
video.currentTime = action.seconds;
} else if (action.action === 'rate' && video.playbackRate !== action.rate) {
video.playbackRate = action.rate;
}
}
// Keep only the most recent RETAINED_BUFFER_SECONDS of media, trimming older
// media and re-declaring the seekable range. The playhead is never snapped
// here: while playing the live-edge tracker keeps it near the edge, and while
// paused it is left holding its frame (a playhead that falls past the lag
// bound is handled by a reconnect, not by dragging it forward).
private _trimSourceBuffer(
mediaSource: MediaSourceInterface,
sourceBuffer: SourceBuffer,
end: number,
): void {
const retainedStart = end - RETAINED_BUFFER_SECONDS;
const bufferedStart = sourceBuffer.buffered.start(0);
if (retainedStart > bufferedStart) {
sourceBuffer.remove(bufferedStart, retainedStart);
mediaSource.setLiveSeekableRange(retainedStart, end);
}
}
}
@@ -0,0 +1,291 @@
import type {
MediaLoadedCapabilities,
MediaTechnology,
UnsubscribeCallback,
} from '../../../../../types';
import { has2WayAudio, hasAudio } from '../../../../../utils/audio';
import { Timer } from '../../../../../utils/timer';
import {
createBrowserPeerConnection,
GO2RTC_PEER_CONNECTION_CONFIG,
type PeerConnectionFactory,
} from '../adapters/peer-connection';
import type {
Go2RTCMessage,
StreamProfile,
StreamSource,
StreamSourceContext,
VideoStreamTarget,
} from '../types';
import { isServerErrorForMode } from '../utils/messages';
import { sdpHasH265 } from '../utils/webrtc-sdp';
// ===========================================================================
// WebRTC Tuning
// ===========================================================================
// Fail if no decoded frame arrives within this window. Covers the case where
// ICE connects and packets flow but frames never decode (the connection looks
// healthy yet nothing plays), so the session can fall back instead of hanging.
// See https://github.com/dermotduffy/advanced-camera-card/issues/1699
// The window length is a tuning value, not derived: long enough to avoid
// failing a slow-but-healthy start, short enough to fall back promptly.
const WEBRTC_CONNECT_TIMEOUT_SECONDS = 5;
// ===========================================================================
// WebRTCStreamSource
// ===========================================================================
export type MediaStreamFactory = (tracks: MediaStreamTrack[]) => MediaStream;
interface WebRTCStreamSourceOptions {
createPeerConnection?: PeerConnectionFactory;
createMediaStream?: MediaStreamFactory;
microphoneStream?: MediaStream | null;
}
export class WebRTCStreamSource implements StreamSource {
private _context: StreamSourceContext<VideoStreamTarget>;
private _stream: MediaStream | null = null;
private _pc: RTCPeerConnection | null = null;
private _createPeerConnection: PeerConnectionFactory;
private _createMediaStream: MediaStreamFactory;
private _microphoneStream: MediaStream | null;
private _microphoneTransceiver: RTCRtpTransceiver | null = null;
private _connectTimer = new Timer();
private _unsubscribeCallbacks: UnsubscribeCallback[] = [];
private _loadedHandler = (): void => {
this._connectTimer.stop();
this._context.callbacks.loadedCallback();
};
constructor(
context: StreamSourceContext<VideoStreamTarget>,
options?: WebRTCStreamSourceOptions,
) {
this._context = context;
this._createPeerConnection =
options?.createPeerConnection ?? createBrowserPeerConnection;
this._createMediaStream =
options?.createMediaStream ?? ((tracks) => new MediaStream(tracks));
this._microphoneStream = options?.microphoneStream ?? null;
}
public start(): void {
const pc = this._createPeerConnection(GO2RTC_PEER_CONNECTION_CONFIG);
this._pc = pc;
// Always pre-arm exactly one outbound audio slot so the microphone track
// can be attached later via `replaceTrack` with no renegotiation. The
// kind-only `addTransceiver('audio', ...)` form never calls getUserMedia,
// so it never raises permission prompt for users.
const microphoneTrack = this._microphoneStream?.getAudioTracks()[0] ?? null;
this._microphoneTransceiver = pc.addTransceiver(microphoneTrack ?? 'audio', {
direction: 'sendonly',
});
pc.addTransceiver('video', { direction: 'recvonly' });
pc.addTransceiver('audio', { direction: 'recvonly' });
pc.addEventListener('icecandidate', (ev) => {
// A late candidate from a superseded connection (e.g. after stop(), if
// WebRTC has lost the race) must not send on the channel a binary lane
// may still be using.
if (this._pc !== pc) {
return;
}
// An empty value signals end-of-candidates, which the server accepts.
this._context.channel.send({
type: 'webrtc/candidate',
value: ev.candidate ? ev.candidate.candidate : '',
});
});
pc.addEventListener('connectionstatechange', () =>
this._handleConnectionStateChange(pc),
);
this._unsubscribeCallbacks.push(
this._context.channel.subscribeToMessages((message) =>
this._handleMessage(pc, message),
),
);
this._connectTimer.start(WEBRTC_CONNECT_TIMEOUT_SECONDS, () =>
this._context.callbacks.failedCallback('connect_timeout'),
);
this._negotiate(pc).catch(() => {
// A rejection from a superseded connection (e.g. after `stop`) is not
// this source's concern.
if (this._pc === pc) {
this._context.callbacks.failedCallback('media_error');
}
});
}
public stop(): void {
this._connectTimer.stop();
this._unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe());
this._unsubscribeCallbacks = [];
this._context.target.video.removeEventListener('loadeddata', this._loadedHandler);
if (this._pc) {
// pc.close() does not stop the sender's tracks, so the outbound microphone
// track keeps running. That is deliberate: MicrophoneManager owns the mic
// (it is shared across cameras), so stopping it here would break it
// elsewhere -- do not add a track.stop() here. See
// https://github.com/dermotduffy/advanced-camera-card/issues/1810
this._pc.close();
this._pc = null;
}
// The transceiver belonged to the now-closed peer connection.
this._microphoneTransceiver = null;
this._context.target.video.srcObject = null;
this._stream = null;
}
public getMediaStream(): MediaStream | null {
return this._stream;
}
public getPeerConnection(): RTCPeerConnection | null {
return this._pc;
}
public getCapabilities(): MediaLoadedCapabilities {
return {
supportsPause: true,
hasAudio: hasAudio(this._context.target.video, { pc: this._pc }),
has2WayAudio: has2WayAudio(this._pc),
};
}
public getTechnology(): MediaTechnology[] {
return ['webrtc'];
}
public getStreamProfile(): StreamProfile {
const sdp = this._pc?.remoteDescription?.sdp ?? null;
return {
hasVideo: (this._stream?.getVideoTracks().length ?? 0) > 0,
hasH265Video: sdp ? sdpHasH265(sdp) : false,
hasAudio: (this._stream?.getAudioTracks().length ?? 0) > 0,
hasAACAudio: false,
};
}
// Swap the outbound microphone track without renegotiating. Guards against a
// late rejection from a superseded call (a newer stream, or teardown)
// bringing a retired connection back or overwriting a fresher request.
public async setMicrophoneStream(stream: MediaStream | null): Promise<void> {
if (this._microphoneStream === stream) {
return;
}
this._microphoneStream = stream;
const transceiver = this._microphoneTransceiver;
if (!transceiver) {
// No peer connection yet; the next `start()` reads the current stream and
// pre-arms the transceiver with it.
return;
}
// A microphone stream carries a single audio track; null detaches the sender.
const desiredTrack = stream?.getAudioTracks()[0] ?? null;
try {
await transceiver.sender.replaceTrack(desiredTrack);
} catch {
const stillCurrent =
transceiver === this._microphoneTransceiver &&
this._microphoneStream === stream &&
this._pc !== null;
if (stillCurrent) {
this._context.callbacks.failedCallback('two_way_audio_error');
}
}
}
private async _negotiate(pc: RTCPeerConnection): Promise<void> {
const offer = await pc.createOffer();
if (this._pc !== pc) {
return;
}
await pc.setLocalDescription(offer);
if (this._pc !== pc) {
return;
}
this._context.channel.send({ type: 'webrtc/offer', value: offer.sdp ?? '' });
}
private _handleMessage(pc: RTCPeerConnection, message: Go2RTCMessage): void {
// Every message type handled here carries a string value.
if (this._pc !== pc || typeof message.value !== 'string') {
return;
}
if (isServerErrorForMode(message, 'webrtc')) {
this._context.callbacks.failedCallback('server_error');
return;
}
switch (message.type) {
case 'webrtc/answer':
pc.setRemoteDescription({ type: 'answer', sdp: message.value }).catch(() => {});
break;
case 'webrtc/candidate':
if (message.value) {
// The server sends no sdpMid; max-bundle puts every track on m-line 0.
pc.addIceCandidate({ candidate: message.value, sdpMid: '0' }).catch(() => {});
}
break;
}
}
private _handleConnectionStateChange(pc: RTCPeerConnection): void {
if (this._pc !== pc) {
return;
}
if (pc.connectionState === 'connected') {
this._attachStream(pc);
} else if (pc.connectionState === 'failed') {
// Only 'failed' is terminal. 'disconnected' is a recoverable ICE blip
// that usually returns to 'connected' on its own, so failing on it would
// turn a brief network hiccup into a needless reconnect; a disconnect
// that never recovers stops delivering frames and is caught by the
// frame-stall watchdog instead.
this._context.callbacks.failedCallback('media_error');
}
}
private _attachStream(pc: RTCPeerConnection): void {
if (this._stream) {
return;
}
const tracks = pc
.getTransceivers()
.filter((transceiver) => transceiver.currentDirection === 'recvonly')
.map((transceiver) => transceiver.receiver.track);
const stream = this._createMediaStream(tracks);
this._stream = stream;
this._context.target.video.addEventListener('loadeddata', this._loadedHandler, {
once: true,
});
this._context.target.video.srcObject = stream;
}
}
@@ -0,0 +1,129 @@
import { z } from 'zod';
import type {
MediaLoadedCapabilities,
MediaTechnology,
UnsubscribeCallback,
} from '../../../../types';
// ===========================================================================
// Control messages
// ===========================================================================
// go2rtc control messages are JSON text frames of this shape; media flows as
// separate binary frames.
export const go2RTCMessageSchema = z.object({
type: z.string(),
// Per-type payload (a codec list, an SDP, an ICE candidate, error text, ...):
// absent for some types (e.g. mjpeg) and not always a string, so it is typed
// `unknown` and each handler narrows it before use.
value: z.unknown().optional(),
});
export type Go2RTCMessage = z.infer<typeof go2RTCMessageSchema>;
export type MessageCallback = (message: Go2RTCMessage) => void;
export type BinaryCallback = (data: ArrayBuffer) => void;
// ===========================================================================
// Signaling channel
// ===========================================================================
// A session's two parallel delivery paths: 'binary' (MSE/MP4/MJPEG media over
// the WebSocket) and 'webrtc' (media over the peer connection).
export type Lane = 'binary' | 'webrtc';
// The narrow view of the signaling channel that stream sources receive: they
// may exchange messages but not manage the connection.
export interface StreamSourceChannel {
send(message: Go2RTCMessage): void;
subscribeToMessages(callback: MessageCallback): UnsubscribeCallback;
setBinaryCallback(callback: BinaryCallback | null): void;
}
// ===========================================================================
// Render targets
// ===========================================================================
// Surface vs Target: A "target" is where a source (binary vs webrtc) puts its
// frames for one mode: a <video> element, or an <img> element (via a showFrame
// callback). The component adds a media-player controller to a target to make a
// fuller "surface" (see SessionSurfaces in session-controller.ts); it hands a
// source only the target, never the surface, so a source cannot reach the
// controller.
// What a source renders onto: the real <video> for modes the browser plays
// (MSE, WebRTC), or an image sink fed decoded frames for modes presented as a
// sequence of still images (MP4, MJPEG).
export interface VideoStreamTarget {
kind: 'video';
video: HTMLVideoElement;
}
// A frame is handed over as a Blob rather than exposing the raw <img> so the
// object-URL lifecycle (create, revoke the previous) stays owned by the image
// surface, not each source. The returned promise resolves once the frame has
// decoded, so a source can defer reporting itself loaded until the surface has
// real dimensions.
export interface ImageStreamTarget {
kind: 'image';
showFrame(frame: Blob): Promise<void>;
}
type StreamSourceTarget = VideoStreamTarget | ImageStreamTarget;
// Which of the two render surfaces media is shown on: the real <video> or the
// <img>. Matches the `kind` of the corresponding stream target.
export type SurfaceKind = 'video' | 'image';
// ===========================================================================
// Stream sources
// ===========================================================================
// Describes a source's negotiated media, used to arbitrate the MSE-vs-WebRTC
// race. The bit-weighted comparison lives in `utils/source-priority.ts`.
export interface StreamProfile {
hasVideo: boolean;
hasH265Video: boolean;
hasAudio: boolean;
// The MSE audio score is inferred from the negotiated codec string, not an
// observed track, so it counts only AAC -- the audio codec browsers decode
// reliably via MSE. Opus and FLAC over MSE are inconsistently supported, so
// crediting them could prefer a stream whose audio never plays.
hasAACAudio: boolean;
}
export type StreamSourceFailureReason =
| 'two_way_audio_error'
| 'buffer_overflow'
| 'connect_timeout'
| 'media_error'
| 'negotiation_timeout'
| 'server_error'
| 'unsupported';
export interface StreamSourceCallbacks {
loadedCallback: () => void;
failedCallback: (reason: StreamSourceFailureReason) => void;
}
export interface StreamSourceContext<T extends StreamSourceTarget = StreamSourceTarget> {
target: T;
channel: StreamSourceChannel;
callbacks: StreamSourceCallbacks;
}
// A single streaming mode implementation (e.g. MSE). Sources negotiate over the
// shared channel, attach media to the provided video element, and report
// readiness or failure via callbacks; connection management and failure
// recovery belong to the session not the source.
export interface StreamSource {
// Single-use: start() begins the source and stop() tears it down for good.
start(): void;
stop(): void;
getCapabilities(): MediaLoadedCapabilities;
getTechnology(): MediaTechnology[];
getStreamProfile(): StreamProfile;
}
@@ -0,0 +1,11 @@
export const arrayBufferToBase64 = (buffer: ArrayBuffer): string => {
const bytes = new Uint8Array(buffer);
let binary = '';
// `btoa` needs bytes as a binary string, build char-by-char.
for (let i = 0; i < bytes.byteLength; ++i) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
};
@@ -0,0 +1,42 @@
// A byte-bounded FIFO queue of ArrayBuffer chunks: a push that would exceed the
// byte cap is rejected.
export class BoundedBufferQueue {
private _buffers: ArrayBuffer[] = [];
private _bytes = 0;
private _maxBytes: number;
constructor(maxBytes: number) {
this._maxBytes = maxBytes;
}
public get isEmpty(): boolean {
return this._buffers.length === 0;
}
// Stage a chunk at the back, or return false (staging nothing) if it would
// exceed the byte cap.
public push(data: ArrayBuffer): boolean {
if (this._bytes + data.byteLength > this._maxBytes) {
return false;
}
this._buffers.push(data);
this._bytes += data.byteLength;
return true;
}
// Remove and return the oldest staged chunk, or null when empty.
public shift(): ArrayBuffer | null {
const data = this._buffers.shift();
if (!data) {
return null;
}
this._bytes -= data.byteLength;
return data;
}
public clear(): void {
this._buffers = [];
this._bytes = 0;
}
}
@@ -0,0 +1,100 @@
import { getSafariMajorVersion } from './user-agent';
// Codec strings offered to the go2rtc server when negotiating an MSE or MP4
// stream. The go2rtc server recognizes only a fixed set of exact strings (one
// canonical spelling per codec, e.g. `avc1.640029` for all H.264) and silently
// ignores the rest; it then matches the camera's tracks by codec name alone,
// never comparing profile or level.
//
// So membership here is not correctness-critical: an unrecognized entry (a
// different H.264 level, AAC-HE) has no effect, and a camera streaming any
// level still matches. The list can stay a broad superset rather than tracking
// one server version's exact set.
//
// Order matters as the offer's preference order: video before audio, and audio
// most to least reliably supported.
//
// H.264/H.265 video, most-compatible first. Offered to every browser.
const GO2RTC_VIDEO_CODECS: readonly string[] = [
// H.264 high 4.1
'avc1.640029',
// H.264 high 4.2
'avc1.64002A',
// H.264 high 5.1
'avc1.640033',
// H.265 main 5.1
'hvc1.1.6.L153.B0',
];
// Audio is layered by the oldest Safari major version that can decode each
// codec via MSE (see `getCodecsForUserAgent`). Each tier extends the previous
// one with the next-most-reliable audio codec.
const GO2RTC_SAFARI_13_CODECS: readonly string[] = [
...GO2RTC_VIDEO_CODECS,
// AAC LC
'mp4a.40.2',
// AAC HE
'mp4a.40.5',
];
const GO2RTC_SAFARI_14_CODECS: readonly string[] = [
...GO2RTC_SAFARI_13_CODECS,
// FLAC
'flac',
];
// Full codec set: offered to non-Safari browsers.
export const GO2RTC_CODECS: readonly string[] = [
...GO2RTC_SAFARI_14_CODECS,
// Opus
'opus',
];
// Safari's `isTypeSupported` over-reports audio codec support, so offer only
// the codec set the running major version can *actually* play via MSE. The
// version boundaries (AAC from 13, FLAC from 14, OPUS never) are empirical
// browser-compatibility observations, not derived values -- treat them as a
// best-known table that may need revising for future Safari versions.
export const getCodecsForUserAgent = (userAgent: string): readonly string[] => {
const version = getSafariMajorVersion(userAgent);
if (version === null) {
return GO2RTC_CODECS;
}
if (version < 13) {
return GO2RTC_VIDEO_CODECS;
}
if (version < 14) {
return GO2RTC_SAFARI_13_CODECS;
}
return GO2RTC_SAFARI_14_CODECS;
};
interface CodecMediaSelection {
audio: boolean;
video: boolean;
}
// A codec is a video codec if it contains 'vc1' beyond position zero (avc1.*,
// hvc1.*); everything else in the go2rtc codec list is audio.
const isVideoCodec = (codec: string): boolean => codec.indexOf('vc1') > 0;
// Filter the codec list to the requested media kinds and to what the given
// support callback accepts.
export const selectSupportedCodecs = (
codecs: readonly string[],
media: CodecMediaSelection,
isSupported: (mimeType: string) => boolean,
): readonly string[] =>
codecs
.filter((codec) => (isVideoCodec(codec) ? media.video : media.audio))
.filter((codec) => isSupported(`video/mp4; codecs="${codec}"`));
// Encode a codec list as the comma-joined string go2rtc expects on the wire.
export const convertToCodecString = (codecs: readonly string[]): string =>
codecs.join(',');
@@ -0,0 +1,27 @@
import type { MediaUnavailableIssueReason } from '../../../../../card-controller/issues/issues/media-unavailable';
import type { StreamSourceFailureReason } from '../types';
// The card's media-unavailable causes are user-facing; a source's failure
// reasons are technical. Only a media error and a buffer overflow have no
// distinct user story, so they map to a generic playback error; the rest each
// keep a meaningful cause -- a server rejection, an unsupported stream, a failed
// two-way-audio call, or a timeout that means the stream never got going.
const FAILURE_TO_ISSUE_REASON: Record<
StreamSourceFailureReason,
MediaUnavailableIssueReason
> = {
two_way_audio_error: 'two_way_audio_error',
buffer_overflow: 'playback_error',
connect_timeout: 'not_loading',
media_error: 'playback_error',
negotiation_timeout: 'not_loading',
server_error: 'server_error',
unsupported: 'unsupported',
};
// A null reason is a connection-level failure with no source detail (e.g. the
// socket dropped), which reads as a generic playback error.
export const mapFailureReasonToIssueReason = (
reason: StreamSourceFailureReason | null,
): MediaUnavailableIssueReason =>
reason === null ? 'playback_error' : FAILURE_TO_ISSUE_REASON[reason];
@@ -0,0 +1,61 @@
// A GOP ("group of pictures") is the span from one keyframe to just before the
// next: a full keyframe followed by the smaller frames that only encode what
// changed since it. Streamed video arrives one GOP at a time, so a camera with
// a 2s keyframe interval delivers media in roughly 2s bursts. That cadence sets
// how far behind the live edge playback must sit to ride over the bursts
// without starving, so this file estimates the GOP length by timing the
// wall-clock interval between buffer advances (each advance is about one GOP of
// new media) and averaging it over a short rolling window.
//
// Measuring, rather than assuming a fixed length, is necessary here: the stream
// carries no declared GOP length (it's raw fragments arriving over a
// WebSocket), and camera GOPs vary widely -- a doorbell at ~1s, an NVR tuned
// for bandwidth at 4s+ -- so any single constant is either too small (high-GOP
// cameras stall) or needlessly laggy for everyone else. Measuring per stream
// right-sizes it without a configuration knob.
//
// This is necessary to adaptively calculate the buffer (and thus inflicted
// latency) to maintain for Safari, in order to avoid it "hitting the end" and
// thus pausing, requiring human intervention to play again.
// The assumed GOP length (seconds) until enough delivery cadence is measured,
// and how many recent cadence samples the rolling average spans.
const DEFAULT_GOP_SECONDS = 1;
export const GOP_SAMPLE_WINDOW_SIZE = 5;
// Estimates the GOP length (the delivery cadence) as a rolling average of the
// interval between buffer advances. A trim updates the buffer without growing
// it, so only advances are sampled.
export class GOPCadenceEstimator {
private _samples: number[] = [];
private _lastBufferedEnd: number | null = null;
private _lastAdvanceTime: Date | null = null;
public sample(bufferedEndSeconds: number, now: Date): void {
const advanced =
this._lastBufferedEnd === null || bufferedEndSeconds > this._lastBufferedEnd;
if (!advanced) {
return;
}
if (this._lastAdvanceTime !== null) {
const interval = (now.getTime() - this._lastAdvanceTime.getTime()) / 1000;
if (interval > 0) {
if (this._samples.length >= GOP_SAMPLE_WINDOW_SIZE) {
this._samples.shift();
}
this._samples.push(interval);
}
}
this._lastAdvanceTime = now;
this._lastBufferedEnd = bufferedEndSeconds;
}
public estimateSeconds(): number {
if (!this._samples.length) {
return DEFAULT_GOP_SECONDS;
}
return this._samples.reduce((sum, sample) => sum + sample, 0) / this._samples.length;
}
}
@@ -0,0 +1,22 @@
// Keeps MSE playback near the live edge. The strategy is browser-specific: see
// ./webkit.ts (seek-based, because WebKit stutters on playbackRate changes) and
// ./non-webkit.ts (playback-rate-based).
import { NonWebKitLiveEdgeStrategy } from './non-webkit';
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
import { WebKitLiveEdgeStrategy } from './webkit';
// Picks the browser-appropriate live-edge strategy and delegates to it.
export class LiveEdgeTracker {
private _strategy: LiveEdgeStrategy;
constructor(options: { webkit: boolean }) {
this._strategy = options.webkit
? new WebKitLiveEdgeStrategy()
: new NonWebKitLiveEdgeStrategy();
}
public next(status: LiveEdgeStatus): LiveEdgeAction {
return this._strategy.next(status);
}
}
@@ -0,0 +1,66 @@
// Everything that is not WebKit: keep playback near the live edge by nudging the
// playback rate (the approach WebKit cannot use; see ./webkit.ts).
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
// The lag (seconds) below which playback stays at realtime, how much the
// adaptive threshold scales with the stream's normal lag, the exponential rate
// curve's scale and steepness, and the rate ceiling.
const CATCH_UP_MIN_LAG_SECONDS = 3;
const CATCH_UP_AVERAGE_LAG_MULTIPLIER = 1.5;
const CATCH_UP_RATE_SCALE = 0.2;
const CATCH_UP_RATE_STEEPNESS = 0.5;
const CATCH_UP_MAX_RATE = 2;
// How many recent lag samples the rolling average spans; a tuning value trading
// responsiveness against noise, not a derived one.
export const LAG_SAMPLE_WINDOW_SIZE = 10;
// Nudge the playback rate up on an exponential curve that stays gentle for
// ordinary drift and only accelerates when playback is far behind, clamped so it
// never drops below realtime.
export class NonWebKitLiveEdgeStrategy implements LiveEdgeStrategy {
private _samples: number[] = [];
private _nextSampleIndex = 0;
public next(status: LiveEdgeStatus): LiveEdgeAction {
const lag = status.bufferedEndSeconds - status.currentTimeSeconds;
this._sampleLag(lag, status.playbackRate);
if (lag < CATCH_UP_MIN_LAG_SECONDS) {
return { action: 'rate', rate: 1 };
}
// The threshold adapts to the stream's normal lag (e.g. long keyframe
// intervals make a constant multi-second lag healthy), so the rate only
// rises when playback is genuinely falling behind.
const threshold = (this._averageLag() ?? 0) * CATCH_UP_AVERAGE_LAG_MULTIPLIER;
const rate = Math.min(
1 + CATCH_UP_RATE_SCALE * Math.exp(CATCH_UP_RATE_STEEPNESS * lag - threshold),
CATCH_UP_MAX_RATE,
);
return { action: 'rate', rate };
}
private _sampleLag(lagSeconds: number, playbackRate: number): void {
// Samples taken while deliberately catching up (rate above realtime and
// still far behind) would inflate the average the threshold derives from,
// so they are excluded.
if (playbackRate !== 1 && lagSeconds >= CATCH_UP_MIN_LAG_SECONDS) {
return;
}
if (this._samples.length < LAG_SAMPLE_WINDOW_SIZE) {
this._samples.push(lagSeconds);
} else {
this._samples[this._nextSampleIndex] = lagSeconds;
this._nextSampleIndex = (this._nextSampleIndex + 1) % LAG_SAMPLE_WINDOW_SIZE;
}
}
private _averageLag(): number | null {
if (!this._samples.length) {
return null;
}
return this._samples.reduce((sum, sample) => sum + sample, 0) / this._samples.length;
}
}
@@ -0,0 +1,17 @@
export type LiveEdgeAction =
| { action: 'none' }
| { action: 'rate'; rate: number }
| { action: 'seek'; seconds: number };
export interface LiveEdgeStatus {
bufferedEndSeconds: number;
currentTimeSeconds: number;
playbackRate: number;
now: Date;
}
// Given the current live-edge lag, returns the action needed to stay near the
// edge (a seek, a playback-rate nudge, or nothing).
export interface LiveEdgeStrategy {
next(status: LiveEdgeStatus): LiveEdgeAction;
}
@@ -0,0 +1,72 @@
// WebKit re-buffers whenever `playbackRate` changes, which in practice locks
// playback into 1fps slow-motion stutter (see
// https://github.com/dermotduffy/advanced-camera-card/issues/2450), so this
// strategy never touches the rate and manages position with seeks instead.
import { GOPCadenceEstimator } from './gop-cadence-estimator';
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
// How many GOPs behind the live edge to hold playback, and the band (seconds)
// that hold-back is clamped to so an anomalous GOP estimate cannot pin playback
// at the edge or strand it far behind live.
const HOLDBACK_GOP_MULTIPLIER = 3;
const MIN_HOLDBACK_SECONDS = 1.5;
const MAX_HOLDBACK_SECONDS = 8;
// How far past the hold-back (in GOPs) playback must fall before a forward
// catch-up seek, and the minimum gap between such seeks.
const CATCHUP_EXCESS_GOP_MULTIPLIER = 2;
const JUMP_COOLDOWN_SECONDS = 5;
// Hold playback a few GOPs behind the live edge. Sitting too close lets the
// playhead outrun the bursty per-keyframe delivery and reach the buffered end,
// at which point WebKit stalls into a pause a muted stream cannot auto-resume.
// The hold-back is sized to the measured GOP cadence, since staying under
// roughly three GOPs behind live lets a delivery gap drain the buffer and
// stall. Playback is seeked forward only when it has fallen well behind the
// hold-back, and back toward it when it drifts within one GOP of the edge.
export class WebKitLiveEdgeStrategy implements LiveEdgeStrategy {
private _cadence = new GOPCadenceEstimator();
private _lastJumpTime: Date | null = null;
public next(status: LiveEdgeStatus): LiveEdgeAction {
this._cadence.sample(status.bufferedEndSeconds, status.now);
const lag = status.bufferedEndSeconds - status.currentTimeSeconds;
const gop = this._cadence.estimateSeconds();
const holdback = this._holdbackSeconds(gop);
const target = status.bufferedEndSeconds - holdback;
// Playback has drifted within one GOP of the edge and is about to starve.
// WebKit cannot slow down with playbackRate to avoid it, so seek back to the
// hold-back to restore runway before it stalls into a pause. Not
// cooldown-gated: avoiding the stall takes priority over jump spacing.
if (lag < gop) {
return { action: 'seek', seconds: target };
}
// Playback has fallen well behind the hold-back (e.g. after a background
// tab): jump forward to the hold-back, rate-limited by a cooldown.
const secondsSinceLastJump =
this._lastJumpTime === null
? null
: (status.now.getTime() - this._lastJumpTime.getTime()) / 1000;
if (
lag > holdback + CATCHUP_EXCESS_GOP_MULTIPLIER * gop &&
(secondsSinceLastJump === null || secondsSinceLastJump >= JUMP_COOLDOWN_SECONDS)
) {
this._lastJumpTime = status.now;
return { action: 'seek', seconds: target };
}
return { action: 'none' };
}
private _holdbackSeconds(gopSeconds: number): number {
return Math.min(
Math.max(HOLDBACK_GOP_MULTIPLIER * gopSeconds, MIN_HOLDBACK_SECONDS),
MAX_HOLDBACK_SECONDS,
);
}
}
@@ -0,0 +1,9 @@
import type { Go2RTCMessage } from '../types';
// The go2rtc server reports a mode failure as `{ type: 'error', value: '<mode>: ...' }`
// (e.g. `mse: stream not found`), so an error is for a given mode when its value
// starts with that mode's name.
export const isServerErrorForMode = (message: Go2RTCMessage, mode: string): boolean =>
message.type === 'error' &&
typeof message.value === 'string' &&
message.value.startsWith(mode);
@@ -0,0 +1,31 @@
import type { Lane, StreamProfile } from '../types';
// Choose which lane to present when both a WebRTC stream and a binary-lane
// stream (MSE, MP4, MJPEG) are available for the same camera. The factors are
// compared in order of decreasing significance -- the first one that differs
// decides, regardless of the rest:
// 1. Has video -- a camera view needs a picture before anything else.
// 2. Has audio -- a complete stream over a silent one.
// 3. H.265 over H.264 -- better quality per bitrate.
// If all three are equal the tie goes to WebRTC, which plays as it arrives
// (lower latency for a live view) rather than from a buffer.
export const getPreferredSource = (
webrtc: StreamProfile,
binary: StreamProfile,
): Lane => {
// WebRTC audio is observed from live tracks; the binary side's audio comes
// from MSE's negotiated codec string, where only AAC plays reliably, so only
// AAC counts as audio on the binary side here (see StreamProfile).
const factors: [boolean, boolean][] = [
[webrtc.hasVideo, binary.hasVideo],
[webrtc.hasAudio, binary.hasAACAudio],
[webrtc.hasH265Video, binary.hasH265Video],
];
for (const [webRTCHas, binaryHas] of factors) {
if (webRTCHas !== binaryHas) {
return webRTCHas ? 'webrtc' : 'binary';
}
}
return 'webrtc';
};
@@ -0,0 +1,13 @@
// Whether the user agent runs the WebKit engine (Safari, and every browser or
// WebView on iOS regardless of branding, e.g. Chrome on iOS is `CriOS`).
// Blink-based browsers also advertise `AppleWebKit`, but are distinguishable by
// their `Chrome/`, `Chromium` or `Android` tokens.
export const isWebKitUserAgent = (userAgent: string): boolean =>
userAgent.includes('AppleWebKit') && !/Chrome\/|Chromium|Android/.test(userAgent);
// The Safari major version (from the `Version/<n>` token), or null when the user
// agent is not Safari.
export const getSafariMajorVersion = (userAgent: string): number | null => {
const match = userAgent.match(/Version\/(\d+).+Safari/);
return match ? Number(match[1]) : null;
};
@@ -0,0 +1,6 @@
// The SDP (Session Description Protocol) is the text blob describing a WebRTC
// session's media. go2rtc advertises an H.265 track with this rtpmap encoding
// name, so its presence in the answer SDP means the browser negotiated H.265 --
// preferred over H.264 when choosing between the WebRTC and MSE streams (see
// https://github.com/dermotduffy/advanced-camera-card/issues/2200).
export const sdpHasH265 = (sdp: string): boolean => sdp.includes('H265/90000');
@@ -1,5 +1,20 @@
import type { MediaUnavailableIssueReason } from '../../../card-controller/issues/issues/media-unavailable';
import { fireAdvancedCameraCardEvent } from '../../../utils/fire-advanced-camera-card-event';
export function dispatchLiveErrorEvent(element: EventTarget): void {
fireAdvancedCameraCardEvent(element, 'live:error');
declare global {
interface HTMLElementEventMap {
'advanced-camera-card:live:error': CustomEvent<
MediaUnavailableIssueReason | undefined
>;
}
}
// The optional reason lets a provider that knows why it failed drive a specific
// media-unavailable message; absent, the liveness detector falls back to a
// generic playback error.
export function dispatchLiveErrorEvent(
element: EventTarget,
reason?: MediaUnavailableIssueReason,
): void {
fireAdvancedCameraCardEvent(element, 'live:error', reason);
}
@@ -148,7 +148,7 @@ export class MediaActionsController {
}
}
private async _play(index: number): Promise<void> {
await (await this._children[index]?.getMediaPlayerController())?.play();
await (await this._children[index]?.getMediaPlayerController())?.playback?.play();
}
private async _unmuteTargetIfConfigured(
condition: AutoUnmuteCondition,
@@ -204,7 +204,7 @@ export class MediaActionsController {
}
}
private async _pause(index: number): Promise<void> {
await (await this._children[index]?.getMediaPlayerController())?.pause();
await (await this._children[index]?.getMediaPlayerController())?.playback?.pause();
}
private async _muteAllIfConfigured(condition: AutoMuteCondition): Promise<void> {
@@ -23,6 +23,11 @@ export interface FrameStallWatchdogConfig {
// Stop receiving frames. Defaults to a no-op, for a source that needs no
// teardown.
stopSource?: () => void;
// Seconds without a frame (while playback is expected) before a stall is
// reported. Defaults to FRAME_STALL_SECONDS; a slow source (e.g. a snapshot
// that refreshes every N seconds) needs a window at least as long as N.
stallAfterSeconds?: number;
}
/**
@@ -39,6 +44,7 @@ export interface FrameStallWatchdogConfig {
*/
export class FrameStallWatchdog {
private _config: FrameStallWatchdogConfig;
private _stallAfterSeconds: number;
private _timer = new Timer();
private _callbacks = new Set<LivenessCallback>();
@@ -51,6 +57,7 @@ export class FrameStallWatchdog {
constructor(config: FrameStallWatchdogConfig) {
this._config = config;
this._stallAfterSeconds = config.stallAfterSeconds ?? FRAME_STALL_SECONDS;
}
public subscribe(callback: LivenessCallback): UnsubscribeCallback {
@@ -74,7 +81,7 @@ export class FrameStallWatchdog {
if (!this._sourceActive) {
return;
}
this._timer.start(FRAME_STALL_SECONDS, () => this._onStall());
this._timer.start(this._stallAfterSeconds, () => this._onStall());
// Notify last: if this recovery notification prompts the final subscriber
// to unsubscribe, `_stop` then clears the timer just armed instead of
@@ -90,7 +97,7 @@ export class FrameStallWatchdog {
// watching begins is still detected. With no source there is nothing to
// arm, so nothing is ever reported.
if (this._sourceActive) {
this._timer.start(FRAME_STALL_SECONDS, () => this._onStall());
this._timer.start(this._stallAfterSeconds, () => this._onStall());
}
}
@@ -113,7 +120,7 @@ export class FrameStallWatchdog {
// Legitimately idle (paused / seeking / ended). Re-arm rather than stop: a
// source that later resumes already frozen delivers no frame to kick the
// timer, so a freeze that only becomes actionable later is still caught.
this._timer.start(FRAME_STALL_SECONDS, () => this._onStall());
this._timer.start(this._stallAfterSeconds, () => this._onStall());
}
private _setLive(isLive: boolean): void {
+110 -24
View File
@@ -1,31 +1,102 @@
import type { LitElement } from 'lit';
import type { FullscreenElement, MediaPlayerController, PIPElement } from '../../types';
import type {
FullscreenElement,
LivenessCallback,
MediaPlayerController,
PIPElement,
PlaybackControl,
UnsubscribeCallback,
} from '../../types';
import { screenshotImage } from '../../utils/screenshot';
import { FrameStallWatchdog } from './frame-stall-watchdog';
// A pausable update loop the controller owns (e.g. an image refreshed on a
// timer). Its presence makes the image player pausable.
export interface ImageUpdateControl {
start(): void;
stop(): void;
isRunning(): boolean;
}
// Liveness for an image stream: each <img> `load` is a frame; a gap longer than
// the window while frames are expected is a stall. `stallWindowSeconds`
// defaults to the standard frame-stall window (suits a push-fed stream); a
// timer-refreshed image may set a different one, should be at least its refresh
// interval.
interface ImageLivenessOptions {
isFrameExpected: () => boolean;
stallWindowSeconds?: number;
}
// Obtaining a screenshot. Defaults to drawing the current <img>; an image
// refreshed on a timer can hand back its cached URL instead.
type ImageScreenshotProvider = () => Promise<string | null>;
interface ImageMediaPlayerControllerOptions {
updateControl?: ImageUpdateControl;
livenessOptions?: ImageLivenessOptions;
screenshotProvider?: ImageScreenshotProvider;
}
// Image player composed from opt-in capabilities.
export class ImageMediaPlayerController implements MediaPlayerController {
private _host: LitElement;
private _getImageCallback: () => HTMLImageElement | null;
private _screenshotProvider: ImageScreenshotProvider | null;
constructor(host: LitElement, getImageCallback: () => HTMLImageElement | null) {
private _stallWatchdog: FrameStallWatchdog | null = null;
private _loadListener: (() => void) | null = null;
public readonly playback?: PlaybackControl;
public readonly subscribeLiveness?: (
callback: LivenessCallback,
) => UnsubscribeCallback;
constructor(
host: LitElement,
getImageCallback: () => HTMLImageElement | null,
options?: ImageMediaPlayerControllerOptions,
) {
this._host = host;
this._getImageCallback = getImageCallback;
}
this._screenshotProvider = options?.screenshotProvider ?? null;
public async play(): Promise<void> {
// Not implemented.
}
const updateControl = options?.updateControl;
if (updateControl) {
this.playback = {
play: async (): Promise<void> => {
await this._host.updateComplete;
updateControl.start();
},
pause: async (): Promise<void> => {
await this._host.updateComplete;
updateControl.stop();
},
isPaused: (): boolean => !updateControl.isRunning(),
};
}
public async pause(): Promise<void> {
// Not implemented.
const livenessOptions = options?.livenessOptions;
if (livenessOptions) {
const stallWatchdog = new FrameStallWatchdog({
isPlaybackExpected: livenessOptions.isFrameExpected,
stallAfterSeconds: livenessOptions.stallWindowSeconds,
startSource: () => this._startFrameSource(),
stopSource: () => this._stopFrameSource(),
});
this._stallWatchdog = stallWatchdog;
this.subscribeLiveness = (callback): UnsubscribeCallback =>
stallWatchdog.subscribe(callback);
}
}
public async mute(): Promise<void> {
// Not implemented.
// No audio.
}
public async unmute(): Promise<void> {
// Not implemented.
// No audio.
}
public isMuted(): boolean {
@@ -33,26 +104,17 @@ export class ImageMediaPlayerController implements MediaPlayerController {
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async seek(_seconds: number): Promise<void> {
// Not implemented.
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async setControls(_controls: boolean): Promise<void> {
// Not implemented.
}
public isPaused(): boolean {
// The image could be an MJPEG, so it is always reported unpaused.
return false;
public async setControls(_controls?: boolean): Promise<void> {
// No playback controls to show.
}
public async getScreenshotURL(): Promise<string | null> {
await this._host.updateComplete;
if (this._screenshotProvider) {
return this._screenshotProvider();
}
const image = this._getImageCallback();
// It might an MJPEG so still need to screenshot it.
return image ? screenshotImage(image) : null;
}
@@ -61,6 +123,30 @@ export class ImageMediaPlayerController implements MediaPlayerController {
}
public getPIPElement(): PIPElement | null {
// Picture-in-picture is video-only.
return null;
}
// The frame source is the image's `load` event: each newly displayed frame
// fires it. With no image element there is nothing to observe, so the watchdog
// is told there is no source and reports no stall.
private _startFrameSource(): boolean {
const image = this._getImageCallback();
if (!image) {
return false;
}
this._loadListener = (): void => {
this._stallWatchdog?.notifyFrame();
};
image.addEventListener('load', this._loadListener);
return true;
}
private _stopFrameSource(): void {
const image = this._getImageCallback();
if (image && this._loadListener) {
image.removeEventListener('load', this._loadListener);
}
this._loadListener = null;
}
}
+17 -18
View File
@@ -6,6 +6,7 @@ import type {
LivenessCallback,
MediaPlayerController,
PIPElement,
PlaybackControl,
UnsubscribeCallback,
} from '../../types';
import { FrameStallWatchdog } from './frame-stall-watchdog';
@@ -20,7 +21,7 @@ export class JSMPEGMediaPlayerController implements MediaPlayerController {
// watchdog's always-available defaults apply. Playback is expected whenever
// not paused; a decode gap for the frame-stall window is then a stall.
private _stallWatchdog = new FrameStallWatchdog({
isPlaybackExpected: () => !this.isPaused(),
isPlaybackExpected: () => !this.playback.isPaused(),
});
constructor(
@@ -42,15 +43,22 @@ export class JSMPEGMediaPlayerController implements MediaPlayerController {
this._stallWatchdog.notifyFrame();
}
public async play(): Promise<void> {
await this._host.updateComplete;
return this._getJSMPEGVideoElementCallback()?.play();
}
// JSMpeg decodes a live stream it can start and stop.
public readonly playback: PlaybackControl = {
play: async (): Promise<void> => {
await this._host.updateComplete;
return this._getJSMPEGVideoElementCallback()?.play();
},
public async pause(): Promise<void> {
await this._host.updateComplete;
return this._getJSMPEGVideoElementCallback()?.stop();
}
pause: async (): Promise<void> => {
await this._host.updateComplete;
return this._getJSMPEGVideoElementCallback()?.stop();
},
isPaused: (): boolean => {
return this._getJSMPEGVideoElementCallback()?.player?.paused ?? true;
},
};
public async mute(): Promise<void> {
await this._host.updateComplete;
@@ -73,20 +81,11 @@ export class JSMPEGMediaPlayerController implements MediaPlayerController {
return player ? player.volume === 0 : true;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async seek(_seconds: number): Promise<void> {
// JSMPEG does not support seeking.
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async setControls(_controls: boolean): Promise<void> {
// Not implemented.
}
public isPaused(): boolean {
return this._getJSMPEGVideoElementCallback()?.player?.paused ?? true;
}
public async getScreenshotURL(): Promise<string | null> {
await this._host.updateComplete;
return this._getCanvasElementCallback()?.toDataURL('image/jpeg') ?? null;
@@ -1,69 +0,0 @@
import type { LitElement } from 'lit';
import type { FullscreenElement, MediaPlayerController, PIPElement } from '../../types';
import type { CachedValueController } from '../cached-value-controller';
export class UpdatingImageMediaPlayerController implements MediaPlayerController {
private _host: LitElement;
private _getImageCallback: () => HTMLImageElement | null;
private _getCachedValueController: () => CachedValueController<string> | null;
constructor(
host: LitElement,
getImageCallback: () => HTMLImageElement | null,
getCachedValueController: () => CachedValueController<string> | null,
) {
this._host = host;
this._getImageCallback = getImageCallback;
this._getCachedValueController = getCachedValueController;
}
public async play(): Promise<void> {
await this._host.updateComplete;
this._getCachedValueController()?.startTimer();
}
public async pause(): Promise<void> {
await this._host.updateComplete;
this._getCachedValueController()?.stopTimer();
}
public async mute(): Promise<void> {
// Not implemented.
}
public async unmute(): Promise<void> {
// Not implemented.
}
public isMuted(): boolean {
return true;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async seek(_seconds: number): Promise<void> {
// Not implemented.
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async setControls(_controls: boolean): Promise<void> {
// Not implemented.
}
public isPaused(): boolean {
return !this._getCachedValueController()?.hasTimer();
}
public async getScreenshotURL(): Promise<string | null> {
await this._host.updateComplete;
return this._getCachedValueController()?.getValue() ?? null;
}
public getFullscreenElement(): FullscreenElement | null {
return this._getImageCallback() ?? null;
}
public getPIPElement(): PIPElement | null {
return null;
}
}
+43 -33
View File
@@ -5,6 +5,7 @@ import type {
LivenessCallback,
MediaPlayerController,
PIPElement,
PlaybackControl,
UnsubscribeCallback,
} from '../../types';
import { hideMediaControlsTemporarily, setControlsOnVideo } from '../../utils/controls';
@@ -19,15 +20,22 @@ export class VideoMediaPlayerController implements MediaPlayerController {
private _rvfcHandle: number | null = null;
private _stallWatchdog = new FrameStallWatchdog({
// Playback is expected unless the video is legitimately idle. Seeking /
// ended is idle. A paused video is idle only if it holds a current frame --
// a genuine user pause; paused with no current frame is not a real pause but
// a source mid-reconnect or buffering (nothing to pause on), so playback is
// still expected and a missing frame is a stall.
// ended is idle. A poster shown with no media loaded is a still-image
// surface (e.g. an MJPEG/MP4 poster slideshow) that never presents video
// frames, so no frame is ever expected -- distinct from a poster shown over
// real media (a loading placeholder), which is still watched. A paused
// video is idle only if it holds a current frame -- a genuine user pause;
// paused with no current frame is not a real pause but a source
// mid-reconnect or buffering (nothing to pause on), so playback is still
// expected and a missing frame is a stall.
isPlaybackExpected: () => {
const video = this._getVideoCallback();
if (!video || video.seeking || video.ended) {
return false;
}
if (video.poster && !video.currentSrc && !video.srcObject) {
return false;
}
return !video.paused || video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA;
},
startSource: () => this._startFrameSource(),
@@ -44,36 +52,42 @@ export class VideoMediaPlayerController implements MediaPlayerController {
this._getControlsDefaultCallback = getControlsDefaultCallback ?? null;
}
public async play(): Promise<void> {
await this._host.updateComplete;
public readonly playback: PlaybackControl = {
play: async (): Promise<void> => {
await this._host.updateComplete;
const video = this._getVideoCallback();
if (!video?.play) {
return;
}
const video = this._getVideoCallback();
if (!video?.play) {
return;
}
// If the play call fails, and the media is not already muted, mute it first
// and then try again. This works around some browsers that prevent
// auto-play unless the video is muted.
try {
await video.play();
} catch (err: unknown) {
if ((err as Error).name === 'NotAllowedError' && !this.isMuted()) {
await this.mute();
try {
await video.play();
// eslint-disable-next-line @typescript-eslint/no-unused-vars
} catch (e) {
// Pass.
// If the play call fails, and the media is not already muted, mute it
// first and then try again. This works around some browsers that prevent
// auto-play unless the video is muted.
try {
await video.play();
} catch (err: unknown) {
if ((err as Error).name === 'NotAllowedError' && !this.isMuted()) {
await this.mute();
try {
await video.play();
// eslint-disable-next-line @typescript-eslint/no-unused-vars
} catch (e) {
// Pass.
}
}
}
}
}
},
public async pause(): Promise<void> {
await this._host.updateComplete;
this._getVideoCallback()?.pause();
}
pause: async (): Promise<void> => {
await this._host.updateComplete;
this._getVideoCallback()?.pause();
},
isPaused: (): boolean => {
return this._getVideoCallback()?.paused ?? true;
},
};
public async mute(): Promise<void> {
await this._host.updateComplete;
@@ -119,10 +133,6 @@ export class VideoMediaPlayerController implements MediaPlayerController {
}
}
public isPaused(): boolean {
return this._getVideoCallback()?.paused ?? true;
}
public async getScreenshotURL(): Promise<string | null> {
await this._host.updateComplete;
+1 -1
View File
@@ -713,7 +713,7 @@ export class MenuButtonController {
currentMediaLoadedInfo.mediaPlayerController &&
currentMediaLoadedInfo.capabilities?.supportsPause
) {
const paused = currentMediaLoadedInfo.mediaPlayerController?.isPaused();
const paused = currentMediaLoadedInfo.mediaPlayerController?.playback?.isPaused();
return {
icon: paused ? 'mdi:play' : 'mdi:pause',
...config.menu.buttons.play,
+52
View File
@@ -0,0 +1,52 @@
import type { Notification } from '../../config/schema/actions/types.js';
import { TROUBLESHOOTING_URL } from '../../const.js';
import { localize } from '../../localize/localize.js';
// A narrower type (than Notification) for UX consistency across the notifications
// shown over a media surface: provider errors, the awaiting-live placeholder, and
// the viewer/gallery no-media state.
export interface MediaNotificationOptions {
// A short heading. A longer explanation goes in `detail`.
title: string;
// The heading icon. Defaults to a generic alert icon.
icon?: string;
// Appended to the heading to identify the media, when it has a title (e.g. the
// camera title `: Front Door`). Absent for untitled media (a url or
// screensaver image).
targetTitle?: string;
// A longer explanation, shown as the body when the title alone is not enough.
detail?: string;
// Whether to show the retry spinner. A failing media surface is retried, so
// this defaults to true.
inProgress?: boolean;
// Whether to show the troubleshooting link. Defaults to true.
troubleshooting?: boolean;
}
// The standard notification block shown over a media surface (a camera stream, a
// non-camera url/screensaver image, or an empty viewer/gallery): a short titled
// heading (with the media title when there is one), an optional longer detail, an
// optional troubleshooting link, and a retry spinner.
export const createMediaNotification = (
options: MediaNotificationOptions,
): Notification => ({
heading: {
icon: options.icon ?? 'mdi:alert-circle',
text: options.targetTitle
? `${options.title}: ${options.targetTitle}`
: options.title,
},
...(options.detail && { body: { text: options.detail } }),
...((options.troubleshooting ?? true) && {
link: {
url: TROUBLESHOOTING_URL,
title: localize('error.troubleshooting'),
},
}),
...((options.inProgress ?? true) && { in_progress: true }),
});
+9 -12
View File
@@ -10,6 +10,7 @@ import {
} from '../ha/web-proxy.js';
import type { Endpoint } from '../types.js';
import { errorToConsole } from '../utils/basic.js';
import { Generation } from '../utils/concurrency/generation.js';
const PROXY_URL_SIGN_EXPIRY_SECONDS = 24 * 60 * 60;
@@ -40,11 +41,11 @@ export class SignedURLController implements ReactiveController {
// Caching and race-condition state.
// The targetURL and proxy config are tracked to detect when inputs change and
// invalidate the cache. The requestID tracks the most recent valid fetch, to
// ensure that older, slower in-flight requests do not overwrite newer ones.
// invalidate the cache. The request generation tracks the most recent valid
// fetch, so older, slower in-flight requests do not overwrite newer ones.
private _targetURL: string | null = null;
private _targetProxyConfig: EnabledProxyConfig | null = null;
private _requestID = 0;
private _requestGeneration = new Generation();
constructor(
host: ReactiveControllerHost,
@@ -74,7 +75,7 @@ export class SignedURLController implements ReactiveController {
}
public hostDisconnected(): void {
++this._requestID;
this._requestGeneration.invalidate();
this._value = null;
this._error = null;
this._cachedAt = null;
@@ -88,7 +89,7 @@ export class SignedURLController implements ReactiveController {
if (!hass || !endpoint || (!proxyConfig?.enabled && !endpoint.sign)) {
// Invalidate any in-flight async work so a stale proxy/sign result cannot
// repopulate the controller after inputs have been cleared or disabled.
++this._requestID;
this._requestGeneration.invalidate();
this._value = null;
this._error = null;
this._targetURL = null;
@@ -137,7 +138,7 @@ export class SignedURLController implements ReactiveController {
// Mark as in-flight so the `!this._cachedAt` guard above prevents
// subsequent hostUpdate() calls from restarting the async work.
this._cachedAt = null;
const requestID = ++this._requestID;
const requestID = this._requestGeneration.next();
const resolvedEndpoint = await this._proxy(
hass,
@@ -146,7 +147,7 @@ export class SignedURLController implements ReactiveController {
proxyConfig,
proxyEndpointOptions,
);
if (this._isStale(requestID)) {
if (!this._requestGeneration.isCurrent(requestID)) {
return;
}
if (!resolvedEndpoint) {
@@ -155,7 +156,7 @@ export class SignedURLController implements ReactiveController {
}
const signedURL = await this._sign(hass, resolvedEndpoint);
if (this._isStale(requestID)) {
if (!this._requestGeneration.isCurrent(requestID)) {
return;
}
if (!signedURL) {
@@ -213,10 +214,6 @@ export class SignedURLController implements ReactiveController {
}
}
private _isStale(requestID: number): boolean {
return this._requestID !== requestID;
}
private _applySuccess(url: string): void {
this._value = url;
this._error = null;
+8 -6
View File
@@ -38,7 +38,7 @@ import { getReviewedQueryFilterFromQuery } from '../../view/utils/query-filter.j
import '../media-filter.js';
import { renderNoMedia } from '../notification/no-media.js';
import { renderNoMediaNotification } from '../notification/media.js';
import '../surround-basic.js';
import '../thumbnail/thumbnail.js';
@@ -219,11 +219,13 @@ export class AdvancedCameraCardGallery extends LitElement {
</advanced-camera-card-media-filter>`
: ''}
${!hasItems
? renderNoMedia({
cameraID: this.viewManagerEpoch?.manager.getView()?.camera ?? null,
cameraManager: this.cameraManager ?? null,
loading: isLoading,
})
? renderNoMediaNotification(
{
cameraID: this.viewManagerEpoch?.manager.getView()?.camera ?? null,
inProgress: isLoading,
},
this.cameraManager,
)
: html`<advanced-camera-card-gallery-core
.hass=${this.hass}
.columnWidth=${this._controller.getColumnWidth(
+56 -20
View File
@@ -12,16 +12,17 @@ import { live } from 'lit/directives/live.js';
import { createRef, ref, type Ref } from 'lit/directives/ref.js';
import { getCameraEntityFromConfig } from '../camera-manager/utils/camera-entity-from-config.js';
import type { MediaUnavailableIssueReason } from '../card-controller/issues/issues/media-unavailable.js';
import type { IssueTriggerEventData } from '../card-controller/issues/types.js';
import { CachedValueController } from '../components-lib/cached-value-controller.js';
import { MediaLoadedInfoSourceController } from '../components-lib/media-loaded-info-source-controller.js';
import { UpdatingImageMediaPlayerController } from '../components-lib/media-player/updating-image.js';
import { ImageMediaPlayerController } from '../components-lib/media-player/image.js';
import { createMediaNotification } from '../components-lib/notification/media.js';
import { SignedURLController } from '../components-lib/signed-url-controller.js';
import type { Notification } from '../config/schema/actions/types.js';
import type { CameraConfig } from '../config/schema/cameras.js';
import { type ImageBaseConfig, type ImageMode } from '../config/schema/common/image.js';
import type { EnabledProxyConfig } from '../config/schema/common/proxy.js';
import { TROUBLESHOOTING_URL } from '../const.js';
import { isHassDifferent } from '../ha/is-hass-different.js';
import type { HomeAssistant } from '../ha/types.js';
import defaultImage from '../images/iris-screensaver.jpg';
@@ -38,6 +39,14 @@ import {
import type { View } from '../view/view.js';
import { renderNotificationBlock } from './notification/block.js';
declare global {
interface HTMLElementEventMap {
// A private signal to the immediate parent that the media failed.
// Non-bubbling.
'advanced-camera-card:image-updating-player:error': CustomEvent<MediaUnavailableIssueReason>;
}
}
// See TOKEN_CHANGE_INTERVAL in https://github.com/home-assistant/core/blob/dev/homeassistant/components/camera/__init__.py .
const HASS_REJECTION_CUTOFF_MS = 5 * 60 * 1000;
@@ -91,6 +100,10 @@ export class AdvancedCameraCardImageUpdatingPlayer
@property({ attribute: false })
public targetID?: string;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
@property({ attribute: false, hasChanged: contentsChanged })
public proxyConfig?: EnabledProxyConfig;
@@ -103,6 +116,10 @@ export class AdvancedCameraCardImageUpdatingPlayer
@state()
private _imageLoadError = false;
// Tracks the signed/proxy error so it is reported once on the transition into
// failure, not on every update.
private _hasSignError = false;
private _refImage: Ref<HTMLImageElement> = createRef();
private _cachedValueController = new CachedValueController(
@@ -144,10 +161,19 @@ export class AdvancedCameraCardImageUpdatingPlayer
private _boundVisibilityHandler = this._visibilityHandler.bind(this);
private _mediaPlayerController = new UpdatingImageMediaPlayerController(
// A poll-refreshed snapshot: the cached-value timer is the pausable update
// loop, and its cached URL is the screenshot.
private _mediaPlayerController = new ImageMediaPlayerController(
this,
() => this._refImage.value ?? null,
() => this._cachedValueController,
{
updateControl: {
start: () => this._cachedValueController.startTimer(),
stop: () => this._cachedValueController.stopTimer(),
isRunning: () => this._cachedValueController.hasTimer(),
},
screenshotProvider: async () => this._cachedValueController.getValue(),
},
);
private _mediaLoadedInfoSourceController = new MediaLoadedInfoSourceController(this, {
@@ -218,6 +244,19 @@ export class AdvancedCameraCardImageUpdatingPlayer
if (!this._cachedValueController?.getValue()) {
this._cachedValueController?.updateValue();
}
const hasSignError = !!this._signedURLController.getError();
if (hasSignError && !this._hasSignError) {
this._dispatchError('server_error');
}
this._hasSignError = hasSignError;
}
private _dispatchError(reason: MediaUnavailableIssueReason): void {
fireAdvancedCameraCardEvent(this, 'image-updating-player:error', reason, {
bubbles: false,
composed: false,
});
}
/**
@@ -411,24 +450,16 @@ export class AdvancedCameraCardImageUpdatingPlayer
private _getDisplayNotification(): Notification | null {
const error = this._signedURLController.getError();
if (error) {
return {
heading: {
text: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
icon: 'mdi:alert-circle',
},
link: { url: TROUBLESHOOTING_URL, title: localize('error.troubleshooting') },
context: this.proxyConfig ? [this.proxyConfig] : undefined,
};
return createMediaNotification({
title: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
targetTitle: this.cameraTitle,
});
}
if (this._imageLoadError) {
return {
heading: {
text: localize('error.image_load_error'),
icon: 'mdi:alert-circle',
},
link: { url: TROUBLESHOOTING_URL, title: localize('error.troubleshooting') },
context: this.imageConfig ? [this.imageConfig] : undefined,
};
return createMediaNotification({
title: localize('error.image_load_error'),
targetTitle: this.cameraTitle,
});
}
return null;
}
@@ -468,6 +499,11 @@ export class AdvancedCameraCardImageUpdatingPlayer
this._forceSafeImage(true);
} else if (mode === 'url') {
this._imageLoadError = true;
// Report the failure to the parent. A live context marks the
// stream not-live so its wrapper stops covering the error with a
// loading overlay; the plain image view ignores it.
this._dispatchError('not_loading');
}
if (this.targetID) {
fireAdvancedCameraCardEvent<IssueTriggerEventData>(
+2 -1
View File
@@ -275,7 +275,7 @@ export class AdvancedCameraCardLiveCarousel extends LitElement {
.microphoneStream=${microphoneStream}
.camera=${resolvedCamera}
.targetID=${cameraID}
.label=${cameraMetadata?.title ?? ''}
.cameraTitle=${cameraMetadata?.title}
.liveConfig=${this.liveConfig}
.hass=${this.hass}
.stateWatcher=${this.stateWatcher}
@@ -284,6 +284,7 @@ export class AdvancedCameraCardLiveCarousel extends LitElement {
.zoom=${!this._isGesturesPTZActive(view, cameraID)}
.forceSelected=${isSelectedSlide}
.locked=${this.locked}
.suppressLoadingImage=${mediaEpoch > 0}
@advanced-camera-card:zoom:change=${(
ev: CustomEvent<ZoomSettingsObserved>,
) =>
+94 -37
View File
@@ -34,7 +34,7 @@ import { getResolvedLiveProvider } from '../../utils/live-provider.js';
import '../icon.js';
import { renderNotificationBlockFromText } from '../notification/block.js';
import { renderMediaNotification } from '../notification/media.js';
import './../media-dimensions-container';
@@ -56,9 +56,10 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
@property({ attribute: false })
public liveConfig?: LiveConfig;
// Label that is used for ARIA support and as tooltip.
// The camera's title, used for ARIA support, as tooltip, and to identify the
// camera in error messages.
@property({ attribute: false })
public label = '';
public cameraTitle?: string;
@property({ attribute: false })
public cardWideConfig?: CardWideConfig;
@@ -85,6 +86,12 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
@property({ attribute: false })
public locked?: boolean;
// When true, suppress the loading snapshot (show_image_during_load). Set on a
// media reload after a failure so the snapshot doesn't flash back in on every
// retry; a first load still shows it.
@property({ attribute: false })
public suppressLoadingImage = false;
private _mediaLoadedInfoSinkController = new MediaLoadedInfoSinkController(this, {
getTargetID: () => this.targetID ?? null,
});
@@ -127,6 +134,7 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
*/
private _shouldShowImageDuringLoading(): boolean {
return (
!this.suppressLoadingImage &&
!this._mediaLoadedInfoSinkController.has() &&
!!this.camera?.getConfig()?.camera_entity &&
!!this.hass &&
@@ -167,6 +175,8 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
this._importPromises.push(import('./providers/image.js'));
} else if (provider === 'go2rtc') {
this._importPromises.push(import('./providers/go2rtc/index.js'));
} else if (provider === 'go2rtc-experimental') {
this._importPromises.push(import('./providers/go2rtc-experimental/index.js'));
}
}
}
@@ -239,57 +249,70 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
// being initialized. This can cause spurious errors (e.g. lack of resolved
// endpoints). Instead, simply never render uninitialized cameras.
if (!this.camera.isInitialized()) {
return renderNotificationBlockFromText(
`${localize('error.awaiting_live')}${this.label ? `: ${this.label}` : ''}`,
{ icon: 'mdi:progress-helper', in_progress: true },
);
return renderMediaNotification({
icon: 'mdi:progress-helper',
title: localize('error.awaiting_live'),
targetTitle: this.cameraTitle,
});
}
// Set title and ariaLabel from the provided label property.
this.title = this.label;
this.ariaLabel = this.label;
this.title = this.cameraTitle ?? '';
this.ariaLabel = this.cameraTitle ?? '';
const provider = getResolvedLiveProvider(this.camera?.getConfig());
// `ha`/`image` cannot stream without a camera entity, so validate that
// here. Entity *availability* (including the always_error immediate path)
// is owned by the liveness controller's EntityAvailabilityDetector and
// surfaces via getPlaceholder() below, for all providers.
// surfaces via getFailure() below, for all providers.
if (
provider === 'ha' ||
provider === 'image' ||
(cameraConfig?.camera_entity && cameraConfig.always_error_if_entity_unavailable)
) {
if (!cameraConfig?.camera_entity) {
return renderNotificationBlockFromText(localize('error.no_live_camera'), {
return renderMediaNotification({
icon: 'mdi:camera',
context: cameraConfig,
title: localize('error.configuration_error'),
detail: localize('error.no_live_camera'),
targetTitle: this.cameraTitle,
});
}
if (!this.hass.states[cameraConfig.camera_entity]) {
return renderNotificationBlockFromText(localize('error.live_camera_not_found'), {
return renderMediaNotification({
icon: 'mdi:camera',
context: cameraConfig,
title: localize('error.configuration_error'),
detail: localize('error.live_camera_not_found'),
targetTitle: this.cameraTitle,
});
}
}
const failure = this._streamLivenessController.getFailure();
// A detector reports the stream is silently lost (the camera entity is
// unavailable, or the stream stalled): render a reconnecting placeholder,
// which unmounts the provider and unloads it via the existing media-loaded
// abort. The message names the specific cause.
const placeholder = this._streamLivenessController.getPlaceholder();
if (placeholder) {
if (failure?.renderPlaceholder) {
const { localizationKey: textKey, icon } =
MEDIA_UNAVAILABLE_REASONS[placeholder.reason];
return renderNotificationBlockFromText(
`${localize(textKey)}${this.label ? `: ${this.label}` : ''}`,
{ icon, in_progress: true },
);
MEDIA_UNAVAILABLE_REASONS[failure.reason];
return renderMediaNotification({
icon,
title: localize(textKey),
targetTitle: this.cameraTitle,
});
}
const showImageDuringLoading = this._shouldShowImageDuringLoading();
const showLoadingIcon = !this._mediaLoadedInfoSinkController.has();
const mediaLoaded = this._mediaLoadedInfoSinkController.has();
// Loaded media or a snapshot gives the frame a size; mark the host `sized`
// when one is present. In its absence CSS reserves an aspect ratio so the
// frame (whose loading/error fill is absolutely positioned) doesn't
// collapse.
this.toggleAttribute('sized', mediaLoaded || showImageDuringLoading);
const classes = {
hidden: showImageDuringLoading,
@@ -302,6 +325,7 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
.hass=${this.hass}
.camera=${this.camera}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
class=${classMap({
...classes,
// The image provider is providing the temporary loading image,
@@ -340,42 +364,75 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
.hass=${this.hass}
.camera=${this.camera}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
.microphoneStream=${this.microphoneStream}
.microphoneConfig=${this.liveConfig.microphone}
?controls=${this._getEffectiveBuiltinControls()}
>
</advanced-camera-card-live-go2rtc>`
: provider === 'webrtc-card'
? html`<advanced-camera-card-live-webrtc-card
: provider === 'go2rtc-experimental'
? html`<advanced-camera-card-live-go2rtc-experimental
${ref(this._refProvider)}
class=${classMap(classes)}
.hass=${this.hass}
.camera=${this.camera}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
.microphoneStream=${this.microphoneStream}
.microphoneConfig=${this.liveConfig.microphone}
.cardWideConfig=${this.cardWideConfig}
?controls=${this._getEffectiveBuiltinControls()}
>
</advanced-camera-card-live-webrtc-card>`
: provider === 'jsmpeg'
? html` <advanced-camera-card-live-jsmpeg
</advanced-camera-card-live-go2rtc-experimental>`
: provider === 'webrtc-card'
? html`<advanced-camera-card-live-webrtc-card
${ref(this._refProvider)}
class=${classMap(classes)}
.hass=${this.hass}
.camera=${this.camera}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
.cardWideConfig=${this.cardWideConfig}
?controls=${this._getEffectiveBuiltinControls()}
>
</advanced-camera-card-live-jsmpeg>`
: html``}
</advanced-camera-card-live-webrtc-card>`
: provider === 'jsmpeg'
? html` <advanced-camera-card-live-jsmpeg
${ref(this._refProvider)}
class=${classMap(classes)}
.hass=${this.hass}
.camera=${this.camera}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
.cardWideConfig=${this.cardWideConfig}
>
</advanced-camera-card-live-jsmpeg>`
: html``}
`)}
${showLoadingIcon
? html`<advanced-camera-card-icon
title=${localize('error.awaiting_live')}
.icon=${{ icon: 'mdi:progress-helper' }}
@click=${() =>
fireAdvancedCameraCardEvent(this, 'issue:notify', 'media_unavailable')}
></advanced-camera-card-icon>`
: ''}`;
${failure || mediaLoaded ? '' : this._renderLoadingOverlay(showImageDuringLoading)}`;
}
// The loading status drawn on top of the mounted provider while its media has
// not loaded: a subtle corner spinner over a snapshot that is already filling
// the frame, or a full "waiting for live" state. The cases that render nothing
// (a failure, or media already loaded) are handled at the call site.
private _renderLoadingOverlay(showImageDuringLoading: boolean): TemplateResult {
if (showImageDuringLoading) {
return html`<advanced-camera-card-icon
title=${localize('error.awaiting_live')}
.icon=${{ icon: 'mdi:progress-helper' }}
@click=${() =>
fireAdvancedCameraCardEvent(this, 'issue:notify', 'media_unavailable')}
></advanced-camera-card-icon>`;
}
return html`<div class="fill">
${renderMediaNotification({
icon: 'mdi:progress-helper',
title: localize('error.awaiting_live'),
targetTitle: this.cameraTitle,
})}
</div>`;
}
static get styles(): CSSResultGroup {
@@ -0,0 +1,299 @@
import {
html,
LitElement,
unsafeCSS,
type CSSResultGroup,
type PropertyValues,
type TemplateResult,
} from 'lit';
import { customElement, property, state } from 'lit/decorators.js';
import { createRef, ref, type Ref } from 'lit/directives/ref.js';
import type { Camera } from '../../../../camera-manager/camera.js';
import {
MEDIA_UNAVAILABLE_REASONS,
type MediaUnavailableIssueReason,
} from '../../../../card-controller/issues/issues/media-unavailable.js';
import { ImageSurfaceController } from '../../../../components-lib/live/providers/go2rtc-experimental/image-surface-controller.js';
import {
Go2RTCSessionController,
type SessionSurfaces,
type VideoSurface,
} from '../../../../components-lib/live/providers/go2rtc-experimental/session-controller.js';
import type { SurfaceKind } from '../../../../components-lib/live/providers/go2rtc-experimental/types.js';
import { mapFailureReasonToIssueReason } from '../../../../components-lib/live/providers/go2rtc-experimental/utils/failure-reason.js';
import { dispatchLiveErrorEvent } from '../../../../components-lib/live/utils/dispatch-live-error.js';
import { MediaLoadedInfoSourceController } from '../../../../components-lib/media-loaded-info-source-controller.js';
import { VideoMediaPlayerController } from '../../../../components-lib/media-player/video.js';
import { SignedURLController } from '../../../../components-lib/signed-url-controller.js';
import type { MicrophoneConfig } from '../../../../config/schema/live.js';
import type { CardWideConfig } from '../../../../config/schema/types.js';
import type { HomeAssistant } from '../../../../ha/types.js';
import { localize } from '../../../../localize/localize.js';
import liveGo2RTCExperimentalStyle from '../../../../scss/live-go2rtc-experimental.scss';
import type { MediaPlayer, MediaPlayerController } from '../../../../types.js';
import {
dispatchMediaPauseEvent,
dispatchMediaPlayEvent,
dispatchMediaVolumeChangeEvent,
} from '../../../../utils/media-info.js';
import { renderMediaNotification } from '../../../notification/media.js';
@customElement('advanced-camera-card-live-go2rtc-experimental')
export class AdvancedCameraCardGo2RTCExperimental
extends LitElement
implements MediaPlayer
{
// Not a reactive property to avoid resetting the video.
public hass?: HomeAssistant;
@property({ attribute: false })
public camera?: Camera;
// The BASE camera ID (camera property may be a substream)
@property({ attribute: false })
public targetID?: string;
@property({ attribute: false })
public microphoneStream?: MediaStream | null;
@property({ attribute: false })
public microphoneConfig?: MicrophoneConfig;
@property({ attribute: false })
public cardWideConfig?: CardWideConfig;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
@property({ attribute: true, type: Boolean })
public controls = false;
private _hasLiveError = false;
// ===========================================================================
// Surface: Video
// ===========================================================================
private _refVideo: Ref<HTMLVideoElement> = createRef();
private _videoMediaPlayerController = new VideoMediaPlayerController(
this,
() => this._refVideo.value ?? null,
() => this.controls,
);
private _videoSurface: VideoSurface = {
getElement: () => this._refVideo.value ?? null,
getMediaPlayer: () => this._videoMediaPlayerController,
};
// ===========================================================================
// Surface: Image
// ===========================================================================
private _refImage: Ref<HTMLImageElement> = createRef();
// A controller rather than a plain object (unlike the video surface): the
// image surface owns state, the object-URL lifecycle -- each frame's
// createObjectURL and revoking the previous one.
private _imageSurface = new ImageSurfaceController(
this,
() => this._refImage.value ?? null,
{
livenessOptions: {
isFrameExpected: () => true,
},
},
);
// ===========================================================================
// Surface Management
// ===========================================================================
// The surface currently showing committed media, or null before anything has
// committed (both surfaces hidden). Driven by the session's
// surfaceCommittedCallback.
@state()
private _activeSurface: SurfaceKind | null = null;
@state()
private _streamError: MediaUnavailableIssueReason | null = null;
// Built once and kept stable: the session compares this object by identity,
// so handing it a new one will trigger a reconnect.
private _surfaces: SessionSurfaces = {
video: this._videoSurface,
image: this._imageSurface,
};
private _signedURLController = new SignedURLController(this, () => {
const endpoint = this.camera?.getEndpoints()?.go2rtc;
if (!this.hass || !endpoint) {
return {};
}
return {
hass: this.hass,
endpoint,
proxyConfig: this.camera?.getLiveProxyConfig(),
proxyEndpointOptions: { websocket: true },
};
});
private _mediaLoadedInfoSourceController = new MediaLoadedInfoSourceController(this, {
getTargetID: () => this.targetID ?? null,
});
private _session = new Go2RTCSessionController({
getControls: () => this.controls,
getCardWideConfig: () => this.cardWideConfig ?? null,
mediaLoadedCallback: (info) => this._mediaLoadedInfoSourceController.set(info),
surfaceCommittedCallback: (surface) => {
this._activeSurface = surface;
// A commit means the stream recovered: drop any prior error.
this._streamError = null;
},
// The session could not recover the stream on its own; surface it (with the
// failure's user-facing cause) so the card's media-load retry (reconnecting
// indicator, backoff, give-up) runs and can name why. The provider renders
// the error itself (below); the event drives the liveness verdict + retry.
errorCallback: (reason) => {
this._streamError = mapFailureReasonToIssueReason(reason);
dispatchLiveErrorEvent(this, this._streamError);
},
});
public async getMediaPlayerController(): Promise<MediaPlayerController | null> {
return this._activeSurface === 'image'
? this._imageSurface.getMediaPlayer()
: this._videoMediaPlayerController;
}
connectedCallback(): void {
super.connectedCallback();
// Re-render (and thus re-establish the session) when reconnected to the
// DOM. https://github.com/dermotduffy/advanced-camera-card/issues/996
this.requestUpdate();
}
disconnectedCallback(): void {
// Tear down synchronously so streams (e.g. 2-way audio backchannels)
// release immediately.
this._session.reset();
this._activeSurface = null;
super.disconnectedCallback();
}
protected willUpdate(changedProps: PropertyValues): void {
if (changedProps.has('camera')) {
// The session is re-established by `updated()` once the new camera's
// signed URL resolves; the next commit picks the live surface. Blank the
// view meanwhile so the previous camera's last frame is not shown.
this._session.reset();
this._activeSurface = null;
this._streamError = null;
}
// Only treat a missing go2rtc endpoint as an error after the camera's
// endpoints have been explicitly set (not undefined / still loading).
const endpoints = this.camera?.getEndpoints();
const hasLiveError =
!!this._signedURLController.getError() || (!!endpoints && !endpoints.go2rtc);
if (hasLiveError && !this._hasLiveError) {
dispatchLiveErrorEvent(this);
}
this._hasLiveError = hasLiveError;
if (changedProps.has('controls')) {
// Only the video surface has native controls; the image surface has none.
this._videoMediaPlayerController.setControls(this.controls).catch(() => {});
}
if (changedProps.has('microphoneStream')) {
// The WebRTC lane swaps the outbound track in place; no visible reload.
this._session.setMicrophoneStream(this.microphoneStream ?? null);
}
}
protected updated(): void {
const url = this._signedURLController.getValue();
if (url) {
this._session.connect(
url,
this._surfaces,
this.camera?.getConfig()?.go2rtc?.modes,
);
} else {
// No usable URL: a signing/proxy error, or no go2rtc endpoint (which
// includes endpoints still loading). The render omits the surfaces, so
// drop the session -- otherwise a later URL, even an identical unsigned
// endpoint, would be skipped by connect()'s identity check and leave the
// session bound to the removed elements.
this._session.reset();
this._activeSurface = null;
}
}
protected render(): TemplateResult | void {
const error = this._signedURLController.getError();
if (error) {
return renderMediaNotification({
title: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
targetTitle: this.cameraTitle,
});
}
if (!this.camera?.getEndpoints()?.go2rtc) {
return renderMediaNotification({
title: localize('error.configuration_error'),
detail: localize('error.live_camera_no_endpoint'),
targetTitle: this.cameraTitle,
});
}
if (this._streamError) {
// A stream-level failure the session gave up on: the provider must render
// its own error (marked in-progress, since the card keeps retrying).
return renderMediaNotification({
icon: MEDIA_UNAVAILABLE_REASONS[this._streamError].icon,
title: localize(MEDIA_UNAVAILABLE_REASONS[this._streamError].localizationKey),
targetTitle: this.cameraTitle,
});
}
// Both image and video surfaces are always rendered; only the committed one
// is shown (the other, and both before anything commits, are hidden). MSE
// and WebRTC play on the <video>; MP4 and MJPEG show frames on the <img>.
//
// Muted is bound as a property: Chrome ignores the `muted` content
// attribute on videos instantiated from cloned templates (as Lit does), so
// an attribute would not actually start the video muted. Media may be
// unmuted later in accordance with user configuration.
return html`
<video
${ref(this._refVideo)}
.muted=${true}
?hidden=${this._activeSurface !== 'video'}
playsinline
preload="auto"
@play=${() => dispatchMediaPlayEvent(this)}
@pause=${() => dispatchMediaPauseEvent(this)}
@volumechange=${() => dispatchMediaVolumeChangeEvent(this)}
></video>
<img ${ref(this._refImage)} ?hidden=${this._activeSurface !== 'image'} alt="" />
`;
}
static get styles(): CSSResultGroup {
return unsafeCSS(liveGo2RTCExperimentalStyle);
}
}
declare global {
interface HTMLElementTagNameMap {
'advanced-camera-card-live-go2rtc-experimental': AdvancedCameraCardGo2RTCExperimental;
}
}
+13 -7
View File
@@ -17,7 +17,7 @@ import type { HomeAssistant } from '../../../../ha/types.js';
import { localize } from '../../../../localize/localize.js';
import liveGo2RTCStyle from '../../../../scss/live-go2rtc.scss';
import type { MediaPlayer, MediaPlayerController } from '../../../../types.js';
import { renderNotificationBlockFromText } from '../../../notification/block.js';
import { renderMediaNotification } from '../../../notification/media.js';
import { VideoRTC } from './video-rtc.js';
customElements.define('advanced-camera-card-live-go2rtc-player', VideoRTC);
@@ -40,6 +40,10 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
@property({ attribute: false })
public microphoneConfig?: MicrophoneConfig;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
@property({ attribute: true, type: Boolean })
public controls = false;
@@ -148,14 +152,16 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
protected render(): TemplateResult | void {
const error = this._signedURLController.getError();
if (error) {
return renderNotificationBlockFromText(
localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
{ context: this.camera?.getConfig() },
);
return renderMediaNotification({
title: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
targetTitle: this.cameraTitle,
});
}
if (!this.camera?.getEndpoints()?.go2rtc) {
return renderNotificationBlockFromText(localize('error.live_camera_no_endpoint'), {
context: this.camera?.getConfig(),
return renderMediaNotification({
title: localize('error.configuration_error'),
detail: localize('error.live_camera_no_endpoint'),
targetTitle: this.cameraTitle,
});
}
return html`${this._player}`;
+10
View File
@@ -9,6 +9,8 @@ import { customElement, property } from 'lit/decorators.js';
import { createRef, ref, type Ref } from 'lit/directives/ref.js';
import type { Camera } from '../../../camera-manager/camera.js';
import type { MediaUnavailableIssueReason } from '../../../card-controller/issues/issues/media-unavailable.js';
import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispatch-live-error.js';
import type { HomeAssistant } from '../../../ha/types';
import basicBlockStyle from '../../../scss/basic-block.scss';
import type {
@@ -31,6 +33,10 @@ export class AdvancedCameraCardLiveImage extends LitElement implements MediaPlay
@property({ attribute: false })
public targetID?: string;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
private _refImage: Ref<MediaPlayerElement> = createRef();
public async getMediaPlayerController(): Promise<MediaPlayerController | null> {
@@ -51,7 +57,11 @@ export class AdvancedCameraCardLiveImage extends LitElement implements MediaPlay
.imageConfig=${cameraConfig.image}
.cameraConfig=${cameraConfig}
.targetID=${this.targetID}
.cameraTitle=${this.cameraTitle}
.proxyConfig=${this.camera?.getLiveProxyConfig()}
@advanced-camera-card:image-updating-player:error=${(
ev: CustomEvent<MediaUnavailableIssueReason>,
) => dispatchLiveErrorEvent(this, ev.detail)}
>
</advanced-camera-card-image-updating-player>
`;
+20 -13
View File
@@ -14,7 +14,7 @@ import type { Camera } from '../../../camera-manager/camera.js';
import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispatch-live-error.js';
import { MediaLoadedInfoSourceController } from '../../../components-lib/media-loaded-info-source-controller.js';
import { JSMPEGMediaPlayerController } from '../../../components-lib/media-player/jsmpeg.js';
import { createNotificationFromText } from '../../../components-lib/notification/factory.js';
import { createMediaNotification } from '../../../components-lib/notification/media.js';
import type { Notification } from '../../../config/schema/actions/types.js';
import type { CardWideConfig } from '../../../config/schema/types.js';
import { homeAssistantGetSignedURLIfNecessary } from '../../../ha/sign-path.js';
@@ -22,13 +22,14 @@ import type { HomeAssistant } from '../../../ha/types.js';
import { localize } from '../../../localize/localize.js';
import liveJSMPEGStyle from '../../../scss/live-jsmpeg.scss';
import type { MediaPlayer, MediaPlayerController } from '../../../types.js';
import { convertHTTPAdressToWebsocket, errorToConsole } from '../../../utils/basic.js';
import { errorToConsole } from '../../../utils/basic.js';
import {
createMediaLoadedInfo,
dispatchMediaPauseEvent,
dispatchMediaPlayEvent,
} from '../../../utils/media-info.js';
import { Timer } from '../../../utils/timer.js';
import { convertToWebSocketURL } from '../../../utils/websocket-url.js';
import '../../notification/block.js';
@@ -58,6 +59,10 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
@property({ attribute: false })
public cardWideConfig?: CardWideConfig;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
@state()
private _notification: Notification | null = null;
@@ -193,10 +198,11 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
const endpoint = this.camera?.getEndpoints()?.jsmpeg;
if (!endpoint) {
this._notification = createNotificationFromText(
localize('error.live_camera_no_endpoint'),
{ context: this.camera?.getConfig() },
);
this._notification = createMediaNotification({
title: localize('error.configuration_error'),
detail: localize('error.live_camera_no_endpoint'),
targetTitle: this.cameraTitle,
});
dispatchLiveErrorEvent(this);
return;
}
@@ -211,11 +217,12 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
} catch (e) {
errorToConsole(e);
}
const address = response ? convertHTTPAdressToWebsocket(response) : null;
const address = response ? convertToWebSocketURL(response) : null;
if (!address) {
this._notification = createNotificationFromText(localize('error.failed_sign'), {
context: this.camera?.getConfig(),
this._notification = createMediaNotification({
title: localize('error.failed_sign'),
targetTitle: this.cameraTitle,
});
dispatchLiveErrorEvent(this);
return;
@@ -238,10 +245,10 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
if (!this._jsmpegVideoPlayer || !this._jsmpegCanvasElement) {
if (!this._notification) {
this._notification = createNotificationFromText(
localize('error.jsmpeg_no_player'),
{ context: this.camera?.getConfig() },
);
this._notification = createMediaNotification({
title: localize('error.jsmpeg_no_player'),
targetTitle: this.cameraTitle,
});
dispatchLiveErrorEvent(this);
}
return;
+11 -16
View File
@@ -14,24 +14,19 @@ import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispa
import { getTechnologyForVideoRTC } from '../../../components-lib/live/utils/get-technology-for-video-rtc.js';
import { MediaLoadedInfoSourceController } from '../../../components-lib/media-loaded-info-source-controller.js';
import { VideoMediaPlayerController } from '../../../components-lib/media-player/video.js';
import { createNotificationFromText } from '../../../components-lib/notification/factory.js';
import { createMediaNotification } from '../../../components-lib/notification/media.js';
import type { Notification } from '../../../config/schema/actions/types.js';
import type { CardWideConfig } from '../../../config/schema/types.js';
import type { HomeAssistant } from '../../../ha/types.js';
import { localize } from '../../../localize/localize.js';
import liveWebRTCCardStyle from '../../../scss/live-webrtc-card.scss';
import {
AdvancedCameraCardError,
type MediaPlayer,
type MediaPlayerController,
} from '../../../types.js';
import type { MediaPlayer, MediaPlayerController } from '../../../types.js';
import { mayHaveAudio } from '../../../utils/audio.js';
import {
hideMediaControlsTemporarily,
MEDIA_LOAD_CONTROLS_HIDE_SECONDS,
setControlsOnVideo,
} from '../../../utils/controls.js';
import { getContextFromError } from '../../../utils/error-context.js';
import {
createMediaLoadedInfo,
dispatchMediaPauseEvent,
@@ -63,6 +58,10 @@ export class AdvancedCameraCardLiveWebRTCCard extends LitElement implements Medi
@property({ attribute: false })
public cardWideConfig?: CardWideConfig;
// The camera's title, shown in error messages to identify the camera.
@property({ attribute: false })
public cameraTitle?: string;
@property({ attribute: true, type: Boolean })
public controls = false;
@@ -169,15 +168,11 @@ export class AdvancedCameraCardLiveWebRTCCard extends LitElement implements Medi
try {
webrtcElement = this._createWebRTC();
} catch (e) {
const context = getContextFromError(e);
this._notification = createNotificationFromText(
e instanceof AdvancedCameraCardError
? e.message
: localize('error.webrtc_card_reported_error') + ': ' + (e as Error).message,
{
...(context && { context }),
},
);
this._notification = createMediaNotification({
title: localize('error.webrtc_card_reported_error'),
detail: e instanceof Error ? e.message : String(e),
targetTitle: this.cameraTitle,
});
dispatchLiveErrorEvent(this);
return;
}
+43
View File
@@ -0,0 +1,43 @@
import type { TemplateResult } from 'lit';
import type { CameraManager } from '../../camera-manager/manager.js';
import {
createMediaNotification,
type MediaNotificationOptions,
} from '../../components-lib/notification/media.js';
import { localize } from '../../localize/localize.js';
import { renderNotificationBlock } from './block.js';
// Render the standard media notification block: a short titled heading (with
// the camera name when there is one), an optional longer detail, a
// troubleshooting link, and a retry spinner. See `createMediaNotification`.
export function renderMediaNotification(
options: MediaNotificationOptions,
): TemplateResult {
return renderNotificationBlock(createMediaNotification(options));
}
interface NoMediaOptions {
cameraID: string | null;
inProgress?: boolean;
}
// The viewer/gallery no-media (or awaiting-media) state.
export function renderNoMediaNotification(
options: NoMediaOptions,
cameraManager?: CameraManager,
): TemplateResult {
const cameraID =
options.cameraID ?? cameraManager?.getStore().getDefaultCameraID() ?? null;
const targetTitle = cameraID
? cameraManager?.getCameraMetadata(cameraID)?.title ?? cameraID
: undefined;
return renderMediaNotification({
title: localize(options.inProgress ? 'error.awaiting_media' : 'common.no_media'),
icon: 'mdi:multimedia',
targetTitle,
inProgress: !!options.inProgress,
troubleshooting: false,
});
}
-32
View File
@@ -1,32 +0,0 @@
import type { TemplateResult } from 'lit';
import type { CameraManager } from '../../camera-manager/manager.js';
import { localize } from '../../localize/localize.js';
import { renderNotificationBlock } from './block.js';
interface NoMediaOptions {
cameraID: string | null;
cameraManager: CameraManager | null;
loading?: boolean;
}
export function renderNoMedia(options: NoMediaOptions): TemplateResult {
const cameraID =
options.cameraID ?? options.cameraManager?.getStore().getDefaultCameraID() ?? null;
const cameraTitle = cameraID
? options.cameraManager?.getCameraMetadata(cameraID)?.title ?? cameraID
: null;
return renderNotificationBlock({
heading: {
text: options.loading
? localize('error.awaiting_media')
: localize('common.no_media'),
icon: 'mdi:multimedia',
},
in_progress: options.loading,
...(cameraTitle && {
metadata: [{ text: cameraTitle, icon: 'mdi:cctv' }],
}),
});
}
+15 -13
View File
@@ -38,7 +38,7 @@ import type { ViewMedia } from '../../view/item.js';
import '../carousel';
import '../next-prev-control.js';
import { renderNoMedia } from '../notification/no-media.js';
import { renderNoMediaNotification } from '../notification/media.js';
import '../ptz.js';
import './provider.js';
@@ -321,13 +321,15 @@ export class AdvancedCameraCardViewerCarousel extends LitElement {
protected render(): TemplateResult | void {
const mediaCount = this._media?.length ?? 0;
if (!this._media || !mediaCount) {
return renderNoMedia({
cameraID:
this.viewFilterCameraID ??
this.viewManagerEpoch?.manager.getView()?.camera ??
null,
cameraManager: this.cameraManager ?? null,
});
return renderNoMediaNotification(
{
cameraID:
this.viewFilterCameraID ??
this.viewManagerEpoch?.manager.getView()?.camera ??
null,
},
this.cameraManager,
);
}
if (!this.hass || !this.cameraManager || this._selected === null) {
@@ -443,17 +445,17 @@ export class AdvancedCameraCardViewerCarousel extends LitElement {
const seekTimeInMedia = selectedMedia.includesTime(seek);
this.toggleAttribute('unseekable', !seekTimeInMedia);
if (!seekTimeInMedia && !mediaPlayerController.isPaused()) {
void mediaPlayerController.pause();
} else if (seekTimeInMedia && mediaPlayerController.isPaused()) {
void mediaPlayerController.play();
if (!seekTimeInMedia && !mediaPlayerController.playback?.isPaused()) {
void mediaPlayerController.playback?.pause();
} else if (seekTimeInMedia && mediaPlayerController.playback?.isPaused()) {
void mediaPlayerController.playback?.play();
}
const seekTime =
(await this.cameraManager?.getMediaSeekTime(selectedMedia, seek)) ?? null;
if (seekTime !== null) {
void mediaPlayerController.seek(seekTime);
void mediaPlayerController.seek?.(seekTime);
}
}
+8 -6
View File
@@ -20,7 +20,7 @@ import '../../patches/ha-hls-player.js';
import viewerStyle from '../../scss/viewer.scss';
import { ViewItemClassifier } from '../../view/item-classifier.js';
import { renderNoMedia } from '../notification/no-media.js';
import { renderNoMediaNotification } from '../notification/media.js';
import './grid';
@@ -84,11 +84,13 @@ export class AdvancedCameraCardViewer extends LitElement {
// Directly render an error message (instead of dispatching it upwards)
// to preserve the mini-timeline if the user pans into an area with no
// media.
return renderNoMedia({
cameraID: this.viewManagerEpoch.manager.getView()?.camera ?? null,
cameraManager: this.cameraManager ?? null,
loading: !!this.viewManagerEpoch.manager.getView()?.context?.loading?.query,
});
return renderNoMediaNotification(
{
cameraID: this.viewManagerEpoch.manager.getView()?.camera ?? null,
inProgress: !!this.viewManagerEpoch.manager.getView()?.context?.loading?.query,
},
this.cameraManager,
);
}
return html` <advanced-camera-card-viewer-grid
+5 -1
View File
@@ -31,6 +31,7 @@ const LIVE_PROVIDERS = [
'ha',
'jsmpeg',
'go2rtc',
'go2rtc-experimental',
'webrtc-card',
] as const;
export type LiveProvider = (typeof LIVE_PROVIDERS)[number];
@@ -40,12 +41,15 @@ const go2rtcConfigDefault = {
metadata_fetch_timeout_seconds: 2,
};
export const GO2RTC_MODES = ['webrtc', 'mse', 'mp4', 'mjpeg'] as const;
export type Go2RTCMode = (typeof GO2RTC_MODES)[number];
const go2rtcConfigSchema = z.object({
url: z
.string()
.transform((input) => input.replace(/\/+$/, ''))
.optional(),
modes: z.enum(['webrtc', 'mse', 'mp4', 'mjpeg']).array().optional(),
modes: z.enum(GO2RTC_MODES).array().optional(),
stream: z.string().optional(),
metadata_fetch_timeout_seconds: z
.number()
+4
View File
@@ -2595,6 +2595,10 @@ export class AdvancedCameraCardEditor extends LitElement implements LovelaceCard
value: 'go2rtc',
label: localize('config.cameras.live_providers.go2rtc'),
},
{
value: 'go2rtc-experimental',
label: localize('config.cameras.live_providers.go2rtc-experimental'),
},
{
value: 'webrtc-card',
label: localize('config.cameras.live_providers.webrtc-card'),
-1
View File
@@ -382,7 +382,6 @@
"awaiting_media": "Warte auf Medium ...",
"camera_initialization": "Kamera-Initialisierung fehlgeschlagen",
"camera_initialization_reolink": "Konnte Reolink-Kamera nicht initialisieren",
"configuration": "Konfiguration überprüfen",
"duplicate_folder_id": "Doppelte Ordner-ID für den folgen Ordner, benutzte Parameter 'id' um Ordner eindeutig zu identifizieren",
"failed_proxy": "Konnte mich nicht über Home Assistant verbinden",
"fetching_diagnostics": "Rufe Diagnosedaten ab",
+6 -2
View File
@@ -141,6 +141,7 @@
"live_providers": {
"auto": "Automatic",
"go2rtc": "go2rtc",
"go2rtc-experimental": "go2rtc (experimental implementation)",
"ha": "Home Assistant video stream (i.e. HLS, LL-HLS, WebRTC via HA)",
"image": "Image",
"jsmpeg": "JSMpeg",
@@ -781,7 +782,7 @@
"call_no_two_way_audio": "This camera does not support two-way audio.",
"camera_initialization": "Camera initialization failed",
"camera_initialization_reolink": "Could not initialize Reolink camera",
"configuration": "Check configuration",
"configuration_error": "Configuration error",
"could_not_create_elements": "Could not create picture elements",
"diagnostics": "Card diagnostics. Please review for confidential information prior to sharing",
"download_failed": "Download failed",
@@ -866,7 +867,10 @@
"entity_unavailable": "Camera entity unavailable",
"not_loading": "Media not loading",
"playback_error": "Playback error",
"stalled": "Stream stalled"
"server_error": "Streaming server error",
"stalled": "Stream stalled",
"two_way_audio_error": "Two-way audio error",
"unsupported": "Stream not supported"
},
"text": "The media is not currently available. This can happen for several reasons, for example a camera becoming unavailable, a stream stalling or failing, or media that has not finished loading. The card keeps retrying automatically. For live views, a still image may be shown while a stream is loading (if configured, and by default)"
},
-1
View File
@@ -654,7 +654,6 @@
"awaiting_media": "Oczekiwanie na załadowanie mediów",
"camera_initialization": "Inicjalizacja kamery nie powiodła się",
"camera_initialization_reolink": "Nie można zainicjować kamery Reolink",
"configuration": "Sprawdź konfigurację",
"could_not_create_elements": "Nie można wyrenderować elementów obrazu",
"diagnostics": "Diagnostyka karty. Przejrzyj pod kątem poufnych informacji przed udostępnieniem",
"download_no_media": "Brak mediów do pobrania",
+6 -3
View File
@@ -16,7 +16,8 @@ import { query } from 'lit/decorators/query.js';
import { dispatchLiveErrorEvent } from '../components-lib/live/utils/dispatch-live-error.js';
import { MediaLoadedInfoSourceController } from '../components-lib/media-loaded-info-source-controller.js';
import { VideoMediaPlayerController } from '../components-lib/media-player/video.js';
import { renderNotificationBlockFromText } from '../components/notification/block.js';
import { renderMediaNotification } from '../components/notification/media.js';
import { localize } from '../localize/localize.js';
import liveHAComponentsStyle from '../scss/live-ha-components.scss';
import type { MediaPlayer, MediaPlayerController } from '../types.js';
import { mayHaveAudio } from '../utils/audio.js';
@@ -72,8 +73,10 @@ void customElements.whenDefined('ha-hls-player').then(() => {
if (this._error) {
if (this._errorIsFatal) {
dispatchLiveErrorEvent(this);
return renderNotificationBlockFromText(this._error, {
metadata: [{ text: this.entityid, icon: 'mdi:cctv' }],
return renderMediaNotification({
title: localize('issues.media_unavailable.reasons.playback_error'),
detail: this._error,
targetTitle: this.entityid,
});
} else {
errorToConsole(this._error, console.error);
+8 -5
View File
@@ -16,7 +16,8 @@ import { ifDefined } from 'lit/directives/if-defined.js';
import { dispatchLiveErrorEvent } from '../components-lib/live/utils/dispatch-live-error.js';
import { MediaLoadedInfoSourceController } from '../components-lib/media-loaded-info-source-controller.js';
import { VideoMediaPlayerController } from '../components-lib/media-player/video.js';
import { renderNotificationBlockFromText } from '../components/notification/block.js';
import { renderMediaNotification } from '../components/notification/media.js';
import { localize } from '../localize/localize.js';
import liveHAComponentsStyle from '../scss/live-ha-components.scss';
import type { MediaPlayer, MediaPlayerController } from '../types.js';
import {
@@ -112,8 +113,10 @@ void customElements.whenDefined('ha-web-rtc-player').then(() => {
protected render(): TemplateResult | void {
if (this._error) {
dispatchLiveErrorEvent(this);
return renderNotificationBlockFromText(this._error, {
metadata: [{ text: this.entityid, icon: 'mdi:cctv' }],
return renderMediaNotification({
title: localize('issues.media_unavailable.reasons.playback_error'),
detail: this._error,
targetTitle: this.entityid,
});
}
return html`
@@ -146,7 +149,7 @@ void customElements.whenDefined('ha-web-rtc-player').then(() => {
mediaPlayerController: this._mediaPlayerController,
capabilities: {
supportsPause: true,
hasAudio: hasAudio(this._videoEl, this._peerConnection),
hasAudio: hasAudio(this._videoEl, { pc: this._peerConnection }),
},
technology: ['webrtc'],
});
@@ -164,7 +167,7 @@ void customElements.whenDefined('ha-web-rtc-player').then(() => {
mediaPlayerController: this._mediaPlayerController,
capabilities: {
supportsPause: true,
hasAudio: hasAudio(this._videoEl, this._peerConnection),
hasAudio: hasAudio(this._videoEl, { pc: this._peerConnection }),
},
technology: ['webrtc'],
});
+33
View File
@@ -0,0 +1,33 @@
@use 'media-layout.scss';
:host {
width: 100%;
height: 100%;
display: block;
}
video,
img {
@include media-layout.media-layout();
width: 100%;
height: 100%;
display: block;
// The inactive surface is `?hidden` in the template (only the committed one
// is shown).
&[hidden] {
display: none;
}
}
// Hide the native seek bar (and its time readouts) on the live video: the
// stream is live, so scrubbing back only fights the live-edge chase and snaps
// to live. Play/pause/volume/fullscreen remain. WebKit/Blink only (Chrome,
// Safari, Edge, Android); Firefox does not expose these controls for styling,
// so its scrubber stays.
video::-webkit-media-controls-timeline,
video::-webkit-media-controls-current-time-display,
video::-webkit-media-controls-time-remaining-display {
display: none;
}
+16
View File
@@ -4,10 +4,26 @@
position: relative;
}
// Until loaded media or a snapshot sizes the frame (the `sized` attribute),
// nothing gives it a size, so reserve a default ratio to stop it collapsing.
// Real media is never letterboxed into this ratio: the attribute is set the
// moment it sizes the frame.
:host(:not([sized])) {
aspect-ratio: 16 / 9;
}
.hidden {
display: none;
}
// Fills the frame with a loading or error status when neither media nor a
// snapshot is present (a provider renders only its own media, so the frame
// would otherwise be blank).
.fill {
position: absolute;
inset: 0;
}
advanced-camera-card-icon {
position: absolute;
top: 10px;
+19 -3
View File
@@ -89,20 +89,36 @@ export type UnsubscribeCallback = () => void;
// Reports each live/stalled transition of a player's media stream.
export type LivenessCallback = (isLive: boolean) => void;
export interface MediaPlayerController {
// Control over a pausable playback loop. Optional on the player: present only
// when the player owns a stream it can start and stop (e.g. a real video, or an
// image refreshed on a timer). A static image, or one fed frames as they arrive
// where the client has no control (e.g. MP4, MJPEG), has no pausable loop and
// omits it.
export interface PlaybackControl {
play(): Promise<void>;
pause(): Promise<void>;
isPaused(): boolean;
}
export interface MediaPlayerController {
mute(): Promise<void>;
unmute(): Promise<void>;
isMuted(): boolean;
seek(seconds: number): Promise<void>;
getScreenshotURL(): Promise<string | null>;
// If no value for controls if specified, the player should use the default.
setControls(controls?: boolean): Promise<void>;
isPaused(): boolean;
getFullscreenElement(): FullscreenElement | null;
getPIPElement(): PIPElement | null;
// Jump to a time position, if the media has a seekable timeline. Optional:
// implemented only by players over seekable media.
seek?(seconds: number): Promise<void>;
// Start/pause the media, if it is pausable. Optional: implemented only by
// players that own a pausable playback loop.
playback?: PlaybackControl;
// Observe whether the player is actively delivering media, so a silent freeze
// (frames stop advancing while playing) can be detected. Optional:
// implemented only by players that can observe their own frame progress. The
+11 -6
View File
@@ -25,15 +25,17 @@ export const mayHaveAudio = (video: HTMLVideoElement & AudioProperties): boolean
/**
* Determine if audio is available for a go2rtc stream.
* @param pc The RTCPeerConnection (for WebRTC streams).
* @param mseCodecs The negotiated MSE codecs string (for MSE streams).
* @param video The video element (fallback for browser-based detection).
* @param options.pc The RTCPeerConnection (for WebRTC streams).
* @param options.mseCodecs The negotiated MSE codecs string (for MSE streams).
* @returns True if audio is available.
*/
export const hasAudio = (
video: HTMLVideoElement & AudioProperties,
pc?: RTCPeerConnection | null,
mseCodecs?: string,
options?: {
pc?: RTCPeerConnection | null;
mseCodecs?: string | null;
},
): boolean => {
// For WebRTC: Check if there's an audio receiver with an active track. We
// check that the track is not muted because muted means no media data is
@@ -43,15 +45,17 @@ export const hasAudio = (
// (e.g. WebRTC failed, fell back to MSE) will have receivers with muted
// tracks that don't reflect actual media availability.
// See: https://github.com/dermotduffy/advanced-camera-card/issues/2417
if (pc && pc.connectionState === 'connected') {
const receivers = pc.getReceivers();
if (options?.pc && options.pc.connectionState === 'connected') {
const receivers = options.pc.getReceivers();
if (receivers.length > 0) {
return receivers.some(
(receiver) => receiver.track?.kind === 'audio' && !receiver.track?.muted,
);
}
}
// For MSE: Check negotiated codecs for audio codecs
const mseCodecs = options?.mseCodecs;
if (mseCodecs) {
return (
mseCodecs.includes('mp4a') ||
@@ -59,6 +63,7 @@ export const hasAudio = (
mseCodecs.includes('flac')
);
}
// Fallback to browser-based detection (unreliable in Chrome)
return mayHaveAudio(video);
};
-4
View File
@@ -352,10 +352,6 @@ export const ignoreFunctionIdentity = (a: unknown, b: unknown): boolean | undefi
? typeof a === 'function' && typeof b === 'function'
: undefined;
export const convertHTTPAdressToWebsocket = (url: string): string => {
return url.replace(/^http/i, 'ws');
};
export const forceReflow = (element: HTMLElement): void => {
// Force reflow by measuring the height.
void element.offsetHeight;
+29
View File
@@ -0,0 +1,29 @@
// A monotonic counter for guarding against stale asynchronous results. Snapshot
// the current generation before an `await`; afterwards, `isCurrent()` reports
// whether anything has invalidated that snapshot since (a reset, a teardown, or
// a newer operation), so a superseded result can be dropped.
export class Generation {
private _value = 0;
// Invalidate all outstanding snapshots (e.g. on reset or teardown).
public invalidate(): void {
this._value++;
}
// Start a new latest-wins operation -- invalidating any outstanding snapshot --
// and return its token.
public next(): number {
return ++this._value;
}
// Snapshot the current generation to compare after an await.
public current(): number {
return this._value;
}
// Whether the token is still current (nothing has invalidated it since it was
// taken).
public isCurrent(token: number): boolean {
return token === this._value;
}
}
@@ -0,0 +1,71 @@
/**
* Runs an async operation for the latest submitted value, one run at a time.
*
* While a run is in flight, only the most recently submitted value is kept;
* values submitted in between are dropped. The kept value runs when the current
* run finishes, so the operation always converges on the newest input without
* running more than once at a time.
*
* Each `submit` returns a promise that resolves after the next run completes, so
* a caller can wait for a run to have happened without tracking which value ran
* (under load its own value may have been superseded by a newer one).
*
* The operation owns its own errors: a run that rejects still counts as done
* (its waiters resolve and draining continues), so a single bad value cannot
* strand later ones.
*/
export class LatestValueRunner<T> {
private _run: (value: T) => Promise<void>;
private _running = false;
private _pending: { value: T } | null = null;
private _waiters: Array<() => void> = [];
constructor(run: (value: T) => Promise<void>) {
this._run = run;
}
// Submit a value to run. Returns a promise that resolves once the next run
// has completed.
public submit(value: T): Promise<void> {
this._pending = { value };
const ran = new Promise<void>((resolve) => this._waiters.push(resolve));
if (!this._running) {
this._running = true;
// The drain loop cannot reject (the operation's errors are caught within
// it); the catch only satisfies the no-floating-promises rule.
/* istanbul ignore next -- @preserve */
this._drain().catch(() => {});
}
return ran;
}
// Drop any value still waiting to run.
public clear(): void {
this._pending = null;
}
private async _drain(): Promise<void> {
try {
while (this._pending) {
const { value } = this._pending;
this._pending = null;
try {
await this._run(value);
} catch {
// The operation owns its errors; the runner only guarantees progress.
}
// Release everyone waiting on a run; values submitted during the run
// were queued after this snapshot and wait for the next one.
const waiters = this._waiters;
this._waiters = [];
waiters.forEach((resolve) => resolve());
}
} finally {
this._running = false;
}
}
}
+8 -1
View File
@@ -22,6 +22,13 @@ export const getResolvedLiveProvider = (
return config?.live_provider ?? 'image';
};
// Live providers that stream from a go2rtc server and share the `go2rtc`
// config block and endpoints.
const GO2RTC_LIVE_PROVIDERS: readonly LiveProvider[] = ['go2rtc', 'go2rtc-experimental'];
export const isGo2RTCLiveProvider = (provider: LiveProvider): boolean =>
GO2RTC_LIVE_PROVIDERS.includes(provider);
export const liveProviderSupports2WayAudio = async (
hass: HomeAssistant,
config: CameraConfig,
@@ -29,7 +36,7 @@ export const liveProviderSupports2WayAudio = async (
go2rtcMetadataEndpoint?: Endpoint | null,
proxyConfig?: EnabledProxyConfig,
): Promise<boolean> => {
if (getResolvedLiveProvider(config) !== 'go2rtc') {
if (!isGo2RTCLiveProvider(getResolvedLiveProvider(config))) {
return false;
}
+10
View File
@@ -0,0 +1,10 @@
// Convert an HTTP(S) or origin-relative URL to its WS(S) equivalent.
export const convertToWebSocketURL = (url: string, origin?: string): string => {
if (/^http/i.test(url)) {
return 'ws' + url.substring(4);
}
if (url.startsWith('/')) {
return 'ws' + (origin ?? location.origin).substring(4) + url;
}
return url;
};