diff --git a/docs/configuration/cameras/live-provider.md b/docs/configuration/cameras/live-provider.md index bbb8291c..1257dc88 100644 --- a/docs/configuration/cameras/live-provider.md +++ b/docs/configuration/cameras/live-provider.md @@ -4,15 +4,16 @@ The `live_provider` parameter determines what provides the live stream for a camera. Each provider offers different capabilities: -| Live Provider | Latency | Frame Rate | Loading Time | Installation | Supports [Proxying](./README.md?id=proxy) | Description | -| ---------------------------------- | ------- | ---------- | ------------ | ------------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `go2rtc` | Best | High | Better | Builtin | :white_check_mark: | Uses [go2rtc](https://github.com/AlexxIT/go2rtc) to stream live feeds and supports 2-way audio. | -| `ha` (Native WebRTC) | Best | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- will offer a very low-latency feed direct to your browser. | -| `ha` (HLS) | Poor | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- HLS fallback when a WebRTC connection cannot be established. | -| `ha` (when configured with LL-HLS) | Better | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- can be configured to use an [LL-HLS](https://www.home-assistant.io/integrations/stream/#ll-hls) feed for lower latency. | -| `image` | Poor | Poor | Best | Builtin | :heavy_multiplication_x: | Use refreshing snapshots of the built-in Home Assistant camera streams. | -| `jsmpeg` | Better | Low | Poor | Builtin | :heavy_multiplication_x: | Use a the JSMPEG stream. | -| `webrtc-card` | Best | High | Better | Separate installation required | :heavy_multiplication_x: | Embed's [AlexxIT's WebRTC Card](https://github.com/AlexxIT/WebRTC) to stream live feed, requires manual extra setup. See [`webrtc_card`](#webrtc_card). Not to be confused with native Home Assistant WebRTC (use the `ha` provider). | +| Live Provider | Latency | Frame Rate | Loading Time | Installation | Supports [Proxying](./README.md?id=proxy) | Description | +| ---------------------------------- | ------- | ---------- | ------------ | ------------------------------ | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `go2rtc` | Best | High | Better | Builtin | :white_check_mark: | Uses [go2rtc](https://github.com/AlexxIT/go2rtc) to stream live feeds and supports 2-way audio. | +| `go2rtc-experimental` | Best | High | Better | Builtin | :white_check_mark: | **Experimental** re-implementation of the `go2rtc` provider client (MSE/WebRTC/MP4/MJPEG) that is expected to eventually replace the `go2rtc` live provider. Uses the same configuration as `go2rtc`. See [`go2rtc (experimental)`](#go2rtc-experimental). | +| `ha` (Native WebRTC) | Best | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- will offer a very low-latency feed direct to your browser. | +| `ha` (HLS) | Poor | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- HLS fallback when a WebRTC connection cannot be established. | +| `ha` (when configured with LL-HLS) | Better | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- can be configured to use an [LL-HLS](https://www.home-assistant.io/integrations/stream/#ll-hls) feed for lower latency. | +| `image` | Poor | Poor | Best | Builtin | :heavy_multiplication_x: | Use refreshing snapshots of the built-in Home Assistant camera streams. | +| `jsmpeg` | Better | Low | Poor | Builtin | :heavy_multiplication_x: | Use a the JSMPEG stream. | +| `webrtc-card` | Best | High | Better | Separate installation required | :heavy_multiplication_x: | Embed's [AlexxIT's WebRTC Card](https://github.com/AlexxIT/WebRTC) to stream live feed, requires manual extra setup. See [`webrtc_card`](#webrtc_card). Not to be confused with native Home Assistant WebRTC (use the `ha` provider). | ## `go2rtc` @@ -43,6 +44,20 @@ cameras: > [`capabilities.force`](./README.md?id=capabilities) to skip metadata > detection entirely. +## `go2rtc (experimental)` + +> [!WARNING] `go2rtc-experimental` is a from-scratch re-implementation of the [`go2rtc`](#go2rtc) live provider client (MSE, WebRTC, MP4 and MJPEG). It is under active development and is expected to eventually replace the `go2rtc` live provider. Please try it and [report any issues you encounter](https://github.com/dermotduffy/advanced-camera-card/issues). + +It uses the exact same configuration as [`go2rtc`](#go2rtc): select it with `live_provider: go2rtc-experimental` and configure the same [`go2rtc`](#go2rtc) block. + +```yaml +cameras: + - camera_entity: camera.office + live_provider: go2rtc-experimental + go2rtc: + # [...] +``` + ## `ha` The `ha` block configures use of the default Home Assistant (`ha`) live provider. It has no configuration options. @@ -164,6 +179,17 @@ cameras: stream: sitting_room url: 'https://my.custom.go2rtc.backend' metadata_fetch_timeout_seconds: 2 + - camera_entity: camera.office_go2rtc_experimental + live_provider: go2rtc-experimental + go2rtc: + modes: + - webrtc + - mse + - mp4 + - mjpeg + stream: sitting_room + url: 'https://my.custom.go2rtc.backend' + metadata_fetch_timeout_seconds: 2 - camera_entity: camera.office_jsmpeg live_provider: jsmpeg jsmpeg: diff --git a/docs/usage/2-way-audio.md b/docs/usage/2-way-audio.md index dcc0a18d..8d14ddbc 100644 --- a/docs/usage/2-way-audio.md +++ b/docs/usage/2-way-audio.md @@ -16,7 +16,7 @@ challenging. ### Card requirements - Only Frigate cameras are supported. -- Only the `go2rtc` live provider is supported. +- Only the `go2rtc` and `go2rtc-experimental` live providers are supported. - Only the `webrtc` mode supports 2-way audio. If your setup supports 2-way audio but detection is intermittent on load: diff --git a/eslint.config.mjs b/eslint.config.mjs index bcb99f66..174c3d99 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -106,7 +106,7 @@ export default defineConfig([ languageOptions: { parser: tsParser, parserOptions: { - project: './tsconfig.json', + projectService: true, tsconfigRootDir: __dirname, }, }, diff --git a/src/camera-manager/camera.ts b/src/camera-manager/camera.ts index 08c5b941..1937b90c 100644 --- a/src/camera-manager/camera.ts +++ b/src/camera-manager/camera.ts @@ -18,7 +18,10 @@ import type { HassStateDifference, HomeAssistant } from '../ha/types'; import { localize } from '../localize/localize'; import type { CapabilitiesRaw, CapabilityKey, Endpoint } from '../types'; import { arrayify } from '../utils/basic'; -import { liveProviderSupports2WayAudio } from '../utils/live-provider'; +import { + isGo2RTCLiveProvider, + liveProviderSupports2WayAudio, +} from '../utils/live-provider'; import { Capabilities } from './capabilities'; import type { CameraManagerEngine } from './engine'; import { CameraNoIDError } from './error'; @@ -345,9 +348,10 @@ export class Camera { ...resolveProxyConfig(this._config.proxy), live: this._config.proxy.live === 'auto' - ? // Live is proxied if the live provider is go2rtc and if a go2rtc - // URL is manually set. - this._config.live_provider === 'go2rtc' && !!this._config.go2rtc?.url + ? // Live is proxied if the live provider streams from go2rtc and a + // go2rtc URL is manually set. + isGo2RTCLiveProvider(this._config.live_provider) && + !!this._config.go2rtc?.url : this._config.proxy.live, media: this._config.proxy.media === 'auto' ? false : this._config.proxy.media, }; diff --git a/src/card-controller/actions/actions/pause.ts b/src/card-controller/actions/actions/pause.ts index da50bb51..b2e717d6 100644 --- a/src/card-controller/actions/actions/pause.ts +++ b/src/card-controller/actions/actions/pause.ts @@ -6,6 +6,9 @@ export class PauseAction extends AdvancedCameraCardAction { public async execute(api: CardActionsAPI): Promise { await super.execute(api); - await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.pause(); + await api + .getMediaLoadedInfoManager() + .get() + ?.mediaPlayerController?.playback?.pause(); } } diff --git a/src/card-controller/actions/actions/play.ts b/src/card-controller/actions/actions/play.ts index a45b85b5..3837fa98 100644 --- a/src/card-controller/actions/actions/play.ts +++ b/src/card-controller/actions/actions/play.ts @@ -6,6 +6,6 @@ export class PlayAction extends AdvancedCameraCardAction { public async execute(api: CardActionsAPI): Promise { await super.execute(api); - await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.play(); + await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.playback?.play(); } } diff --git a/src/card-controller/call/manager.ts b/src/card-controller/call/manager.ts index cd1a9bcf..8ea3b11b 100644 --- a/src/card-controller/call/manager.ts +++ b/src/card-controller/call/manager.ts @@ -1,6 +1,7 @@ import { createNotificationFromText } from '../../components-lib/notification/factory'; import type { ConditionStateChange } from '../../condition-trigger/conditions/types'; import { localize } from '../../localize/localize'; +import { Generation } from '../../utils/concurrency/generation'; import { Timer } from '../../utils/timer'; import { getStreamCameraID } from '../../view/substream'; import type { View } from '../../view/view'; @@ -22,7 +23,7 @@ export class CallManager { // resumed tail leaks audio onto the shared lock from an instance the user // can no longer see or control, and may install state into a fresh // lifecycle from a request that belongs to the previous one. - private _initEpoch = 0; + private _initGeneration = new Generation(); constructor(api: CardCallAPI) { this._api = api; @@ -103,7 +104,7 @@ export class CallManager { return false; } - const initEpoch = this._initEpoch; + const initGeneration = this._initGeneration.current(); const microphoneConnected = await this._connectMicrophone(); // If the init/uninit lifecycle advanced while the microphone connect was // in flight, this request belongs to a previous lifecycle -- the view we @@ -111,7 +112,7 @@ export class CallManager { // clean teardown path for state we'd install here. Bail before touching // `_call`, the ringtone lock, or surfacing a notification onto a torn-down // NotificationManager. - if (initEpoch !== this._initEpoch) { + if (!this._initGeneration.isCurrent(initGeneration)) { return false; } if (!microphoneConnected) { @@ -241,7 +242,7 @@ export class CallManager { // // Safe to re-initialize afterwards via `initialize()`. public uninitialize(): void { - this._initEpoch++; + this._initGeneration.invalidate(); this._ringtone.stop(); this._unansweredTimer.stop(); if (this._call) { diff --git a/src/card-controller/issues/issues/media-unavailable.ts b/src/card-controller/issues/issues/media-unavailable.ts index 0eca830d..f5002460 100644 --- a/src/card-controller/issues/issues/media-unavailable.ts +++ b/src/card-controller/issues/issues/media-unavailable.ts @@ -21,7 +21,10 @@ export type MediaUnavailableIssueReason = | 'entity_unavailable' | 'not_loading' | 'playback_error' - | 'stalled'; + | 'server_error' + | 'stalled' + | 'two_way_audio_error' + | 'unsupported'; declare module 'issue' { interface IssueTriggerContext { @@ -50,10 +53,22 @@ export const MEDIA_UNAVAILABLE_REASONS: Record< localizationKey: 'issues.media_unavailable.reasons.playback_error', icon: 'mdi:alert-circle', }, + server_error: { + localizationKey: 'issues.media_unavailable.reasons.server_error', + icon: 'mdi:server-network-off', + }, stalled: { localizationKey: 'issues.media_unavailable.reasons.stalled', icon: 'mdi:motion-pause', }, + two_way_audio_error: { + localizationKey: 'issues.media_unavailable.reasons.two_way_audio_error', + icon: 'mdi:microphone-off', + }, + unsupported: { + localizationKey: 'issues.media_unavailable.reasons.unsupported', + icon: 'mdi:video-off-outline', + }, }; export class MediaUnavailableIssue implements Issue { diff --git a/src/components-lib/live/liveness/detectors/provider-error.ts b/src/components-lib/live/liveness/detectors/provider-error.ts index ec4a9096..67d6e7b7 100644 --- a/src/components-lib/live/liveness/detectors/provider-error.ts +++ b/src/components-lib/live/liveness/detectors/provider-error.ts @@ -1,3 +1,4 @@ +import type { MediaUnavailableIssueReason } from '../../../../card-controller/issues/issues/media-unavailable'; import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller'; const LIVE_ERROR_EVENT = 'advanced-camera-card:live:error'; @@ -38,12 +39,19 @@ export class ProviderErrorDetector implements LivenessDetector { return this._verdict; } - private _handler = (ev: Event): void => { + private _handler = ( + ev: CustomEvent, + ): void => { ev.stopPropagation(); if (this._verdict.state !== 'not_live') { // Authoritative: an explicit provider error overrides even direct frame - // evidence. No placeholder -- the provider renders its own error. - this._verdict = { state: 'not_live', authority: 'hard', reason: 'playback_error' }; + // evidence. No placeholder -- the provider renders its own error. The + // provider may name the cause; otherwise it is a generic playback error. + this._verdict = { + state: 'not_live', + authority: 'hard', + reason: ev.detail ?? 'playback_error', + }; this._onChange(); } }; diff --git a/src/components-lib/live/liveness/stream-liveness-controller.ts b/src/components-lib/live/liveness/stream-liveness-controller.ts index bfd2ee9c..c6fc6eaa 100644 --- a/src/components-lib/live/liveness/stream-liveness-controller.ts +++ b/src/components-lib/live/liveness/stream-liveness-controller.ts @@ -41,10 +41,19 @@ export type LivenessVerdict = renderPlaceholder?: boolean; }; -// The reconnecting placeholder the wrapper renders in place of a frozen -// provider, carrying the cause so it can show a cause-specific message. -interface LivenessPlaceholder { +// The wrapper-facing projection of the internal LivenessVerdict: it keeps the +// detector-internal `authority` from leaking out and collapses `unknown` into +// "no failure", so a consumer sees only a confirmed failure (with its cause) or +// nothing. Modeled as the failure rather than a `live` flag because a stream +// that is merely still connecting is not a failure yet is not playing either, so +// a positive `live` boolean would misleadingly read as "media is playing". +interface StreamFailure { reason: MediaUnavailableIssueReason; + + // Whether the wrapper should replace the provider with a reconnecting + // placeholder (a silent freeze, e.g. an unavailable camera). False when the + // provider renders its own error and should stay mounted. + renderPlaceholder: boolean; } export interface LivenessDetector { @@ -116,17 +125,15 @@ export class StreamLivenessController implements ReactiveController { } public isLive(): boolean { - return this._getVerdict().state !== 'not_live'; + return !this.getFailure(); } - // The reconnecting placeholder to render in place of the (frozen) provider, - // carrying the cause so the wrapper can show a cause-specific message. Null - // when the provider should stay mounted (live, or a provider error that - // renders its own error). - public getPlaceholder(): LivenessPlaceholder | null { + // The stream's confirmed liveness failure, or null when there is none (the + // stream is live, or still connecting). + public getFailure(): StreamFailure | null { const verdict = this._getVerdict(); - return verdict.state === 'not_live' && verdict.renderPlaceholder - ? { reason: verdict.reason } + return verdict.state === 'not_live' + ? { reason: verdict.reason, renderPlaceholder: !!verdict.renderPlaceholder } : null; } diff --git a/src/components-lib/live/providers/go2rtc-experimental/README.md b/src/components-lib/live/providers/go2rtc-experimental/README.md new file mode 100644 index 00000000..a0497a4d --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/README.md @@ -0,0 +1,13 @@ +# go2rtc-experimental provider + +![Architecture of the go2rtc-experimental provider](architecture.drawio.svg) + +The SVG is also its own editable source: it embeds the draw.io diagram, so open +`architecture.drawio.svg` directly in draw.io / diagrams.net to change it. + +## Maintaining the diagram + +`architecture.drawio.svg` is a dual file: the **rendered SVG** (shown above / +in GitHub) plus the **editable mxGraph XML** embedded in its `content="..."` +attribute. Keep them in sync -- never commit an SVG whose embedded XML changed +but whose rendered picture did not, or the image above goes stale. diff --git a/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts b/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts new file mode 100644 index 00000000..477fad1b --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts @@ -0,0 +1,148 @@ +// Wraps the browser's two incompatible MediaSource "flavors" behind one +// interface, so mse.ts stays flavor-agnostic (and mockable in tests): +// - classic MediaSource (Chrome/Firefox/desktop): attach via an object URL +// on `video.src`. +// - ManagedMediaSource (Safari/iOS 17+; iOS had no MediaSource at all before +// then): attach via `video.srcObject`, and needs `disableRemotePlayback`. +import type { UnsubscribeCallback } from '../../../../../types'; + +declare global { + interface Window { + // Safari 17+ managed variant of MediaSource; API-compatible for the + // subset used here. + ManagedMediaSource?: typeof MediaSource; + } +} + +// The unified MediaSource surface mse.ts programs against; the flavor-specific +// attach/detach lives in the implementations below. +export interface MediaSourceInterface { + attach(video: HTMLVideoElement): void; + detach(video: HTMLVideoElement): void; + + // Fires once the source is ready to accept SourceBuffers; nothing can be + // appended before it. + subscribeToSourceOpen(callback: () => void): UnsubscribeCallback; + + // Creates the SourceBuffer that media chunks are appended to, configured for + // the given codec MIME string. + addSourceBuffer(codecs: string): SourceBuffer; + + // Declares the seekable live window to the browser, which cannot infer it for + // an open-ended live source. + setLiveSeekableRange(startSeconds: number, endSeconds: number): void; + + // Whether the source is still attached and accepting SourceBuffer operations. + isOpen(): boolean; + + // Whether a codec MIME string is playable. + isTypeSupported(mimeType: string): boolean; +} + +// Creates a wrapped MediaSource, or null when the browser supports no variant. +export type MediaSourceFactory = () => MediaSourceInterface | null; + +abstract class MediaSourceInstanceBase implements MediaSourceInterface { + protected _mediaSource: MediaSource; + + constructor(mediaSource: MediaSource) { + this._mediaSource = mediaSource; + } + + public abstract attach(video: HTMLVideoElement): void; + public abstract detach(video: HTMLVideoElement): void; + public abstract isTypeSupported(mimeType: string): boolean; + + public subscribeToSourceOpen(callback: () => void): UnsubscribeCallback { + this._mediaSource.addEventListener('sourceopen', callback); + return () => this._mediaSource.removeEventListener('sourceopen', callback); + } + + public addSourceBuffer(codecs: string): SourceBuffer { + return this._mediaSource.addSourceBuffer(codecs); + } + + public setLiveSeekableRange(startSeconds: number, endSeconds: number): void { + this._mediaSource.setLiveSeekableRange(startSeconds, endSeconds); + } + + public isOpen(): boolean { + return this._mediaSource.readyState === 'open'; + } +} + +class ManagedMediaSourceInstance extends MediaSourceInstanceBase { + private _mediaSourceConstructor: typeof MediaSource; + + constructor(mediaSourceConstructor: typeof MediaSource) { + super(new mediaSourceConstructor()); + this._mediaSourceConstructor = mediaSourceConstructor; + } + + public attach(video: HTMLVideoElement): void { + // ManagedMediaSource does not deliver data while remote playback (e.g. + // AirPlay) is possible. + video.disableRemotePlayback = true; + video.srcObject = this._mediaSource; + } + + public detach(video: HTMLVideoElement): void { + video.srcObject = null; + } + + public isTypeSupported(mimeType: string): boolean { + return this._mediaSourceConstructor.isTypeSupported(mimeType); + } +} + +class ClassicMediaSourceInstance extends MediaSourceInstanceBase { + private _objectURL: string | null = null; + + constructor() { + super(new MediaSource()); + } + + public attach(video: HTMLVideoElement): void { + this._objectURL = URL.createObjectURL(this._mediaSource); + + // The object URL is only needed until the browser has opened the media + // source. + this._mediaSource.addEventListener('sourceopen', () => this._revokeObjectURL(), { + once: true, + }); + + video.src = this._objectURL; + video.srcObject = null; + } + + public detach(video: HTMLVideoElement): void { + video.src = ''; + this._revokeObjectURL(); + } + + public isTypeSupported(mimeType: string): boolean { + return MediaSource.isTypeSupported(mimeType); + } + + private _revokeObjectURL(): void { + if (this._objectURL) { + URL.revokeObjectURL(this._objectURL); + this._objectURL = null; + } + } +} + +export const createBrowserMediaSource: MediaSourceFactory = () => { + // Safari exposes both flavors, so check Managed first and prefer it there: it + // lets the browser throttle buffering to save battery and memory and hands + // off cleanly to AirPlay, which classic MediaSource does not. Classic is the + // fallback for other browsers. + const managedMediaSource = window.ManagedMediaSource; + if (managedMediaSource) { + return new ManagedMediaSourceInstance(managedMediaSource); + } + if ('MediaSource' in window) { + return new ClassicMediaSourceInstance(); + } + return null; +}; diff --git a/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts b/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts new file mode 100644 index 00000000..41679207 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts @@ -0,0 +1,14 @@ +export type PeerConnectionFactory = (config: RTCConfiguration) => RTCPeerConnection; + +export const GO2RTC_PEER_CONNECTION_CONFIG: RTCConfiguration = { + bundlePolicy: 'max-bundle', + iceServers: [ + { + // Two public STUN servers so connectivity survives one being unreachable. + urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'], + }, + ], +}; + +export const createBrowserPeerConnection: PeerConnectionFactory = (config) => + new RTCPeerConnection(config); diff --git a/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg new file mode 100644 index 00000000..df3bfa93 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg @@ -0,0 +1,4 @@ + + + +go2rtc-experimental provider -- architecture
<advanced-camera-card-live-go2rtc-experimental>
(components/live/providers/go2rtc-experimental)
Web component -- renders a <video> and an <img> (two surfaces), owns the session
Go2RTCSessionController (session-controller.ts)
Owns the signaling channel; starts both lanes; arbitrates the race
and commits the winner to the live surface (<video> or <img>)
via surfaceCommittedCallback. RetryTimer -- bounded reconnects.
BINARY lane
(binary frames on the WebSocket)
WEBRTC lane
(media over an RTCPeerConnection)
sources/factory.ts
createBinarySource(mode)
MSEStreamSource (sources/mse.ts)
MP4StreamSource (sources/mp4.ts)
MJPEGStreamSource (sources/mjpeg.ts)
MP4 & MJPEG extend ImageFrameStreamSource
sources/factory.ts
createWebRTCSource()
WebRTCStreamSource (sources/webrtc.ts)
negotiates SDP / ICE over the socket;
media rides an RTCPeerConnection
SignalingChannel (signaling.ts) -- one WebSocket to go2rtc
JSON text frames to message subscribers
binary frames to the single binary consumer
go2rtc server
adapters/media-source.ts
createBrowserMediaSource
(MSE / ManagedMediaSource)
utils/ (pure helpers)
codecs · live-edge-tracker · bounded-buffer-queue
base64 · user-agent · messages (binary sources)
webrtc-sdp · messages (WebRTC)
source-priority (the race)
<video> surface
(the live video element)
<img> surface
(ImageSurfaceController)
getPreferredSource (utils/source-priority.ts)
video > audio(AAC) > H.265, else WebRTC
OffscreenVideo (offscreen-video.ts)
off-screen <video> decode / probe
adapters/peer-connection.ts
createPeerConnection
(RTCPeerConnection)
connect(url, surfaces, modes)media-loadedinfo / error (escalate)start (sequential)start (parallel)fallbackfallbacknegotiate (JSON) + media (binary)signaling (JSON) onlyWebSocketmedia (RTP / SRTP)rendersrendersMP4 / MJPEGframesbinary profiledecodes intowebrtc profilewinner: MSE orWebRTC \ No newline at end of file diff --git a/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts b/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts new file mode 100644 index 00000000..00a4f0b4 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts @@ -0,0 +1,144 @@ +import type { LitElement } from 'lit'; + +import { Generation } from '../../../../utils/concurrency/generation'; +import { LatestValueRunner } from '../../../../utils/concurrency/latest-value-runner'; +import { ImageMediaPlayerController } from '../../../media-player/image'; +import { OffscreenImage } from './offscreen-image'; +import type { ImageSurface } from './session-controller'; + +// Liveness: while frames are expected, a gap beyond the window is a stall. +// Omitted -> the surface reports no liveness. `stallWindowSeconds` defaults to +// the standard frame-stall window. +interface ImageSurfaceLivenessOptions { + isFrameExpected: () => boolean; + stallWindowSeconds?: number; +} + +interface ImageSurfaceOptions { + livenessOptions?: ImageSurfaceLivenessOptions; + + // Factory for the detached image loader used to decode a frame. + createImage?: () => HTMLImageElement; +} + +// Presents a go2rtc MJPEG/MP4 stream as a sequence of frames on an : each +// frame is decoded off-DOM, then shown via an object URL (the previous one +// revoked), and an ImageMediaPlayerController exposes it to the card as a media +// player. The renderer owns the element; this owns what is shown on it. +export class ImageSurfaceController implements ImageSurface { + private _getImageCallback: () => HTMLImageElement | null; + private _mediaPlayerController: ImageMediaPlayerController; + + // An off-screen used to decode each frame before it is shown on the + // visible element (reused: only one frame decodes at a time). + private _decoder: OffscreenImage; + + // Current frame's object URL. + private _currentObjectURL: string | null = null; + + // Invalidated by reset() so a frame whose decode was still in flight when the + // surface was switched or torn down does not paint a stale frame afterwards. + private _generation = new Generation(); + + // Frames can arrive faster than the browser decodes one. Present at most one + // at a time and keep only the newest frame while busy, so the display + // converges on the latest without unbounded decode work. + private _frameRunner = new LatestValueRunner((frame) => + this._presentFrame(frame), + ); + + constructor( + host: LitElement, + getImageCallback: () => HTMLImageElement | null, + options?: ImageSurfaceOptions, + ) { + this._getImageCallback = getImageCallback; + this._decoder = new OffscreenImage(options?.createImage); + this._mediaPlayerController = new ImageMediaPlayerController( + host, + getImageCallback, + { + livenessOptions: options?.livenessOptions, + }, + ); + } + + public getElement(): HTMLImageElement | null { + return this._getImageCallback(); + } + + public getMediaPlayer(): ImageMediaPlayerController { + return this._mediaPlayerController; + } + + // Show a new frame. A newer frame supersedes any still waiting to show; the + // returned promise resolves once *a* frame has been presented. + public showFrame(blob: Blob): Promise { + return this._frameRunner.submit(blob); + } + + private async _presentFrame(blob: Blob): Promise { + const image = this._getImageCallback(); + + // Drop frames for a detached element (e.g. a surface superseded + // mid-teardown) so a retired surface never paints. + if (!image || !image.isConnected) { + return; + } + + const generation = this._generation.current(); + const url = URL.createObjectURL(blob); + + // Decode the frame on the off-screen loader before showing it. Assigning an + // undecoded object URL to the visible makes WebKit repaint the + // element empty until the decode completes, flashing the media background + // between frames; decoding first lets the swap paint from cache (as it's + // the same url) with no empty state. + const decoder = this._decoder.get(); + decoder.src = url; + try { + await decoder.decode(); + } catch { + // An undecodable frame is not fatal: drop it and keep the current one. + URL.revokeObjectURL(url); + return; + } + + // The surface may have been switched (reset) or detached during the decode; + // painting now would show a stale frame on a retired surface. + if (!this._generation.isCurrent(generation) || !image.isConnected) { + URL.revokeObjectURL(url); + return; + } + + image.src = url; + + const previousURL = this._currentObjectURL; + this._currentObjectURL = url; + if (previousURL) { + URL.revokeObjectURL(previousURL); + } + } + + // Drop the current frame and its object URL (e.g. on a surface switch or + // disconnect). + public reset(): void { + // Invalidate any frame still decoding so it cannot paint onto a surface + // that has since been switched or disconnected. + this._generation.invalidate(); + + // Drop any frame still waiting to show, for the same reason. + this._frameRunner.clear(); + + if (this._currentObjectURL) { + URL.revokeObjectURL(this._currentObjectURL); + this._currentObjectURL = null; + } + this._decoder.clear(); + + const image = this._getImageCallback(); + if (image) { + image.removeAttribute('src'); + } + } +} diff --git a/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts b/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts new file mode 100644 index 00000000..d9c5f39e --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts @@ -0,0 +1,26 @@ +type ImageElementFactory = () => HTMLImageElement; + +// A held, injectable off-screen for decoding a frame separate from the +// visible display element. Created lazily on first `get()` and reused until +// `clear()` detaches its source and forgets it. +export class OffscreenImage { + private _create: ImageElementFactory; + private _image: HTMLImageElement | null = null; + + constructor(create?: ImageElementFactory) { + this._create = create ?? (() => new Image()); + } + + // Return the held image, creating it on first use. + public get(): HTMLImageElement { + return (this._image ??= this._create()); + } + + // Detach any source and forget the image. Safe to call when none is held. + public clear(): void { + if (this._image) { + this._image.removeAttribute('src'); + this._image = null; + } + } +} diff --git a/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts b/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts new file mode 100644 index 00000000..3ad94d40 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts @@ -0,0 +1,28 @@ +export type VideoElementFactory = () => HTMLVideoElement; + +// A held, injectable off-screen