From c02c692f68e3fbf161ff06506042e7156ffb060b Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Tue, 14 Jul 2026 14:22:41 -0700 Subject: [PATCH] feat: Add experimental rewrite of go2rtc live provider (MSE/WebRTC/MP4/MJPEG) (#2580) - Closes #2556 - Closes #2450 **Key intended features:** - go2rtc compatible - 100% test coverage to significantly improve ability to test, maintain and work around browser weirdnesses (e.g. Safari). - Written from the ground up in the style of the rest of the project. **To use:** - Change `live_provider` from `go2rtc` to `go2rtc-experimental`. --- docs/configuration/cameras/live-provider.md | 44 +- docs/usage/2-way-audio.md | 2 +- eslint.config.mjs | 2 +- src/camera-manager/camera.ts | 12 +- src/card-controller/actions/actions/pause.ts | 5 +- src/card-controller/actions/actions/play.ts | 2 +- src/card-controller/call/manager.ts | 9 +- .../issues/issues/media-unavailable.ts | 17 +- .../live/liveness/detectors/provider-error.ts | 14 +- .../liveness/stream-liveness-controller.ts | 29 +- .../providers/go2rtc-experimental/README.md | 13 + .../adapters/media-source.ts | 148 ++ .../adapters/peer-connection.ts | 14 + .../architecture.drawio.svg | 4 + .../image-surface-controller.ts | 144 ++ .../go2rtc-experimental/offscreen-image.ts | 26 + .../go2rtc-experimental/offscreen-video.ts | 28 + .../go2rtc-experimental/session-controller.ts | 721 ++++++++++ .../go2rtc-experimental/signaling.ts | 149 ++ .../go2rtc-experimental/sources/factory.ts | 102 ++ .../sources/image-frame.ts | 112 ++ .../go2rtc-experimental/sources/mjpeg.ts | 16 + .../go2rtc-experimental/sources/mp4.ts | 98 ++ .../go2rtc-experimental/sources/mse.ts | 347 +++++ .../go2rtc-experimental/sources/webrtc.ts | 291 ++++ .../providers/go2rtc-experimental/types.ts | 129 ++ .../go2rtc-experimental/utils/base64.ts | 11 + .../utils/bounded-buffer-queue.ts | 42 + .../go2rtc-experimental/utils/codecs.ts | 100 ++ .../utils/failure-reason.ts | 27 + .../gop-cadence-estimator.ts | 61 + .../utils/live-edge-tracker/index.ts | 22 + .../utils/live-edge-tracker/non-webkit.ts | 66 + .../utils/live-edge-tracker/types.ts | 17 + .../utils/live-edge-tracker/webkit.ts | 72 + .../go2rtc-experimental/utils/messages.ts | 9 + .../utils/source-priority.ts | 31 + .../go2rtc-experimental/utils/user-agent.ts | 13 + .../go2rtc-experimental/utils/webrtc-sdp.ts | 6 + .../live/utils/dispatch-live-error.ts | 19 +- .../media-actions-controller.ts | 4 +- .../media-player/frame-stall-watchdog.ts | 13 +- src/components-lib/media-player/image.ts | 134 +- src/components-lib/media-player/jsmpeg.ts | 35 +- .../media-player/updating-image.ts | 69 - src/components-lib/media-player/video.ts | 76 +- src/components-lib/menu-button-controller.ts | 2 +- src/components-lib/notification/media.ts | 52 + src/components-lib/signed-url-controller.ts | 21 +- src/components/gallery/gallery.ts | 14 +- src/components/image-updating-player.ts | 76 +- src/components/live/carousel.ts | 3 +- src/components/live/provider.ts | 131 +- .../providers/go2rtc-experimental/index.ts | 299 ++++ src/components/live/providers/go2rtc/index.ts | 20 +- src/components/live/providers/image.ts | 10 + src/components/live/providers/jsmpeg.ts | 33 +- src/components/live/providers/webrtc-card.ts | 27 +- src/components/notification/media.ts | 43 + src/components/notification/no-media.ts | 32 - src/components/viewer/carousel.ts | 28 +- src/components/viewer/index.ts | 14 +- src/config/schema/cameras.ts | 6 +- src/editor.ts | 4 + src/localize/languages/de.json | 1 - src/localize/languages/en.json | 8 +- src/localize/languages/pl.json | 1 - src/patches/ha-hls-player.ts | 9 +- src/patches/ha-web-rtc-player.ts | 13 +- src/scss/live-go2rtc-experimental.scss | 33 + src/scss/live-provider.scss | 16 + src/types.ts | 22 +- src/utils/audio.ts | 17 +- src/utils/basic.ts | 4 - src/utils/concurrency/generation.ts | 29 + src/utils/concurrency/latest-value-runner.ts | 71 + src/utils/live-provider.ts | 9 +- src/utils/websocket-url.ts | 10 + tests/camera-manager/camera.test.ts | 33 +- .../actions/actions/pause.test.ts | 8 +- .../actions/actions/play.test.ts | 8 +- .../liveness/detectors/provider-error.test.ts | 16 + .../stream-liveness-controller.test.ts | 27 +- .../adapters/media-source.test.ts | 226 +++ .../adapters/peer-connection.test.ts | 29 + .../image-surface-controller.test.ts | 232 +++ .../offscreen-image.test.ts | 54 + .../offscreen-video.test.ts | 59 + .../session-controller.test.ts | 1273 +++++++++++++++++ .../go2rtc-experimental/signaling.test.ts | 251 ++++ .../sources/factory.test.ts | 90 ++ .../go2rtc-experimental/sources/mjpeg.test.ts | 170 +++ .../go2rtc-experimental/sources/mp4.test.ts | 232 +++ .../go2rtc-experimental/sources/mse.test.ts | 646 +++++++++ .../sources/webrtc.test.ts | 549 +++++++ .../go2rtc-experimental/test-utils.ts | 270 ++++ .../go2rtc-experimental/utils/base64.test.ts | 15 + .../utils/bounded-buffer-queue.test.ts | 64 + .../go2rtc-experimental/utils/codecs.test.ts | 102 ++ .../utils/failure-reason.test.ts | 21 + .../gop-cadence-estimator.test.ts | 54 + .../utils/live-edge-tracker/index.test.ts | 17 + .../live-edge-tracker/non-webkit.test.ts | 75 + .../utils/live-edge-tracker/test-utils.ts | 10 + .../utils/live-edge-tracker/webkit.test.ts | 100 ++ .../utils/messages.test.ts | 29 + .../utils/source-priority.test.ts | 76 + .../utils/user-agent.test.ts | 86 ++ .../utils/webrtc-sdp.test.ts | 13 + .../live/utils/dispatch-live-error.test.ts | 11 + .../media-actions-controller.test.ts | 110 +- .../components-lib/media-player/image.test.ts | 287 +++- .../media-player/jsmpeg.test.ts | 22 +- .../media-player/updating-image.test.ts | 203 --- .../components-lib/media-player/video.test.ts | 80 +- .../menu-button-controller.test.ts | 15 +- .../components-lib/notification/media.test.ts | 84 ++ tests/components/notification/media.test.ts | 54 + .../components/notification/no-media.test.ts | 75 - tests/utils/audio.test.ts | 32 +- tests/utils/basic.test.ts | 21 - tests/utils/concurrency/generation.test.ts | 38 + .../concurrency/latest-value-runner.test.ts | 117 ++ tests/utils/live-provider.test.ts | 30 + tests/utils/websocket-url.test.ts | 38 + 125 files changed, 9608 insertions(+), 807 deletions(-) create mode 100644 src/components-lib/live/providers/go2rtc-experimental/README.md create mode 100644 src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg create mode 100644 src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/session-controller.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/signaling.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/factory.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/image-frame.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/mp4.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/mse.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/sources/webrtc.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/types.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/base64.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/bounded-buffer-queue.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/codecs.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/failure-reason.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/gop-cadence-estimator.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/index.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/non-webkit.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/types.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/webkit.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/messages.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/source-priority.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/user-agent.ts create mode 100644 src/components-lib/live/providers/go2rtc-experimental/utils/webrtc-sdp.ts delete mode 100644 src/components-lib/media-player/updating-image.ts create mode 100644 src/components-lib/notification/media.ts create mode 100644 src/components/live/providers/go2rtc-experimental/index.ts create mode 100644 src/components/notification/media.ts delete mode 100644 src/components/notification/no-media.ts create mode 100644 src/scss/live-go2rtc-experimental.scss create mode 100644 src/utils/concurrency/generation.ts create mode 100644 src/utils/concurrency/latest-value-runner.ts create mode 100644 src/utils/websocket-url.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/adapters/media-source.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/image-surface-controller.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/offscreen-image.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/offscreen-video.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/session-controller.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/signaling.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/sources/factory.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/sources/mp4.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/sources/mse.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/sources/webrtc.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/test-utils.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/base64.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/bounded-buffer-queue.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/codecs.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/failure-reason.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/gop-cadence-estimator.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/index.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/non-webkit.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/test-utils.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/webkit.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/messages.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/source-priority.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/user-agent.test.ts create mode 100644 tests/components-lib/live/providers/go2rtc-experimental/utils/webrtc-sdp.test.ts delete mode 100644 tests/components-lib/media-player/updating-image.test.ts create mode 100644 tests/components-lib/notification/media.test.ts create mode 100644 tests/components/notification/media.test.ts delete mode 100644 tests/components/notification/no-media.test.ts create mode 100644 tests/utils/concurrency/generation.test.ts create mode 100644 tests/utils/concurrency/latest-value-runner.test.ts create mode 100644 tests/utils/websocket-url.test.ts diff --git a/docs/configuration/cameras/live-provider.md b/docs/configuration/cameras/live-provider.md index bbb8291c..1257dc88 100644 --- a/docs/configuration/cameras/live-provider.md +++ b/docs/configuration/cameras/live-provider.md @@ -4,15 +4,16 @@ The `live_provider` parameter determines what provides the live stream for a camera. Each provider offers different capabilities: -| Live Provider | Latency | Frame Rate | Loading Time | Installation | Supports [Proxying](./README.md?id=proxy) | Description | -| ---------------------------------- | ------- | ---------- | ------------ | ------------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `go2rtc` | Best | High | Better | Builtin | :white_check_mark: | Uses [go2rtc](https://github.com/AlexxIT/go2rtc) to stream live feeds and supports 2-way audio. | -| `ha` (Native WebRTC) | Best | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- will offer a very low-latency feed direct to your browser. | -| `ha` (HLS) | Poor | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- HLS fallback when a WebRTC connection cannot be established. | -| `ha` (when configured with LL-HLS) | Better | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- can be configured to use an [LL-HLS](https://www.home-assistant.io/integrations/stream/#ll-hls) feed for lower latency. | -| `image` | Poor | Poor | Best | Builtin | :heavy_multiplication_x: | Use refreshing snapshots of the built-in Home Assistant camera streams. | -| `jsmpeg` | Better | Low | Poor | Builtin | :heavy_multiplication_x: | Use a the JSMPEG stream. | -| `webrtc-card` | Best | High | Better | Separate installation required | :heavy_multiplication_x: | Embed's [AlexxIT's WebRTC Card](https://github.com/AlexxIT/WebRTC) to stream live feed, requires manual extra setup. See [`webrtc_card`](#webrtc_card). Not to be confused with native Home Assistant WebRTC (use the `ha` provider). | +| Live Provider | Latency | Frame Rate | Loading Time | Installation | Supports [Proxying](./README.md?id=proxy) | Description | +| ---------------------------------- | ------- | ---------- | ------------ | ------------------------------ | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `go2rtc` | Best | High | Better | Builtin | :white_check_mark: | Uses [go2rtc](https://github.com/AlexxIT/go2rtc) to stream live feeds and supports 2-way audio. | +| `go2rtc-experimental` | Best | High | Better | Builtin | :white_check_mark: | **Experimental** re-implementation of the `go2rtc` provider client (MSE/WebRTC/MP4/MJPEG) that is expected to eventually replace the `go2rtc` live provider. Uses the same configuration as `go2rtc`. See [`go2rtc (experimental)`](#go2rtc-experimental). | +| `ha` (Native WebRTC) | Best | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- will offer a very low-latency feed direct to your browser. | +| `ha` (HLS) | Poor | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- HLS fallback when a WebRTC connection cannot be established. | +| `ha` (when configured with LL-HLS) | Better | High | Better | Builtin | :heavy_multiplication_x: | Use the built-in Home Assistant camera streams -- can be configured to use an [LL-HLS](https://www.home-assistant.io/integrations/stream/#ll-hls) feed for lower latency. | +| `image` | Poor | Poor | Best | Builtin | :heavy_multiplication_x: | Use refreshing snapshots of the built-in Home Assistant camera streams. | +| `jsmpeg` | Better | Low | Poor | Builtin | :heavy_multiplication_x: | Use a the JSMPEG stream. | +| `webrtc-card` | Best | High | Better | Separate installation required | :heavy_multiplication_x: | Embed's [AlexxIT's WebRTC Card](https://github.com/AlexxIT/WebRTC) to stream live feed, requires manual extra setup. See [`webrtc_card`](#webrtc_card). Not to be confused with native Home Assistant WebRTC (use the `ha` provider). | ## `go2rtc` @@ -43,6 +44,20 @@ cameras: > [`capabilities.force`](./README.md?id=capabilities) to skip metadata > detection entirely. +## `go2rtc (experimental)` + +> [!WARNING] `go2rtc-experimental` is a from-scratch re-implementation of the [`go2rtc`](#go2rtc) live provider client (MSE, WebRTC, MP4 and MJPEG). It is under active development and is expected to eventually replace the `go2rtc` live provider. Please try it and [report any issues you encounter](https://github.com/dermotduffy/advanced-camera-card/issues). + +It uses the exact same configuration as [`go2rtc`](#go2rtc): select it with `live_provider: go2rtc-experimental` and configure the same [`go2rtc`](#go2rtc) block. + +```yaml +cameras: + - camera_entity: camera.office + live_provider: go2rtc-experimental + go2rtc: + # [...] +``` + ## `ha` The `ha` block configures use of the default Home Assistant (`ha`) live provider. It has no configuration options. @@ -164,6 +179,17 @@ cameras: stream: sitting_room url: 'https://my.custom.go2rtc.backend' metadata_fetch_timeout_seconds: 2 + - camera_entity: camera.office_go2rtc_experimental + live_provider: go2rtc-experimental + go2rtc: + modes: + - webrtc + - mse + - mp4 + - mjpeg + stream: sitting_room + url: 'https://my.custom.go2rtc.backend' + metadata_fetch_timeout_seconds: 2 - camera_entity: camera.office_jsmpeg live_provider: jsmpeg jsmpeg: diff --git a/docs/usage/2-way-audio.md b/docs/usage/2-way-audio.md index dcc0a18d..8d14ddbc 100644 --- a/docs/usage/2-way-audio.md +++ b/docs/usage/2-way-audio.md @@ -16,7 +16,7 @@ challenging. ### Card requirements - Only Frigate cameras are supported. -- Only the `go2rtc` live provider is supported. +- Only the `go2rtc` and `go2rtc-experimental` live providers are supported. - Only the `webrtc` mode supports 2-way audio. If your setup supports 2-way audio but detection is intermittent on load: diff --git a/eslint.config.mjs b/eslint.config.mjs index bcb99f66..174c3d99 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -106,7 +106,7 @@ export default defineConfig([ languageOptions: { parser: tsParser, parserOptions: { - project: './tsconfig.json', + projectService: true, tsconfigRootDir: __dirname, }, }, diff --git a/src/camera-manager/camera.ts b/src/camera-manager/camera.ts index 08c5b941..1937b90c 100644 --- a/src/camera-manager/camera.ts +++ b/src/camera-manager/camera.ts @@ -18,7 +18,10 @@ import type { HassStateDifference, HomeAssistant } from '../ha/types'; import { localize } from '../localize/localize'; import type { CapabilitiesRaw, CapabilityKey, Endpoint } from '../types'; import { arrayify } from '../utils/basic'; -import { liveProviderSupports2WayAudio } from '../utils/live-provider'; +import { + isGo2RTCLiveProvider, + liveProviderSupports2WayAudio, +} from '../utils/live-provider'; import { Capabilities } from './capabilities'; import type { CameraManagerEngine } from './engine'; import { CameraNoIDError } from './error'; @@ -345,9 +348,10 @@ export class Camera { ...resolveProxyConfig(this._config.proxy), live: this._config.proxy.live === 'auto' - ? // Live is proxied if the live provider is go2rtc and if a go2rtc - // URL is manually set. - this._config.live_provider === 'go2rtc' && !!this._config.go2rtc?.url + ? // Live is proxied if the live provider streams from go2rtc and a + // go2rtc URL is manually set. + isGo2RTCLiveProvider(this._config.live_provider) && + !!this._config.go2rtc?.url : this._config.proxy.live, media: this._config.proxy.media === 'auto' ? false : this._config.proxy.media, }; diff --git a/src/card-controller/actions/actions/pause.ts b/src/card-controller/actions/actions/pause.ts index da50bb51..b2e717d6 100644 --- a/src/card-controller/actions/actions/pause.ts +++ b/src/card-controller/actions/actions/pause.ts @@ -6,6 +6,9 @@ export class PauseAction extends AdvancedCameraCardAction { public async execute(api: CardActionsAPI): Promise { await super.execute(api); - await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.pause(); + await api + .getMediaLoadedInfoManager() + .get() + ?.mediaPlayerController?.playback?.pause(); } } diff --git a/src/card-controller/actions/actions/play.ts b/src/card-controller/actions/actions/play.ts index a45b85b5..3837fa98 100644 --- a/src/card-controller/actions/actions/play.ts +++ b/src/card-controller/actions/actions/play.ts @@ -6,6 +6,6 @@ export class PlayAction extends AdvancedCameraCardAction { public async execute(api: CardActionsAPI): Promise { await super.execute(api); - await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.play(); + await api.getMediaLoadedInfoManager().get()?.mediaPlayerController?.playback?.play(); } } diff --git a/src/card-controller/call/manager.ts b/src/card-controller/call/manager.ts index cd1a9bcf..8ea3b11b 100644 --- a/src/card-controller/call/manager.ts +++ b/src/card-controller/call/manager.ts @@ -1,6 +1,7 @@ import { createNotificationFromText } from '../../components-lib/notification/factory'; import type { ConditionStateChange } from '../../condition-trigger/conditions/types'; import { localize } from '../../localize/localize'; +import { Generation } from '../../utils/concurrency/generation'; import { Timer } from '../../utils/timer'; import { getStreamCameraID } from '../../view/substream'; import type { View } from '../../view/view'; @@ -22,7 +23,7 @@ export class CallManager { // resumed tail leaks audio onto the shared lock from an instance the user // can no longer see or control, and may install state into a fresh // lifecycle from a request that belongs to the previous one. - private _initEpoch = 0; + private _initGeneration = new Generation(); constructor(api: CardCallAPI) { this._api = api; @@ -103,7 +104,7 @@ export class CallManager { return false; } - const initEpoch = this._initEpoch; + const initGeneration = this._initGeneration.current(); const microphoneConnected = await this._connectMicrophone(); // If the init/uninit lifecycle advanced while the microphone connect was // in flight, this request belongs to a previous lifecycle -- the view we @@ -111,7 +112,7 @@ export class CallManager { // clean teardown path for state we'd install here. Bail before touching // `_call`, the ringtone lock, or surfacing a notification onto a torn-down // NotificationManager. - if (initEpoch !== this._initEpoch) { + if (!this._initGeneration.isCurrent(initGeneration)) { return false; } if (!microphoneConnected) { @@ -241,7 +242,7 @@ export class CallManager { // // Safe to re-initialize afterwards via `initialize()`. public uninitialize(): void { - this._initEpoch++; + this._initGeneration.invalidate(); this._ringtone.stop(); this._unansweredTimer.stop(); if (this._call) { diff --git a/src/card-controller/issues/issues/media-unavailable.ts b/src/card-controller/issues/issues/media-unavailable.ts index 0eca830d..f5002460 100644 --- a/src/card-controller/issues/issues/media-unavailable.ts +++ b/src/card-controller/issues/issues/media-unavailable.ts @@ -21,7 +21,10 @@ export type MediaUnavailableIssueReason = | 'entity_unavailable' | 'not_loading' | 'playback_error' - | 'stalled'; + | 'server_error' + | 'stalled' + | 'two_way_audio_error' + | 'unsupported'; declare module 'issue' { interface IssueTriggerContext { @@ -50,10 +53,22 @@ export const MEDIA_UNAVAILABLE_REASONS: Record< localizationKey: 'issues.media_unavailable.reasons.playback_error', icon: 'mdi:alert-circle', }, + server_error: { + localizationKey: 'issues.media_unavailable.reasons.server_error', + icon: 'mdi:server-network-off', + }, stalled: { localizationKey: 'issues.media_unavailable.reasons.stalled', icon: 'mdi:motion-pause', }, + two_way_audio_error: { + localizationKey: 'issues.media_unavailable.reasons.two_way_audio_error', + icon: 'mdi:microphone-off', + }, + unsupported: { + localizationKey: 'issues.media_unavailable.reasons.unsupported', + icon: 'mdi:video-off-outline', + }, }; export class MediaUnavailableIssue implements Issue { diff --git a/src/components-lib/live/liveness/detectors/provider-error.ts b/src/components-lib/live/liveness/detectors/provider-error.ts index ec4a9096..67d6e7b7 100644 --- a/src/components-lib/live/liveness/detectors/provider-error.ts +++ b/src/components-lib/live/liveness/detectors/provider-error.ts @@ -1,3 +1,4 @@ +import type { MediaUnavailableIssueReason } from '../../../../card-controller/issues/issues/media-unavailable'; import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller'; const LIVE_ERROR_EVENT = 'advanced-camera-card:live:error'; @@ -38,12 +39,19 @@ export class ProviderErrorDetector implements LivenessDetector { return this._verdict; } - private _handler = (ev: Event): void => { + private _handler = ( + ev: CustomEvent, + ): void => { ev.stopPropagation(); if (this._verdict.state !== 'not_live') { // Authoritative: an explicit provider error overrides even direct frame - // evidence. No placeholder -- the provider renders its own error. - this._verdict = { state: 'not_live', authority: 'hard', reason: 'playback_error' }; + // evidence. No placeholder -- the provider renders its own error. The + // provider may name the cause; otherwise it is a generic playback error. + this._verdict = { + state: 'not_live', + authority: 'hard', + reason: ev.detail ?? 'playback_error', + }; this._onChange(); } }; diff --git a/src/components-lib/live/liveness/stream-liveness-controller.ts b/src/components-lib/live/liveness/stream-liveness-controller.ts index bfd2ee9c..c6fc6eaa 100644 --- a/src/components-lib/live/liveness/stream-liveness-controller.ts +++ b/src/components-lib/live/liveness/stream-liveness-controller.ts @@ -41,10 +41,19 @@ export type LivenessVerdict = renderPlaceholder?: boolean; }; -// The reconnecting placeholder the wrapper renders in place of a frozen -// provider, carrying the cause so it can show a cause-specific message. -interface LivenessPlaceholder { +// The wrapper-facing projection of the internal LivenessVerdict: it keeps the +// detector-internal `authority` from leaking out and collapses `unknown` into +// "no failure", so a consumer sees only a confirmed failure (with its cause) or +// nothing. Modeled as the failure rather than a `live` flag because a stream +// that is merely still connecting is not a failure yet is not playing either, so +// a positive `live` boolean would misleadingly read as "media is playing". +interface StreamFailure { reason: MediaUnavailableIssueReason; + + // Whether the wrapper should replace the provider with a reconnecting + // placeholder (a silent freeze, e.g. an unavailable camera). False when the + // provider renders its own error and should stay mounted. + renderPlaceholder: boolean; } export interface LivenessDetector { @@ -116,17 +125,15 @@ export class StreamLivenessController implements ReactiveController { } public isLive(): boolean { - return this._getVerdict().state !== 'not_live'; + return !this.getFailure(); } - // The reconnecting placeholder to render in place of the (frozen) provider, - // carrying the cause so the wrapper can show a cause-specific message. Null - // when the provider should stay mounted (live, or a provider error that - // renders its own error). - public getPlaceholder(): LivenessPlaceholder | null { + // The stream's confirmed liveness failure, or null when there is none (the + // stream is live, or still connecting). + public getFailure(): StreamFailure | null { const verdict = this._getVerdict(); - return verdict.state === 'not_live' && verdict.renderPlaceholder - ? { reason: verdict.reason } + return verdict.state === 'not_live' + ? { reason: verdict.reason, renderPlaceholder: !!verdict.renderPlaceholder } : null; } diff --git a/src/components-lib/live/providers/go2rtc-experimental/README.md b/src/components-lib/live/providers/go2rtc-experimental/README.md new file mode 100644 index 00000000..a0497a4d --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/README.md @@ -0,0 +1,13 @@ +# go2rtc-experimental provider + +![Architecture of the go2rtc-experimental provider](architecture.drawio.svg) + +The SVG is also its own editable source: it embeds the draw.io diagram, so open +`architecture.drawio.svg` directly in draw.io / diagrams.net to change it. + +## Maintaining the diagram + +`architecture.drawio.svg` is a dual file: the **rendered SVG** (shown above / +in GitHub) plus the **editable mxGraph XML** embedded in its `content="..."` +attribute. Keep them in sync -- never commit an SVG whose embedded XML changed +but whose rendered picture did not, or the image above goes stale. diff --git a/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts b/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts new file mode 100644 index 00000000..477fad1b --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/adapters/media-source.ts @@ -0,0 +1,148 @@ +// Wraps the browser's two incompatible MediaSource "flavors" behind one +// interface, so mse.ts stays flavor-agnostic (and mockable in tests): +// - classic MediaSource (Chrome/Firefox/desktop): attach via an object URL +// on `video.src`. +// - ManagedMediaSource (Safari/iOS 17+; iOS had no MediaSource at all before +// then): attach via `video.srcObject`, and needs `disableRemotePlayback`. +import type { UnsubscribeCallback } from '../../../../../types'; + +declare global { + interface Window { + // Safari 17+ managed variant of MediaSource; API-compatible for the + // subset used here. + ManagedMediaSource?: typeof MediaSource; + } +} + +// The unified MediaSource surface mse.ts programs against; the flavor-specific +// attach/detach lives in the implementations below. +export interface MediaSourceInterface { + attach(video: HTMLVideoElement): void; + detach(video: HTMLVideoElement): void; + + // Fires once the source is ready to accept SourceBuffers; nothing can be + // appended before it. + subscribeToSourceOpen(callback: () => void): UnsubscribeCallback; + + // Creates the SourceBuffer that media chunks are appended to, configured for + // the given codec MIME string. + addSourceBuffer(codecs: string): SourceBuffer; + + // Declares the seekable live window to the browser, which cannot infer it for + // an open-ended live source. + setLiveSeekableRange(startSeconds: number, endSeconds: number): void; + + // Whether the source is still attached and accepting SourceBuffer operations. + isOpen(): boolean; + + // Whether a codec MIME string is playable. + isTypeSupported(mimeType: string): boolean; +} + +// Creates a wrapped MediaSource, or null when the browser supports no variant. +export type MediaSourceFactory = () => MediaSourceInterface | null; + +abstract class MediaSourceInstanceBase implements MediaSourceInterface { + protected _mediaSource: MediaSource; + + constructor(mediaSource: MediaSource) { + this._mediaSource = mediaSource; + } + + public abstract attach(video: HTMLVideoElement): void; + public abstract detach(video: HTMLVideoElement): void; + public abstract isTypeSupported(mimeType: string): boolean; + + public subscribeToSourceOpen(callback: () => void): UnsubscribeCallback { + this._mediaSource.addEventListener('sourceopen', callback); + return () => this._mediaSource.removeEventListener('sourceopen', callback); + } + + public addSourceBuffer(codecs: string): SourceBuffer { + return this._mediaSource.addSourceBuffer(codecs); + } + + public setLiveSeekableRange(startSeconds: number, endSeconds: number): void { + this._mediaSource.setLiveSeekableRange(startSeconds, endSeconds); + } + + public isOpen(): boolean { + return this._mediaSource.readyState === 'open'; + } +} + +class ManagedMediaSourceInstance extends MediaSourceInstanceBase { + private _mediaSourceConstructor: typeof MediaSource; + + constructor(mediaSourceConstructor: typeof MediaSource) { + super(new mediaSourceConstructor()); + this._mediaSourceConstructor = mediaSourceConstructor; + } + + public attach(video: HTMLVideoElement): void { + // ManagedMediaSource does not deliver data while remote playback (e.g. + // AirPlay) is possible. + video.disableRemotePlayback = true; + video.srcObject = this._mediaSource; + } + + public detach(video: HTMLVideoElement): void { + video.srcObject = null; + } + + public isTypeSupported(mimeType: string): boolean { + return this._mediaSourceConstructor.isTypeSupported(mimeType); + } +} + +class ClassicMediaSourceInstance extends MediaSourceInstanceBase { + private _objectURL: string | null = null; + + constructor() { + super(new MediaSource()); + } + + public attach(video: HTMLVideoElement): void { + this._objectURL = URL.createObjectURL(this._mediaSource); + + // The object URL is only needed until the browser has opened the media + // source. + this._mediaSource.addEventListener('sourceopen', () => this._revokeObjectURL(), { + once: true, + }); + + video.src = this._objectURL; + video.srcObject = null; + } + + public detach(video: HTMLVideoElement): void { + video.src = ''; + this._revokeObjectURL(); + } + + public isTypeSupported(mimeType: string): boolean { + return MediaSource.isTypeSupported(mimeType); + } + + private _revokeObjectURL(): void { + if (this._objectURL) { + URL.revokeObjectURL(this._objectURL); + this._objectURL = null; + } + } +} + +export const createBrowserMediaSource: MediaSourceFactory = () => { + // Safari exposes both flavors, so check Managed first and prefer it there: it + // lets the browser throttle buffering to save battery and memory and hands + // off cleanly to AirPlay, which classic MediaSource does not. Classic is the + // fallback for other browsers. + const managedMediaSource = window.ManagedMediaSource; + if (managedMediaSource) { + return new ManagedMediaSourceInstance(managedMediaSource); + } + if ('MediaSource' in window) { + return new ClassicMediaSourceInstance(); + } + return null; +}; diff --git a/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts b/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts new file mode 100644 index 00000000..41679207 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts @@ -0,0 +1,14 @@ +export type PeerConnectionFactory = (config: RTCConfiguration) => RTCPeerConnection; + +export const GO2RTC_PEER_CONNECTION_CONFIG: RTCConfiguration = { + bundlePolicy: 'max-bundle', + iceServers: [ + { + // Two public STUN servers so connectivity survives one being unreachable. + urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'], + }, + ], +}; + +export const createBrowserPeerConnection: PeerConnectionFactory = (config) => + new RTCPeerConnection(config); diff --git a/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg new file mode 100644 index 00000000..df3bfa93 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg @@ -0,0 +1,4 @@ + + + +go2rtc-experimental provider -- architecture
<advanced-camera-card-live-go2rtc-experimental>
(components/live/providers/go2rtc-experimental)
Web component -- renders a <video> and an <img> (two surfaces), owns the session
Go2RTCSessionController (session-controller.ts)
Owns the signaling channel; starts both lanes; arbitrates the race
and commits the winner to the live surface (<video> or <img>)
via surfaceCommittedCallback. RetryTimer -- bounded reconnects.
BINARY lane
(binary frames on the WebSocket)
WEBRTC lane
(media over an RTCPeerConnection)
sources/factory.ts
createBinarySource(mode)
MSEStreamSource (sources/mse.ts)
MP4StreamSource (sources/mp4.ts)
MJPEGStreamSource (sources/mjpeg.ts)
MP4 & MJPEG extend ImageFrameStreamSource
sources/factory.ts
createWebRTCSource()
WebRTCStreamSource (sources/webrtc.ts)
negotiates SDP / ICE over the socket;
media rides an RTCPeerConnection
SignalingChannel (signaling.ts) -- one WebSocket to go2rtc
JSON text frames to message subscribers
binary frames to the single binary consumer
go2rtc server
adapters/media-source.ts
createBrowserMediaSource
(MSE / ManagedMediaSource)
utils/ (pure helpers)
codecs · live-edge-tracker · bounded-buffer-queue
base64 · user-agent · messages (binary sources)
webrtc-sdp · messages (WebRTC)
source-priority (the race)
<video> surface
(the live video element)
<img> surface
(ImageSurfaceController)
getPreferredSource (utils/source-priority.ts)
video > audio(AAC) > H.265, else WebRTC
OffscreenVideo (offscreen-video.ts)
off-screen <video> decode / probe
adapters/peer-connection.ts
createPeerConnection
(RTCPeerConnection)
connect(url, surfaces, modes)media-loadedinfo / error (escalate)start (sequential)start (parallel)fallbackfallbacknegotiate (JSON) + media (binary)signaling (JSON) onlyWebSocketmedia (RTP / SRTP)rendersrendersMP4 / MJPEGframesbinary profiledecodes intowebrtc profilewinner: MSE orWebRTC \ No newline at end of file diff --git a/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts b/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts new file mode 100644 index 00000000..00a4f0b4 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/image-surface-controller.ts @@ -0,0 +1,144 @@ +import type { LitElement } from 'lit'; + +import { Generation } from '../../../../utils/concurrency/generation'; +import { LatestValueRunner } from '../../../../utils/concurrency/latest-value-runner'; +import { ImageMediaPlayerController } from '../../../media-player/image'; +import { OffscreenImage } from './offscreen-image'; +import type { ImageSurface } from './session-controller'; + +// Liveness: while frames are expected, a gap beyond the window is a stall. +// Omitted -> the surface reports no liveness. `stallWindowSeconds` defaults to +// the standard frame-stall window. +interface ImageSurfaceLivenessOptions { + isFrameExpected: () => boolean; + stallWindowSeconds?: number; +} + +interface ImageSurfaceOptions { + livenessOptions?: ImageSurfaceLivenessOptions; + + // Factory for the detached image loader used to decode a frame. + createImage?: () => HTMLImageElement; +} + +// Presents a go2rtc MJPEG/MP4 stream as a sequence of frames on an : each +// frame is decoded off-DOM, then shown via an object URL (the previous one +// revoked), and an ImageMediaPlayerController exposes it to the card as a media +// player. The renderer owns the element; this owns what is shown on it. +export class ImageSurfaceController implements ImageSurface { + private _getImageCallback: () => HTMLImageElement | null; + private _mediaPlayerController: ImageMediaPlayerController; + + // An off-screen used to decode each frame before it is shown on the + // visible element (reused: only one frame decodes at a time). + private _decoder: OffscreenImage; + + // Current frame's object URL. + private _currentObjectURL: string | null = null; + + // Invalidated by reset() so a frame whose decode was still in flight when the + // surface was switched or torn down does not paint a stale frame afterwards. + private _generation = new Generation(); + + // Frames can arrive faster than the browser decodes one. Present at most one + // at a time and keep only the newest frame while busy, so the display + // converges on the latest without unbounded decode work. + private _frameRunner = new LatestValueRunner((frame) => + this._presentFrame(frame), + ); + + constructor( + host: LitElement, + getImageCallback: () => HTMLImageElement | null, + options?: ImageSurfaceOptions, + ) { + this._getImageCallback = getImageCallback; + this._decoder = new OffscreenImage(options?.createImage); + this._mediaPlayerController = new ImageMediaPlayerController( + host, + getImageCallback, + { + livenessOptions: options?.livenessOptions, + }, + ); + } + + public getElement(): HTMLImageElement | null { + return this._getImageCallback(); + } + + public getMediaPlayer(): ImageMediaPlayerController { + return this._mediaPlayerController; + } + + // Show a new frame. A newer frame supersedes any still waiting to show; the + // returned promise resolves once *a* frame has been presented. + public showFrame(blob: Blob): Promise { + return this._frameRunner.submit(blob); + } + + private async _presentFrame(blob: Blob): Promise { + const image = this._getImageCallback(); + + // Drop frames for a detached element (e.g. a surface superseded + // mid-teardown) so a retired surface never paints. + if (!image || !image.isConnected) { + return; + } + + const generation = this._generation.current(); + const url = URL.createObjectURL(blob); + + // Decode the frame on the off-screen loader before showing it. Assigning an + // undecoded object URL to the visible makes WebKit repaint the + // element empty until the decode completes, flashing the media background + // between frames; decoding first lets the swap paint from cache (as it's + // the same url) with no empty state. + const decoder = this._decoder.get(); + decoder.src = url; + try { + await decoder.decode(); + } catch { + // An undecodable frame is not fatal: drop it and keep the current one. + URL.revokeObjectURL(url); + return; + } + + // The surface may have been switched (reset) or detached during the decode; + // painting now would show a stale frame on a retired surface. + if (!this._generation.isCurrent(generation) || !image.isConnected) { + URL.revokeObjectURL(url); + return; + } + + image.src = url; + + const previousURL = this._currentObjectURL; + this._currentObjectURL = url; + if (previousURL) { + URL.revokeObjectURL(previousURL); + } + } + + // Drop the current frame and its object URL (e.g. on a surface switch or + // disconnect). + public reset(): void { + // Invalidate any frame still decoding so it cannot paint onto a surface + // that has since been switched or disconnected. + this._generation.invalidate(); + + // Drop any frame still waiting to show, for the same reason. + this._frameRunner.clear(); + + if (this._currentObjectURL) { + URL.revokeObjectURL(this._currentObjectURL); + this._currentObjectURL = null; + } + this._decoder.clear(); + + const image = this._getImageCallback(); + if (image) { + image.removeAttribute('src'); + } + } +} diff --git a/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts b/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts new file mode 100644 index 00000000..d9c5f39e --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/offscreen-image.ts @@ -0,0 +1,26 @@ +type ImageElementFactory = () => HTMLImageElement; + +// A held, injectable off-screen for decoding a frame separate from the +// visible display element. Created lazily on first `get()` and reused until +// `clear()` detaches its source and forgets it. +export class OffscreenImage { + private _create: ImageElementFactory; + private _image: HTMLImageElement | null = null; + + constructor(create?: ImageElementFactory) { + this._create = create ?? (() => new Image()); + } + + // Return the held image, creating it on first use. + public get(): HTMLImageElement { + return (this._image ??= this._create()); + } + + // Detach any source and forget the image. Safe to call when none is held. + public clear(): void { + if (this._image) { + this._image.removeAttribute('src'); + this._image = null; + } + } +} diff --git a/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts b/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts new file mode 100644 index 00000000..3ad94d40 --- /dev/null +++ b/src/components-lib/live/providers/go2rtc-experimental/offscreen-video.ts @@ -0,0 +1,28 @@ +export type VideoElementFactory = () => HTMLVideoElement; + +// A held, injectable off-screen