feat: Automatically recover from frozen live streams (#2569)
- Closes: #2099
This commit is contained in:
Dermot Duffy
@@ -0,0 +1,142 @@
|
||||
import { isEqual } from 'lodash-es';
|
||||
|
||||
import type { StateWatcherSubscriptionInterface } from '../../../../card-controller/hass/state-watcher';
|
||||
import type { HassStateDifference, HomeAssistant } from '../../../../ha/types';
|
||||
import { Timer } from '../../../../utils/timer';
|
||||
import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller';
|
||||
|
||||
// A camera entity must stay `unavailable` this long before the stream is
|
||||
// treated as lost. Shorter blips (e.g. during PTZ, see issue #2124) are
|
||||
// tolerated.
|
||||
export const LIVENESS_ENTITY_UNAVAILABLE_GRACE_SECONDS = 10;
|
||||
|
||||
interface EntityAvailabilityDetectorConfig {
|
||||
getHASS: () => HomeAssistant | null;
|
||||
getStateWatcher: () => StateWatcherSubscriptionInterface | null;
|
||||
getCameraEntity: () => string | null;
|
||||
|
||||
// `always_error_if_entity_unavailable` (issue #1650): report the loss with no
|
||||
// grace tolerance, so any unavailability surfaces immediately.
|
||||
isAlwaysError: () => boolean;
|
||||
|
||||
onChange: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects a silent freeze via the camera entity: a `camera_entity` that stays
|
||||
* `unavailable` past the grace window (e.g. a Frigate restart or camera
|
||||
* power-cycle) is reported as not live (asking for a reconnecting placeholder),
|
||||
* and live again when it returns. Observes the entity via StateWatcher (an event
|
||||
* subscription), not by polling.
|
||||
*/
|
||||
export class EntityAvailabilityDetector implements LivenessDetector {
|
||||
private _config: EntityAvailabilityDetectorConfig;
|
||||
private _timer = new Timer();
|
||||
|
||||
private _active = false;
|
||||
private _watchedEntity: string | null = null;
|
||||
private _verdict: LivenessVerdict = { state: 'unknown' };
|
||||
|
||||
constructor(config: EntityAvailabilityDetectorConfig) {
|
||||
this._config = config;
|
||||
}
|
||||
|
||||
public subscribe(): void {
|
||||
this._active = true;
|
||||
this._watch();
|
||||
}
|
||||
|
||||
public unsubscribe(): void {
|
||||
// Stop watching and pause the grace timer, but keep `_verdict` so a
|
||||
// reconnect resumes rather than restarts.
|
||||
this._active = false;
|
||||
this._config.getStateWatcher()?.unsubscribe(this._onEntityStateChange);
|
||||
this._watchedEntity = null;
|
||||
this._timer.stop();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
// Re-point the subscription at the (possibly different) camera entity and
|
||||
// start fresh.
|
||||
this._verdict = { state: 'unknown' };
|
||||
this._timer.stop();
|
||||
this._watch();
|
||||
}
|
||||
|
||||
public getVerdict(): LivenessVerdict {
|
||||
return this._verdict;
|
||||
}
|
||||
|
||||
// Point the subscription at the current camera entity and re-check its state.
|
||||
private _watch(): void {
|
||||
if (!this._active) {
|
||||
return;
|
||||
}
|
||||
const stateWatcher = this._config.getStateWatcher();
|
||||
const entityID = this._config.getCameraEntity();
|
||||
if (entityID !== this._watchedEntity) {
|
||||
stateWatcher?.unsubscribe(this._onEntityStateChange);
|
||||
this._watchedEntity = entityID;
|
||||
if (entityID) {
|
||||
stateWatcher?.subscribe(this._onEntityStateChange, [entityID]);
|
||||
}
|
||||
}
|
||||
this._check();
|
||||
}
|
||||
|
||||
private _onEntityStateChange = (difference: HassStateDifference): void =>
|
||||
// Trap: Evaluate the state carried by the event, not `getHASS()`: the
|
||||
// StateWatcher fires synchronously from the card-level hass update, before
|
||||
// the wrapper's `hass` prop (what `getHASS()` reads) has propagated via
|
||||
// Lit, so re-reading it would still see the pre-change state and miss the
|
||||
// transition.
|
||||
this._evaluate(difference.newState.state);
|
||||
|
||||
private _check(): void {
|
||||
const stateObj = this._watchedEntity
|
||||
? this._config.getHASS()?.states[this._watchedEntity]
|
||||
: undefined;
|
||||
this._evaluate(stateObj?.state);
|
||||
}
|
||||
|
||||
private _evaluate(state?: string): void {
|
||||
if (state === 'unavailable') {
|
||||
if (this._config.isAlwaysError()) {
|
||||
// No grace time, and authoritative (overrides direct frame evidence):
|
||||
// the user opted into treating any unavailability as an error.
|
||||
this._setVerdict({
|
||||
state: 'not_live',
|
||||
authority: 'hard',
|
||||
renderPlaceholder: true,
|
||||
reason: 'entity_unavailable',
|
||||
});
|
||||
} else if (this._verdict.state !== 'not_live' && !this._timer.isRunning()) {
|
||||
// Wait out the grace window before declaring the stream lost, so short
|
||||
// blips are tolerated. This is only an indirect signal of stream health,
|
||||
// so it is suppressed when the media itself is confirmed live (frames
|
||||
// arriving).
|
||||
this._timer.start(LIVENESS_ENTITY_UNAVAILABLE_GRACE_SECONDS, () =>
|
||||
this._setVerdict({
|
||||
state: 'not_live',
|
||||
authority: 'indirect',
|
||||
renderPlaceholder: true,
|
||||
reason: 'entity_unavailable',
|
||||
}),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Entity available is not positive proof the stream is live (it can be
|
||||
// available while the stream is frozen), so report `unknown`, not `live`.
|
||||
this._timer.stop();
|
||||
this._setVerdict({ state: 'unknown' });
|
||||
}
|
||||
}
|
||||
|
||||
private _setVerdict(verdict: LivenessVerdict): void {
|
||||
if (isEqual(this._verdict, verdict)) {
|
||||
return;
|
||||
}
|
||||
this._verdict = verdict;
|
||||
this._config.onChange();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
import { isEqual } from 'lodash-es';
|
||||
|
||||
import type {
|
||||
MediaLoadedInfoEventDetail,
|
||||
MediaPlayerController,
|
||||
UnsubscribeCallback,
|
||||
} from '../../../../types';
|
||||
import { onAbort } from '../../../../utils/abort-signal';
|
||||
import { VisibilityObserver } from '../../../visibility-observer';
|
||||
import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller';
|
||||
|
||||
const MEDIA_LOADED_EVENT = 'advanced-camera-card:media:loaded';
|
||||
|
||||
/**
|
||||
* Detects a silent freeze by observing the media player's own liveness signal
|
||||
* (`subscribeLiveness`): when the player reports it has stopped delivering
|
||||
* media, this reports the stream not live so the wrapper shows the reconnecting
|
||||
* placeholder. How a player detects that (e.g. a video watching frame progress)
|
||||
* is the player's concern.
|
||||
*
|
||||
* It only watches while a stall is actionable -- the stream is loaded AND this
|
||||
* provider is actually visible to the user (on-screen and the tab is focused).
|
||||
* An off-screen or backgrounded video legitimately stops presenting media (the
|
||||
* browser pauses `requestVideoFrameCallback`), which is not considered a real
|
||||
* freeze. Visibility comes from the shared `VisibilityObserver` (intersection +
|
||||
* tab visibility), which correctly tracks every visible provider in a grid, not
|
||||
* just the single selected camera.
|
||||
*
|
||||
* Recovery is not self-driven. Once a freeze is reported and the placeholder
|
||||
* unmounts the frozen stream, the media player is gone -- so the detector holds
|
||||
* its not-live verdict rather than flipping back to live, which would clear the
|
||||
* placeholder and remount immediately in an unthrottled loop. The throttled
|
||||
* media_unavailable issue (see issue-manager.ts) retry remounts a fresh provider (a
|
||||
* new detector) to re-check the stream instead.
|
||||
*/
|
||||
export class MediaPlayerLivenessDetector implements LivenessDetector {
|
||||
private _host: HTMLElement;
|
||||
private _onChange: () => void;
|
||||
|
||||
private _verdict: LivenessVerdict = { state: 'unknown' };
|
||||
|
||||
private _visibilityObserver: VisibilityObserver | null = null;
|
||||
private _visible = false;
|
||||
|
||||
private _mediaPlayer: MediaPlayerController | null = null;
|
||||
|
||||
private _watchedPlayer: MediaPlayerController | null = null;
|
||||
private _unsubscribeLiveness: UnsubscribeCallback | null = null;
|
||||
|
||||
constructor(host: HTMLElement, onChange: () => void) {
|
||||
this._host = host;
|
||||
this._onChange = onChange;
|
||||
}
|
||||
|
||||
public subscribe(): void {
|
||||
this._host.addEventListener(MEDIA_LOADED_EVENT, this._onMediaLoaded);
|
||||
this._visibilityObserver = new VisibilityObserver(this._onVisibleChange, {
|
||||
emitInitial: true,
|
||||
});
|
||||
this._visibilityObserver.setRoot(this._host);
|
||||
}
|
||||
|
||||
public unsubscribe(): void {
|
||||
// Retain the verdict so a reconnect resumes where it left off; use reset()
|
||||
// to discard it.
|
||||
this._host.removeEventListener(MEDIA_LOADED_EVENT, this._onMediaLoaded);
|
||||
this._visibilityObserver?.destroy();
|
||||
this._visibilityObserver = null;
|
||||
this._unwatch();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
// The underlying stream changed (e.g. a substream switch): discard the
|
||||
// verdict and re-evaluate against the new media.
|
||||
this._unwatch();
|
||||
this._setVerdict({ state: 'unknown' });
|
||||
}
|
||||
|
||||
public getVerdict(): LivenessVerdict {
|
||||
return this._verdict;
|
||||
}
|
||||
|
||||
private _onVisibleChange = (visible: boolean): void => {
|
||||
this._visible = visible;
|
||||
this._watch();
|
||||
};
|
||||
|
||||
private _onMediaLoaded = (ev: CustomEvent<MediaLoadedInfoEventDetail>): void => {
|
||||
const player = ev.detail.info.mediaPlayerController ?? null;
|
||||
this._mediaPlayer = player;
|
||||
|
||||
// Drop the player when the source retires this media (unmount), so a stale
|
||||
// player is never watched.
|
||||
onAbort(ev.detail.signal, () => {
|
||||
if (this._mediaPlayer === player) {
|
||||
this._mediaPlayer = null;
|
||||
this._watch();
|
||||
}
|
||||
});
|
||||
|
||||
this._watch();
|
||||
};
|
||||
|
||||
private _watch(): void {
|
||||
// Only a visible player that exposes the liveness capability carries an
|
||||
// actionable stall.
|
||||
const player = this._visible ? this._mediaPlayer : null;
|
||||
const target = player?.subscribeLiveness ? player : null;
|
||||
if (target === this._watchedPlayer) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._unwatch();
|
||||
this._watchedPlayer = target;
|
||||
|
||||
if (player?.subscribeLiveness) {
|
||||
// Start (or resume) watching; the verdict stays `unknown` until a real
|
||||
// frame or a stall is observed.
|
||||
this._unsubscribeLiveness = player.subscribeLiveness((isLive) =>
|
||||
this._onLiveness(isLive),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// The media is gone because our own not-live placeholder unmounted the
|
||||
// frozen stream: hold that verdict. Flipping back to live here would clear
|
||||
// the placeholder and remount into the same freeze; recovery is instead the
|
||||
// throttled media_unavailable retry, which replaces this whole provider.
|
||||
if (!this._mediaPlayer && this._verdict.state === 'not_live') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise there is no current evidence: the provider is off-screen (an
|
||||
// off-screen video legitimately stops presenting frames) or exposes no
|
||||
// liveness signal, or the media went away for an ordinary reason. Report
|
||||
// `unknown` rather than leaving a stale `live` that would suppress other
|
||||
// detectors (e.g. entity availability).
|
||||
this._setVerdict({ state: 'unknown' });
|
||||
}
|
||||
|
||||
private _onLiveness(isLive: boolean): void {
|
||||
this._setVerdict(
|
||||
isLive
|
||||
? { state: 'live', authority: 'direct' }
|
||||
: {
|
||||
state: 'not_live',
|
||||
authority: 'direct',
|
||||
renderPlaceholder: true,
|
||||
reason: 'stalled',
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private _unwatch(): void {
|
||||
this._unsubscribeLiveness?.();
|
||||
this._unsubscribeLiveness = null;
|
||||
this._watchedPlayer = null;
|
||||
}
|
||||
|
||||
private _setVerdict(verdict: LivenessVerdict): void {
|
||||
if (isEqual(this._verdict, verdict)) {
|
||||
return;
|
||||
}
|
||||
this._verdict = verdict;
|
||||
this._onChange();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller';
|
||||
|
||||
const LIVE_ERROR_EVENT = 'advanced-camera-card:live:error';
|
||||
|
||||
/**
|
||||
* Detects a provider-reported stream failure: a live provider (or one of its
|
||||
* inner players) dispatches `live:error` when it cannot play. The listener sits
|
||||
* on the wrapper host and catches errors bubbling up from any descendant
|
||||
* provider.
|
||||
*
|
||||
* The provider is expected to render its own error, so no reconnecting
|
||||
* placeholder is asked for; the wrapper leaves the provider mounted (and
|
||||
* suppresses the load image).
|
||||
*/
|
||||
export class ProviderErrorDetector implements LivenessDetector {
|
||||
private _host: HTMLElement;
|
||||
private _onChange: () => void;
|
||||
private _verdict: LivenessVerdict = { state: 'unknown' };
|
||||
|
||||
constructor(host: HTMLElement, onChange: () => void) {
|
||||
this._host = host;
|
||||
this._onChange = onChange;
|
||||
}
|
||||
|
||||
public subscribe(): void {
|
||||
this._host.addEventListener(LIVE_ERROR_EVENT, this._handler);
|
||||
}
|
||||
|
||||
public unsubscribe(): void {
|
||||
this._host.removeEventListener(LIVE_ERROR_EVENT, this._handler);
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this._verdict = { state: 'unknown' };
|
||||
}
|
||||
|
||||
public getVerdict(): LivenessVerdict {
|
||||
return this._verdict;
|
||||
}
|
||||
|
||||
private _handler = (ev: Event): void => {
|
||||
ev.stopPropagation();
|
||||
if (this._verdict.state !== 'not_live') {
|
||||
// Authoritative: an explicit provider error overrides even direct frame
|
||||
// evidence. No placeholder -- the provider renders its own error.
|
||||
this._verdict = { state: 'not_live', authority: 'hard', reason: 'playback_error' };
|
||||
this._onChange();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import type { ReactiveController, ReactiveControllerHost } from 'lit';
|
||||
|
||||
import type { Camera } from '../../../camera-manager/camera';
|
||||
import type { StateWatcherSubscriptionInterface } from '../../../card-controller/hass/state-watcher';
|
||||
import type { MediaUnavailableIssueReason } from '../../../card-controller/issues/issues/media-unavailable';
|
||||
import type { IssueTriggerEventData } from '../../../card-controller/issues/types';
|
||||
import type { CameraConfig } from '../../../config/schema/cameras';
|
||||
import type { HomeAssistant } from '../../../ha/types';
|
||||
import { fireAdvancedCameraCardEvent } from '../../../utils/fire-advanced-camera-card-event';
|
||||
import { EntityAvailabilityDetector } from './detectors/entity-availability';
|
||||
import { MediaPlayerLivenessDetector } from './detectors/media-player-liveness';
|
||||
import { ProviderErrorDetector } from './detectors/provider-error';
|
||||
|
||||
// How far a verdict's evidence is trusted, so direct observation of the media
|
||||
// outweighs an indirect signal:
|
||||
// - `direct`: observed from the media itself (e.g. frames arriving or
|
||||
// stalling).
|
||||
// - `indirect`: inferred from a correlated signal (e.g. the camera entity's
|
||||
// state).
|
||||
// - `hard`: an authoritative failure (e.g. a provider error, or the user's
|
||||
// always_error opt-in) that overrides even direct evidence of life.
|
||||
type LivenessAuthority = 'hard' | 'direct' | 'indirect';
|
||||
|
||||
export type LivenessVerdict =
|
||||
// No evidence: the detector is not observing, so it neither confirms nor
|
||||
// denies liveness. A silent detector must never masquerade as proof of life.
|
||||
| { state: 'unknown' }
|
||||
|
||||
// Media is confirmed to be flowing.
|
||||
| { state: 'live'; authority: LivenessAuthority }
|
||||
|
||||
// Media is confirmed not to be flowing.
|
||||
| {
|
||||
state: 'not_live';
|
||||
authority: LivenessAuthority;
|
||||
reason: MediaUnavailableIssueReason;
|
||||
|
||||
// Whether the wrapper should replace the provider with a reconnecting
|
||||
// placeholder (a silent freeze, e.g. an unavailable camera). Omitted when
|
||||
// the provider renders its own error and should stay mounted.
|
||||
renderPlaceholder?: boolean;
|
||||
};
|
||||
|
||||
// The reconnecting placeholder the wrapper renders in place of a frozen
|
||||
// provider, carrying the cause so it can show a cause-specific message.
|
||||
interface LivenessPlaceholder {
|
||||
reason: MediaUnavailableIssueReason;
|
||||
}
|
||||
|
||||
export interface LivenessDetector {
|
||||
// Start observing the signal.
|
||||
subscribe(): void;
|
||||
|
||||
// Stop observing (e.g. on disconnect). Accumulated state is retained so a
|
||||
// later reconnect resumes where it left off; use reset() to discard it.
|
||||
unsubscribe(): void;
|
||||
|
||||
// Discard accumulated state because the underlying stream changed (e.g. a
|
||||
// substream switch), so detection restarts from scratch.
|
||||
reset?(): void;
|
||||
|
||||
// Reports the stream's current liveness, calling `onChange` (passed at
|
||||
// construction) whenever that verdict changes.
|
||||
getVerdict(): LivenessVerdict;
|
||||
}
|
||||
|
||||
interface StreamLivenessControllerConfig {
|
||||
getTargetID: () => string | null;
|
||||
getHASS: () => HomeAssistant | null;
|
||||
getCamera: () => Camera | null;
|
||||
getStateWatcher: () => StateWatcherSubscriptionInterface | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coordinates liveness detection for a single live provider and surfaces a
|
||||
* `media_unavailable` "issue" when the underlying stream stops delivering media. The
|
||||
* issue framework owns the throttled reload that recovers the stream.
|
||||
*/
|
||||
export class StreamLivenessController implements ReactiveController {
|
||||
private _host: ReactiveControllerHost & HTMLElement;
|
||||
private _config: StreamLivenessControllerConfig;
|
||||
private _detectors: LivenessDetector[];
|
||||
|
||||
constructor(
|
||||
host: ReactiveControllerHost & HTMLElement,
|
||||
config: StreamLivenessControllerConfig,
|
||||
) {
|
||||
this._host = host;
|
||||
this._config = config;
|
||||
|
||||
const onChange = (): void => this._onDetectorChange();
|
||||
const getCameraConfig = (): CameraConfig | null =>
|
||||
config.getCamera()?.getConfig() ?? null;
|
||||
|
||||
this._detectors = [
|
||||
new ProviderErrorDetector(host, onChange),
|
||||
new EntityAvailabilityDetector({
|
||||
getHASS: config.getHASS,
|
||||
getStateWatcher: config.getStateWatcher,
|
||||
getCameraEntity: () => getCameraConfig()?.camera_entity ?? null,
|
||||
isAlwaysError: () =>
|
||||
getCameraConfig()?.always_error_if_entity_unavailable ?? false,
|
||||
onChange,
|
||||
}),
|
||||
new MediaPlayerLivenessDetector(host, onChange),
|
||||
];
|
||||
this._host.addController(this);
|
||||
}
|
||||
|
||||
public hostConnected(): void {
|
||||
this._detectors.forEach((detector) => detector.subscribe());
|
||||
}
|
||||
|
||||
public hostDisconnected(): void {
|
||||
this._detectors.forEach((detector) => detector.unsubscribe());
|
||||
}
|
||||
|
||||
public isLive(): boolean {
|
||||
return this._getVerdict().state !== 'not_live';
|
||||
}
|
||||
|
||||
// The reconnecting placeholder to render in place of the (frozen) provider,
|
||||
// carrying the cause so the wrapper can show a cause-specific message. Null
|
||||
// when the provider should stay mounted (live, or a provider error that
|
||||
// renders its own error).
|
||||
public getPlaceholder(): LivenessPlaceholder | null {
|
||||
const verdict = this._getVerdict();
|
||||
return verdict.state === 'not_live' && verdict.renderPlaceholder
|
||||
? { reason: verdict.reason }
|
||||
: null;
|
||||
}
|
||||
|
||||
// Discard detector state on a stream change (e.g. a stream switch).
|
||||
public reset(): void {
|
||||
this._detectors.forEach((detector) => detector.reset?.());
|
||||
}
|
||||
|
||||
// Reduce the detectors to a single verdict. Direct evidence from the media
|
||||
// itself (e.g. lack of frame stalls) outranks indirect signals (e.g.
|
||||
// entity-availability), so a stream that is demonstrably delivering frames is
|
||||
// never torn down just because its camera entity blipped unavailable. `hard`
|
||||
// failures (a provider error, or the always_error opt-in) outrank everything.
|
||||
private _getVerdict(): LivenessVerdict {
|
||||
const verdicts = this._detectors.map((detector) => detector.getVerdict());
|
||||
const find = (
|
||||
state: 'live' | 'not_live',
|
||||
authority: LivenessAuthority,
|
||||
): LivenessVerdict | null =>
|
||||
verdicts.find(
|
||||
(v) => 'authority' in v && v.state === state && v.authority === authority,
|
||||
) ?? null;
|
||||
|
||||
// `unknown` verdicts carry no authority, so they match none of these lookups
|
||||
// and are skipped; if every detector is silent the reduction is `unknown`.
|
||||
return (
|
||||
find('not_live', 'hard') ??
|
||||
find('not_live', 'direct') ??
|
||||
find('live', 'direct') ??
|
||||
find('not_live', 'indirect') ?? { state: 'unknown' }
|
||||
);
|
||||
}
|
||||
|
||||
private _onDetectorChange(): void {
|
||||
const verdict = this._getVerdict();
|
||||
if (verdict.state === 'not_live') {
|
||||
this._triggerMediaUnavailableIssue(verdict.reason);
|
||||
}
|
||||
this._host.requestUpdate();
|
||||
}
|
||||
|
||||
// Tell the issue framework this target's media is not loaded, surfacing the
|
||||
// media_unavailable issue (status bar + retry) and its throttled reload.
|
||||
private _triggerMediaUnavailableIssue(reason: MediaUnavailableIssueReason): void {
|
||||
const targetID = this._config.getTargetID();
|
||||
if (!targetID) {
|
||||
return;
|
||||
}
|
||||
fireAdvancedCameraCardEvent<IssueTriggerEventData>(this._host, 'issue:trigger', {
|
||||
key: 'media_unavailable',
|
||||
targetID,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user