feat: Automatically recover from frozen live streams (#2569)

- Closes: #2099

src/card-controller/hass/hass-manager.ts

@@ -1,6 +1,7 @@
 import { isHassReady } from '../../ha/is-hass-ready';
-import type { HASSListener, HASSUnlistenCallback } from '../../ha/source';
+import type { HASSListener } from '../../ha/source';
 import type { HomeAssistant } from '../../ha/types';
+import type { UnsubscribeCallback } from '../../types';
 import { log } from '../../utils/debug';
 import { InitializationAspect } from '../initialization-manager';
 import type { CardHASSAPI } from '../types';
@@ -39,7 +40,7 @@ export class HASSManager implements HASSManagerReadonlyInterface {
     return this._eventWatcher;
   }
 
-  public addListener(listener: HASSListener): HASSUnlistenCallback {
+  public addListener(listener: HASSListener): UnsubscribeCallback {
     this._hassListeners.add(listener);
     return () => {
       this._hassListeners.delete(listener);

src/card-controller/hass/state-watcher.ts

@@ -1,6 +1,7 @@
 import { getHassDifferences } from '../../ha/get-hass-differences';
-import type { HASSSource, HASSUnlistenCallback } from '../../ha/source';
+import type { HASSSource } from '../../ha/source';
 import type { HassStateDifference, HomeAssistant } from '../../ha/types';
+import type { UnsubscribeCallback } from '../../types';
 
 type StateWatcherCallback = (difference: HassStateDifference) => void;
 
@@ -12,7 +13,7 @@ export interface StateWatcherSubscriptionInterface {
 export class StateWatcher implements StateWatcherSubscriptionInterface {
   private _source: HASSSource;
   private _watcherCallbacks = new Map<StateWatcherCallback, string[]>();
-  private _unlisten: HASSUnlistenCallback | null = null;
+  private _unlisten: UnsubscribeCallback | null = null;
 
   constructor(source: HASSSource) {
     this._source = source;

src/card-controller/initialization-manager.ts

@@ -234,9 +234,10 @@ export class InitializationManager {
     this._api.getCardElementManager().update();
   }
 
-  private async _tryInitialize(fn: () => Promise<void>): Promise<boolean> {
+  private async _tryInitialize(fn: () => Promise<boolean>): Promise<boolean> {
+    let initialized = false;
     try {
-      await fn();
+      initialized = await fn();
     } catch (e: unknown) {
       if (e instanceof Error) {
         errorToConsole(e);
@@ -249,7 +250,7 @@ export class InitializationManager {
       return false;
     }
 
-    return true;
+    return initialized;
   }
 
   private _setInitializationIssue(error: unknown): void {

src/card-controller/issues/factory.ts

@@ -8,8 +8,8 @@ import { ConnectionIssue } from './issues/connection';
 import { EventSubscriptionIssue } from './issues/event-subscription';
 import { InitializationIssue } from './issues/initialization';
 import { LegacyResourceIssue } from './issues/legacy-resource';
-import { MediaLoadIssue } from './issues/media-load';
 import { MediaQueryIssue } from './issues/media-query';
+import { MediaUnavailableIssue } from './issues/media-unavailable';
 import { ViewIncompatibleIssue } from './issues/view-incompatible';
 
 export const createIssueManager = (
@@ -32,7 +32,7 @@ export const createIssueManager = (
   manager.addIssue(new InitializationIssue(api));
   manager.addIssue(new LegacyResourceIssue(changeCallback));
   manager.addIssue(new MediaQueryIssue(api));
-  manager.addIssue(new MediaLoadIssue(api, changeCallback));
+  manager.addIssue(new MediaUnavailableIssue(api, changeCallback));
 
   return manager;
 };

src/card-controller/issues/issue-manager.ts

@@ -14,10 +14,10 @@ import type {
   IssueTriggerContextKey,
 } from './types';
 
-// Exponential backoff schedule for 'auto' retry. The base is set above the
-// per-media retry threshold (~10s) so the issue-level backoff kicks in *after*
-// lower-level recovery has had a chance to work, not in parallel with it.
-export const RETRY_EXPONENTIAL_BASE_SECONDS = 30;
+// Exponential backoff schedule for 'auto' retry. The first attempt fires
+// quickly so recovery is prompt, then the delay doubles (up to the max) so a
+// persistently-failing issue is not retried indefinitely at a tight interval.
+export const RETRY_EXPONENTIAL_BASE_SECONDS = 5;
 export const RETRY_EXPONENTIAL_MAX_SECONDS = 600;
 
 // Wraps the passive IssueStateManager with reaction logic. A single

src/card-controller/issues/issues/event-subscription.ts

@@ -1,7 +1,7 @@
 import type { Notification } from '../../../config/schema/actions/types';
 import type { SubscriptionHealthInterface } from '../../../ha/connection/subscription-health-monitor';
-import type { UnlistenCallback } from '../../../health';
 import { localize } from '../../../localize/localize';
+import type { UnsubscribeCallback } from '../../../types';
 import { createRetryControl } from '../retry-control';
 import type { Issue, IssueDescription } from '../types';
 
@@ -27,7 +27,7 @@ export class EventSubscriptionIssue implements Issue {
   public readonly key = 'event_subscription' as const;
 
   private _health: SubscriptionHealthInterface<string>;
-  private _unsubscribe: UnlistenCallback;
+  private _unsubscribe: UnsubscribeCallback;
 
   constructor(health: SubscriptionHealthInterface<string>, changeCallback: () => void) {
     this._health = health;

src/card-controller/issues/issues/media-unavailable.ts

@@ -1,9 +1,13 @@
 import type { IssueTriggerContext } from 'issue';
 
 import type { ConditionState } from '../../../condition-trigger/conditions/types.js';
-import type { Notification } from '../../../config/schema/actions/types.js';
+import type {
+  Notification,
+  NotificationDetail,
+} from '../../../config/schema/actions/types.js';
 import { TROUBLESHOOTING_MEDIA_URL } from '../../../const.js';
 import { localize } from '../../../localize/localize.js';
+import type { UnsubscribeCallback } from '../../../types.js';
 import { Timer } from '../../../utils/timer.js';
 import { IMAGE_VIEW_TARGET_ID_SENTINEL } from '../../../view/target-id.js';
 import { isAnyMediaViewName } from '../../../view/view.js';
@@ -11,19 +15,52 @@ import type { CardIssueManagerAPI } from '../../types.js';
 import { createRetryControl } from '../retry-control.js';
 import type { Issue, IssueDescription } from '../types.js';
 
+// Why the media_unavailable issue fired, so the notification and the reconnecting
+// placeholder can explain the specific cause rather than a generic message.
+export type MediaUnavailableIssueReason =
+  | 'entity_unavailable'
+  | 'not_loading'
+  | 'playback_error'
+  | 'stalled';
+
 declare module 'issue' {
   interface IssueTriggerContext {
-    media_load: { targetID: string };
+    media_unavailable: { targetID: string; reason: MediaUnavailableIssueReason };
   }
 }
 
 const MEDIA_LOADING_TIMEOUT_SECONDS = 10;
 
-export class MediaLoadIssue implements Issue {
-  public readonly key = 'media_load' as const;
+// The per-cause presentation (localization key + icon), shared by the
+// notification metadata and the reconnecting placeholder so each cause is
+// described in exactly one place.
+export const MEDIA_UNAVAILABLE_REASONS: Record<
+  MediaUnavailableIssueReason,
+  { localizationKey: string; icon: string }
+> = {
+  entity_unavailable: {
+    localizationKey: 'issues.media_unavailable.reasons.entity_unavailable',
+    icon: 'mdi:cctv-off',
+  },
+  not_loading: {
+    localizationKey: 'issues.media_unavailable.reasons.not_loading',
+    icon: 'mdi:progress-helper',
+  },
+  playback_error: {
+    localizationKey: 'issues.media_unavailable.reasons.playback_error',
+    icon: 'mdi:alert-circle',
+  },
+  stalled: {
+    localizationKey: 'issues.media_unavailable.reasons.stalled',
+    icon: 'mdi:motion-pause',
+  },
+};
+
+export class MediaUnavailableIssue implements Issue {
+  public readonly key = 'media_unavailable' as const;
 
   private _issueActive = false;
-  private _erroredTargetIDs = new Set<string>();
+  private _erroredTargets = new Map<string, MediaUnavailableIssueReason>();
 
   // Timer fires when a target has been loading too long without success.
   private _timer = new Timer();
@@ -31,18 +68,31 @@ export class MediaLoadIssue implements Issue {
 
   private _api: CardIssueManagerAPI;
   private _onChange: (() => void) | null;
+  private _unsubscribeCallback: UnsubscribeCallback;
 
   constructor(api: CardIssueManagerAPI, onChange?: () => void) {
     this._api = api;
     this._onChange = onChange ?? null;
+
+    // Clear a target's error on a genuine media (re)load.
+    this._unsubscribeCallback = this._api
+      .getMediaLoadedInfoManager()
+      .subscribe((change) => {
+        // A reconnect replay (`cached`) did not actually reload the media, and
+        // unload / select changes are irrelevant here; only a genuine load
+        // clears the error.
+        if (change.type === 'load' && !change.cached) {
+          this._onMediaLoad(change.targetID);
+        }
+      });
   }
 
   // =========================================================================
   // Explicit trigger -- called when a component fires an issue:trigger event.
   // =========================================================================
 
-  public trigger(context: IssueTriggerContext['media_load']): void {
-    this._erroredTargetIDs.add(context.targetID);
+  public trigger(context: IssueTriggerContext['media_unavailable']): void {
+    this._erroredTargets.set(context.targetID, context.reason);
   }
 
   // =========================================================================
@@ -55,10 +105,29 @@ export class MediaLoadIssue implements Issue {
       return;
     }
 
+    // A known error for the current target activates immediately, even if its
+    // (frozen) media still reads as loaded (it might be loaded but then
+    // reported a playback error that stops playback but leaves the player
+    // attached). Errors are cleared out-of-band by `_onMediaLoad` on a genuine
+    // reload.
+    if (this._hasError(state)) {
+      this._activate();
+      return;
+    }
+
     if (state.mediaLoadedInfo) {
-      this._handleMediaLoaded(state);
-    } else {
-      this._handleMediaNotLoaded(state);
+      // Loaded with no known error: healthy.
+      this._deactivate();
+      return;
+    }
+
+    this._handlePendingLoad(state);
+  }
+
+  // A genuine media (re)load for a target clears its error.
+  private _onMediaLoad(targetID: string): void {
+    if (this._erroredTargets.delete(targetID)) {
+      this._onChange?.();
     }
   }
 
@@ -82,28 +151,26 @@ export class MediaLoadIssue implements Issue {
   }
 
   public getNotification(): Notification {
-    const targets = new Set(this._erroredTargetIDs);
-    if (this._timerTargetID) {
-      targets.add(this._timerTargetID);
+    const targets = new Map(this._erroredTargets);
+    // The pending-load timer's target is a slow initial load that has not yet
+    // errored.
+    if (this._timerTargetID && !targets.has(this._timerTargetID)) {
+      targets.set(this._timerTargetID, 'not_loading');
     }
 
     return {
       heading: {
-        text: localize('issues.media_load.heading'),
+        text: localize('issues.media_unavailable.heading'),
         icon: 'mdi:cctv-off',
         severity: 'high' as const,
       },
       body: {
-        text: localize('issues.media_load.text'),
+        text: localize('issues.media_unavailable.text'),
       },
       ...(targets.size && {
-        metadata: Array.from(targets).map((id) => ({
-          text:
-            id === IMAGE_VIEW_TARGET_ID_SENTINEL
-              ? localize('editor.image')
-              : this._api.getCameraManager().getCameraMetadata(id)?.title ?? id,
-          icon: id === IMAGE_VIEW_TARGET_ID_SENTINEL ? 'mdi:image' : 'mdi:cctv',
-        })),
+        metadata: Array.from(targets).map(([id, reason]) =>
+          this._getTargetDetail(id, reason),
+        ),
       }),
       link: {
         url: TROUBLESHOOTING_MEDIA_URL,
@@ -113,6 +180,22 @@ export class MediaLoadIssue implements Issue {
     };
   }
 
+  // A per-camera notification detail: the target's name and its specific cause
+  // (e.g. "Office: Stream stalled"), with that cause's icon.
+  private _getTargetDetail(
+    id: string,
+    reason: MediaUnavailableIssueReason,
+  ): NotificationDetail {
+    const isImage = id === IMAGE_VIEW_TARGET_ID_SENTINEL;
+    const name = isImage
+      ? localize('editor.image')
+      : this._api.getCameraManager().getCameraMetadata(id)?.title ?? id;
+    return {
+      text: `${name}: ${localize(MEDIA_UNAVAILABLE_REASONS[reason].localizationKey)}`,
+      icon: isImage ? 'mdi:image' : MEDIA_UNAVAILABLE_REASONS[reason].icon,
+    };
+  }
+
   // =========================================================================
   // Retry -- called by the manager to schedule a media reload.
   // =========================================================================
@@ -125,7 +208,7 @@ export class MediaLoadIssue implements Issue {
     // Build the set of targets to retry: all errored targets plus the
     // target the pending timer was tracking (so a user-initiated retry
     // works even before the timeout fires).
-    const retryTargets = new Set(this._erroredTargetIDs);
+    const retryTargets = new Set(this._erroredTargets.keys());
     if (this._timerTargetID) {
       retryTargets.add(this._timerTargetID);
     }
@@ -140,11 +223,11 @@ export class MediaLoadIssue implements Issue {
       mediaEpoch[id] = (mediaEpoch[id] ?? 0) + 1;
     }
 
-    // Intentionally keep _issueActive, _erroredTargetIDs, and the pending
+    // Intentionally keep _issueActive, _erroredTargets, and the pending
     // timer in place. The issue stays visible while the provider
-    // re-attempts loading underneath. If the retry succeeds,
-    // _handleMediaLoaded will clear everything when media:loaded fires. If
-    // it fails silently (e.g. bogus stream name), the error stays visible
+    // re-attempts loading underneath. If the retry succeeds, the fresh media
+    // load clears everything (_onMediaLoad drops the errored target). If it
+    // fails silently (e.g. bogus stream name), the error stays visible
     // immediately -- no new 10s grace period.
     this._api.getViewManager().setViewWithMergedContext({ mediaEpoch });
     return false;
@@ -156,14 +239,19 @@ export class MediaLoadIssue implements Issue {
 
   public reset(): void {
     this._deactivate();
-    this._erroredTargetIDs.clear();
+    this._erroredTargets.clear();
+  }
+
+  // Stop reacting to media loads at end of life.
+  public destroy(): void {
+    this._unsubscribeCallback();
   }
 
   // Stop the pending-load timer so offscreen time doesn't count toward the
-  // 10s threshold. Preserve _issueActive, _erroredTargetIDs, and
+  // 10s threshold. Preserve _issueActive, _erroredTargets, and
   // _timerTargetID: already-visible errors remain visible on reattach, and
   // retaining _timerTargetID lets the existing active/target-mismatch guard
-  // in _handleMediaNotLoaded avoid spuriously deactivating the preserved
+  // in _handlePendingLoad avoid spuriously deactivating the preserved
   // issue when the same target is still loading on resume. The timer is
   // re-armed with a fresh window by the next detectDynamic pass.
   public suspend(): void {
@@ -174,36 +262,20 @@ export class MediaLoadIssue implements Issue {
   // Private helpers.
   // =========================================================================
 
-  // Media loaded successfully: deactivate and clear the error for this target
-  // so it won't immediately re-trigger on the next evaluation.
-  private _handleMediaLoaded(state: ConditionState): void {
-    this._deactivate();
-    if (state.targetID) {
-      this._erroredTargetIDs.delete(state.targetID);
-    }
-  }
-
-  // Media not yet loaded: activate immediately if there is a known provider
-  // error for this target, otherwise start a timeout to detect slow loads.
-  private _handleMediaNotLoaded(state: ConditionState): void {
-    // No targetID means no provider is actively rendering media (e.g. the
-    // viewer shows "No media to display" instead of showing a player). Don't
-    // start the timeout as there's nothing to wait for.
+  // Media not yet loaded and no known error: start (or keep) a timeout to catch
+  // a slow or failed initial load. No targetID means no provider is rendering
+  // media (e.g. the viewer shows "No media to display"), so there's nothing to
+  // wait for.
+  private _handlePendingLoad(state: ConditionState): void {
     if (!state.targetID) {
       this._deactivate();
       return;
     }
 
-    if (this._hasError(state)) {
-      this._activate();
-      return;
-    }
-
     const targetID = state.targetID;
 
-    // When the target changes to one without a known error, clear the active
-    // state so the new target gets its own timeout window instead of
-    // inheriting the previous target's error.
+    // When the target changes, clear the active state so the new target gets
+    // its own timeout window instead of inheriting the previous target's.
     if (this._issueActive && this._timerTargetID !== targetID) {
       this._deactivate();
     }
@@ -213,9 +285,7 @@ export class MediaLoadIssue implements Issue {
       this._timerTargetID = targetID;
       this._timer.start(MEDIA_LOADING_TIMEOUT_SECONDS, () => {
         // Record the error on timeout so retry() knows which epoch to bump.
-        // targetID is guaranteed non-null here -- the null case bails at the
-        // top of _handleMediaNotLoaded.
-        this._erroredTargetIDs.add(targetID);
+        this._erroredTargets.set(targetID, 'not_loading');
         this._activate();
         this._onChange?.();
       });
@@ -223,7 +293,7 @@ export class MediaLoadIssue implements Issue {
   }
 
   private _hasError(state: ConditionState): boolean {
-    return !!state.targetID && this._erroredTargetIDs.has(state.targetID);
+    return !!state.targetID && this._erroredTargets.has(state.targetID);
   }
 
   private _activate(): void {

src/card-controller/issues/types.ts

@@ -13,8 +13,8 @@ export type IssueKey =
   | 'event_subscription'
   | 'initialization'
   | 'legacy_resource'
-  | 'media_load'
   | 'media_query'
+  | 'media_unavailable'
   | 'view_incompatible';
 
 export interface IssueDescription {

src/card-controller/media-info-manager.ts

@@ -2,6 +2,7 @@ import type {
   MediaLoadedInfo,
   MediaLoadedInfoEventDetail,
   MediaLoadedInfoOwner,
+  UnsubscribeCallback,
 } from '../types';
 import { onAbort } from '../utils/abort-signal';
 import { log } from '../utils/debug';
@@ -13,6 +14,21 @@ interface ActiveEntry {
   owner: MediaLoadedInfoOwner;
 }
 
+// A change delivered to subscribers: a target's active media loading or
+// unloading, or the selected target changing.
+export type MediaLoadedInfoChange =
+  // A target's media (re)loaded. `cached` marks a replay (a reconnect
+  // re-dispatch, not an actual reload) rather than a genuine load.
+  | { type: 'load'; targetID: string; info: MediaLoadedInfo; cached: boolean }
+
+  // A target's media was retired.
+  | { type: 'unload'; targetID: string }
+
+  // The selected (current) target changed.
+  | { type: 'select'; targetID: string | null };
+
+type MediaLoadedInfoChangeCallback = (change: MediaLoadedInfoChange) => void;
+
 export class MediaLoadedInfoManager {
   private _api: CardMediaLoadedAPI;
 
@@ -31,18 +47,32 @@ export class MediaLoadedInfoManager {
   // and card-level side effects. Driven by ViewManager on every view change.
   private _selected: string | null = null;
 
+  // Subscribers are notified of every change to the tracked media: a target's
+  // load / unload, and selection changes.
+  private _subscribers = new Set<MediaLoadedInfoChangeCallback>();
+
   constructor(api: CardMediaLoadedAPI) {
     this._api = api;
   }
 
   public initialize(): void {
+    const cleared = [...this._active.keys()];
+    const hadSelected = this._selected !== null;
     this._active.clear();
     this._lastKnown.clear();
     this._selected = null;
     this._api.getConditionStateManager().setState({ mediaLoadedInfo: null });
+    cleared.forEach((targetID) => this._notify({ type: 'unload', targetID }));
+    if (hadSelected) {
+      this._notify({ type: 'select', targetID: null });
+    }
   }
 
-  public set(mediaLoadedInfo: MediaLoadedInfo, owner: MediaLoadedInfoOwner): void {
+  public set(
+    mediaLoadedInfo: MediaLoadedInfo,
+    owner: MediaLoadedInfoOwner,
+    cached?: boolean,
+  ): void {
     if (!isValidMediaLoadedInfo(mediaLoadedInfo) || !mediaLoadedInfo.targetID) {
       return;
     }
@@ -60,6 +90,15 @@ export class MediaLoadedInfoManager {
     if (targetID === this._selected) {
       this._emitChange(mediaLoadedInfo);
     }
+
+    // Notify for every target, not just the selected one (e.g. a background
+    // grid camera).
+    this._notify({
+      type: 'load',
+      targetID,
+      info: mediaLoadedInfo,
+      cached: cached ?? false,
+    });
   }
 
   public setSelected(targetID: string | null): void {
@@ -69,6 +108,7 @@ export class MediaLoadedInfoManager {
 
     this._selected = targetID;
     this._emitChange(targetID ? this._active.get(targetID)?.info ?? null : null);
+    this._notify({ type: 'select', targetID });
   }
 
   public get(): MediaLoadedInfo | null {
@@ -92,16 +132,18 @@ export class MediaLoadedInfoManager {
     if (!(owner instanceof HTMLElement) || !targetID) {
       return;
     }
-    this.set(ev.detail.info, owner);
+    this.set(ev.detail.info, owner, ev.detail.cached);
     onAbort(ev.detail.signal, () => this._clearTarget(targetID, owner));
   }
 
   public clear(): void {
+    const cleared = [...this._active.keys()];
     const selectedHadInfo = !!this._selected && this._active.has(this._selected);
     this._active.clear();
     if (selectedHadInfo) {
       this._api.getConditionStateManager().setState({ mediaLoadedInfo: null });
     }
+    cleared.forEach((targetID) => this._notify({ type: 'unload', targetID }));
   }
 
   private _clearTarget(targetID: string, owner: MediaLoadedInfoOwner): void {
@@ -114,6 +156,8 @@ export class MediaLoadedInfoManager {
     if (targetID === this._selected) {
       this._api.getConditionStateManager().setState({ mediaLoadedInfo: null });
     }
+
+    this._notify({ type: 'unload', targetID });
   }
 
   private _emitChange(mediaLoadedInfo: MediaLoadedInfo | null): void {
@@ -121,4 +165,13 @@ export class MediaLoadedInfoManager {
     this._api.getStyleManager().setExpandedMode();
     this._api.getCardElementManager().update();
   }
+
+  public subscribe(listener: MediaLoadedInfoChangeCallback): UnsubscribeCallback {
+    this._subscribers.add(listener);
+    return () => this._subscribers.delete(listener);
+  }
+
+  private _notify(change: MediaLoadedInfoChange): void {
+    [...this._subscribers].forEach((listener) => listener(change));
+  }
 }

src/card-controller/types.ts

@@ -289,6 +289,7 @@ export interface CardIssueManagerAPI {
   getConfigManager(): ConfigManager;
   getInitializationManager(): InitializationManager;
   getInteractionManager(): InteractionManager;
+  getMediaLoadedInfoManager(): MediaLoadedInfoManager;
   getNotificationManager(): NotificationManager;
   getViewManager(): ViewManager;
 }

src/card-controller/view/view-manager.ts

@@ -354,16 +354,26 @@ export class ViewManager implements ViewManagerInterface {
     );
   }
 
-  public initialize = async (): Promise<void> => {
+  // Returns whether view initialization succeeded. A `false` return leaves the
+  // view "aspect" uninitialized so a later initialization attempt retries --
+  // the default-view set can legitimately produce no view (e.g. it declines to
+  // run while the cameras are concurrently re-initializing), and treating that
+  // as initialized would leave the card without a view permanently.
+  public initialize = async (): Promise<boolean> => {
     // If the query string contains a view related action, we don't set any view
     // here and allow that action to be triggered by the next call of to execute
     // query actions (called at least once per render cycle).
     // Related: https://github.com/dermotduffy/advanced-camera-card/issues/1200
-    if (!this._api.getQueryStringManager().hasViewRelatedActionsToRun()) {
-      // This is not awaited to allow the initialization to complete before the
-      // query is answered.
-      void this.setViewDefaultWithNewQuery({ failSafe: true });
+    if (this._api.getQueryStringManager().hasViewRelatedActionsToRun()) {
+      return true;
     }
+
+    // This is not awaited to allow the initialization to complete before the
+    // query is answered. The view itself (without query results) is set
+    // synchronously before the query executes, so whether a view exists on
+    // return accurately reports success.
+    void this.setViewDefaultWithNewQuery({ failSafe: true });
+    return this.hasView();
   };
 
   private _setView(view: Readonly<View> | null): void {