feat: Add experimental rewrite of go2rtc live provider (MSE/WebRTC/MP4/MJPEG) (#2580)

- Closes #2556
- Closes #2450

**Key intended features:**
 - go2rtc compatible
- 100% test coverage to significantly improve ability to test, maintain
and work around browser weirdnesses (e.g. Safari).
 - Written from the ground up in the style of the rest of the project.

**To use:**
 - Change `live_provider` from `go2rtc` to `go2rtc-experimental`.

src/components/live/providers/go2rtc-experimental/index.ts

@@ -0,0 +1,299 @@
+import {
+  html,
+  LitElement,
+  unsafeCSS,
+  type CSSResultGroup,
+  type PropertyValues,
+  type TemplateResult,
+} from 'lit';
+import { customElement, property, state } from 'lit/decorators.js';
+import { createRef, ref, type Ref } from 'lit/directives/ref.js';
+
+import type { Camera } from '../../../../camera-manager/camera.js';
+import {
+  MEDIA_UNAVAILABLE_REASONS,
+  type MediaUnavailableIssueReason,
+} from '../../../../card-controller/issues/issues/media-unavailable.js';
+import { ImageSurfaceController } from '../../../../components-lib/live/providers/go2rtc-experimental/image-surface-controller.js';
+import {
+  Go2RTCSessionController,
+  type SessionSurfaces,
+  type VideoSurface,
+} from '../../../../components-lib/live/providers/go2rtc-experimental/session-controller.js';
+import type { SurfaceKind } from '../../../../components-lib/live/providers/go2rtc-experimental/types.js';
+import { mapFailureReasonToIssueReason } from '../../../../components-lib/live/providers/go2rtc-experimental/utils/failure-reason.js';
+import { dispatchLiveErrorEvent } from '../../../../components-lib/live/utils/dispatch-live-error.js';
+import { MediaLoadedInfoSourceController } from '../../../../components-lib/media-loaded-info-source-controller.js';
+import { VideoMediaPlayerController } from '../../../../components-lib/media-player/video.js';
+import { SignedURLController } from '../../../../components-lib/signed-url-controller.js';
+import type { MicrophoneConfig } from '../../../../config/schema/live.js';
+import type { CardWideConfig } from '../../../../config/schema/types.js';
+import type { HomeAssistant } from '../../../../ha/types.js';
+import { localize } from '../../../../localize/localize.js';
+import liveGo2RTCExperimentalStyle from '../../../../scss/live-go2rtc-experimental.scss';
+import type { MediaPlayer, MediaPlayerController } from '../../../../types.js';
+import {
+  dispatchMediaPauseEvent,
+  dispatchMediaPlayEvent,
+  dispatchMediaVolumeChangeEvent,
+} from '../../../../utils/media-info.js';
+import { renderMediaNotification } from '../../../notification/media.js';
+
+@customElement('advanced-camera-card-live-go2rtc-experimental')
+export class AdvancedCameraCardGo2RTCExperimental
+  extends LitElement
+  implements MediaPlayer
+{
+  // Not a reactive property to avoid resetting the video.
+  public hass?: HomeAssistant;
+
+  @property({ attribute: false })
+  public camera?: Camera;
+
+  // The BASE camera ID (camera property may be a substream)
+  @property({ attribute: false })
+  public targetID?: string;
+
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream | null;
+
+  @property({ attribute: false })
+  public microphoneConfig?: MicrophoneConfig;
+
+  @property({ attribute: false })
+  public cardWideConfig?: CardWideConfig;
+
+  // The camera's title, shown in error messages to identify the camera.
+  @property({ attribute: false })
+  public cameraTitle?: string;
+
+  @property({ attribute: true, type: Boolean })
+  public controls = false;
+
+  private _hasLiveError = false;
+
+  // ===========================================================================
+  // Surface: Video
+  // ===========================================================================
+  private _refVideo: Ref<HTMLVideoElement> = createRef();
+
+  private _videoMediaPlayerController = new VideoMediaPlayerController(
+    this,
+    () => this._refVideo.value ?? null,
+    () => this.controls,
+  );
+
+  private _videoSurface: VideoSurface = {
+    getElement: () => this._refVideo.value ?? null,
+    getMediaPlayer: () => this._videoMediaPlayerController,
+  };
+
+  // ===========================================================================
+  // Surface: Image
+  // ===========================================================================
+
+  private _refImage: Ref<HTMLImageElement> = createRef();
+
+  // A controller rather than a plain object (unlike the video surface): the
+  // image surface owns state, the object-URL lifecycle -- each frame's
+  // createObjectURL and revoking the previous one.
+  private _imageSurface = new ImageSurfaceController(
+    this,
+    () => this._refImage.value ?? null,
+    {
+      livenessOptions: {
+        isFrameExpected: () => true,
+      },
+    },
+  );
+
+  // ===========================================================================
+  // Surface Management
+  // ===========================================================================
+
+  // The surface currently showing committed media, or null before anything has
+  // committed (both surfaces hidden). Driven by the session's
+  // surfaceCommittedCallback.
+  @state()
+  private _activeSurface: SurfaceKind | null = null;
+
+  @state()
+  private _streamError: MediaUnavailableIssueReason | null = null;
+
+  // Built once and kept stable: the session compares this object by identity,
+  // so handing it a new one will trigger a reconnect.
+  private _surfaces: SessionSurfaces = {
+    video: this._videoSurface,
+    image: this._imageSurface,
+  };
+
+  private _signedURLController = new SignedURLController(this, () => {
+    const endpoint = this.camera?.getEndpoints()?.go2rtc;
+    if (!this.hass || !endpoint) {
+      return {};
+    }
+    return {
+      hass: this.hass,
+      endpoint,
+      proxyConfig: this.camera?.getLiveProxyConfig(),
+      proxyEndpointOptions: { websocket: true },
+    };
+  });
+
+  private _mediaLoadedInfoSourceController = new MediaLoadedInfoSourceController(this, {
+    getTargetID: () => this.targetID ?? null,
+  });
+
+  private _session = new Go2RTCSessionController({
+    getControls: () => this.controls,
+    getCardWideConfig: () => this.cardWideConfig ?? null,
+    mediaLoadedCallback: (info) => this._mediaLoadedInfoSourceController.set(info),
+
+    surfaceCommittedCallback: (surface) => {
+      this._activeSurface = surface;
+
+      // A commit means the stream recovered: drop any prior error.
+      this._streamError = null;
+    },
+
+    // The session could not recover the stream on its own; surface it (with the
+    // failure's user-facing cause) so the card's media-load retry (reconnecting
+    // indicator, backoff, give-up) runs and can name why. The provider renders
+    // the error itself (below); the event drives the liveness verdict + retry.
+    errorCallback: (reason) => {
+      this._streamError = mapFailureReasonToIssueReason(reason);
+      dispatchLiveErrorEvent(this, this._streamError);
+    },
+  });
+
+  public async getMediaPlayerController(): Promise<MediaPlayerController | null> {
+    return this._activeSurface === 'image'
+      ? this._imageSurface.getMediaPlayer()
+      : this._videoMediaPlayerController;
+  }
+
+  connectedCallback(): void {
+    super.connectedCallback();
+
+    // Re-render (and thus re-establish the session) when reconnected to the
+    // DOM. https://github.com/dermotduffy/advanced-camera-card/issues/996
+    this.requestUpdate();
+  }
+
+  disconnectedCallback(): void {
+    // Tear down synchronously so streams (e.g. 2-way audio backchannels)
+    // release immediately.
+    this._session.reset();
+    this._activeSurface = null;
+    super.disconnectedCallback();
+  }
+
+  protected willUpdate(changedProps: PropertyValues): void {
+    if (changedProps.has('camera')) {
+      // The session is re-established by `updated()` once the new camera's
+      // signed URL resolves; the next commit picks the live surface. Blank the
+      // view meanwhile so the previous camera's last frame is not shown.
+      this._session.reset();
+      this._activeSurface = null;
+      this._streamError = null;
+    }
+
+    // Only treat a missing go2rtc endpoint as an error after the camera's
+    // endpoints have been explicitly set (not undefined / still loading).
+    const endpoints = this.camera?.getEndpoints();
+    const hasLiveError =
+      !!this._signedURLController.getError() || (!!endpoints && !endpoints.go2rtc);
+
+    if (hasLiveError && !this._hasLiveError) {
+      dispatchLiveErrorEvent(this);
+    }
+    this._hasLiveError = hasLiveError;
+
+    if (changedProps.has('controls')) {
+      // Only the video surface has native controls; the image surface has none.
+      this._videoMediaPlayerController.setControls(this.controls).catch(() => {});
+    }
+
+    if (changedProps.has('microphoneStream')) {
+      // The WebRTC lane swaps the outbound track in place; no visible reload.
+      this._session.setMicrophoneStream(this.microphoneStream ?? null);
+    }
+  }
+
+  protected updated(): void {
+    const url = this._signedURLController.getValue();
+    if (url) {
+      this._session.connect(
+        url,
+        this._surfaces,
+        this.camera?.getConfig()?.go2rtc?.modes,
+      );
+    } else {
+      // No usable URL: a signing/proxy error, or no go2rtc endpoint (which
+      // includes endpoints still loading). The render omits the surfaces, so
+      // drop the session -- otherwise a later URL, even an identical unsigned
+      // endpoint, would be skipped by connect()'s identity check and leave the
+      // session bound to the removed elements.
+      this._session.reset();
+      this._activeSurface = null;
+    }
+  }
+
+  protected render(): TemplateResult | void {
+    const error = this._signedURLController.getError();
+    if (error) {
+      return renderMediaNotification({
+        title: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
+        targetTitle: this.cameraTitle,
+      });
+    }
+    if (!this.camera?.getEndpoints()?.go2rtc) {
+      return renderMediaNotification({
+        title: localize('error.configuration_error'),
+        detail: localize('error.live_camera_no_endpoint'),
+        targetTitle: this.cameraTitle,
+      });
+    }
+    if (this._streamError) {
+      // A stream-level failure the session gave up on: the provider must render
+      // its own error (marked in-progress, since the card keeps retrying).
+      return renderMediaNotification({
+        icon: MEDIA_UNAVAILABLE_REASONS[this._streamError].icon,
+        title: localize(MEDIA_UNAVAILABLE_REASONS[this._streamError].localizationKey),
+        targetTitle: this.cameraTitle,
+      });
+    }
+
+    // Both image and video surfaces are always rendered; only the committed one
+    // is shown (the other, and both before anything commits, are hidden). MSE
+    // and WebRTC play on the <video>; MP4 and MJPEG show frames on the <img>.
+    //
+    // Muted is bound as a property: Chrome ignores the `muted` content
+    // attribute on videos instantiated from cloned templates (as Lit does), so
+    // an attribute would not actually start the video muted. Media may be
+    // unmuted later in accordance with user configuration.
+    return html`
+      <video
+        ${ref(this._refVideo)}
+        .muted=${true}
+        ?hidden=${this._activeSurface !== 'video'}
+        playsinline
+        preload="auto"
+        @play=${() => dispatchMediaPlayEvent(this)}
+        @pause=${() => dispatchMediaPauseEvent(this)}
+        @volumechange=${() => dispatchMediaVolumeChangeEvent(this)}
+      ></video>
+      <img ${ref(this._refImage)} ?hidden=${this._activeSurface !== 'image'} alt="" />
+    `;
+  }
+
+  static get styles(): CSSResultGroup {
+    return unsafeCSS(liveGo2RTCExperimentalStyle);
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    'advanced-camera-card-live-go2rtc-experimental': AdvancedCameraCardGo2RTCExperimental;
+  }
+}

src/components/live/providers/go2rtc/index.ts

@@ -17,7 +17,7 @@ import type { HomeAssistant } from '../../../../ha/types.js';
 import { localize } from '../../../../localize/localize.js';
 import liveGo2RTCStyle from '../../../../scss/live-go2rtc.scss';
 import type { MediaPlayer, MediaPlayerController } from '../../../../types.js';
-import { renderNotificationBlockFromText } from '../../../notification/block.js';
+import { renderMediaNotification } from '../../../notification/media.js';
 import { VideoRTC } from './video-rtc.js';
 
 customElements.define('advanced-camera-card-live-go2rtc-player', VideoRTC);
@@ -40,6 +40,10 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
   @property({ attribute: false })
   public microphoneConfig?: MicrophoneConfig;
 
+  // The camera's title, shown in error messages to identify the camera.
+  @property({ attribute: false })
+  public cameraTitle?: string;
+
   @property({ attribute: true, type: Boolean })
   public controls = false;
 
@@ -148,14 +152,16 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
   protected render(): TemplateResult | void {
     const error = this._signedURLController.getError();
     if (error) {
-      return renderNotificationBlockFromText(
-        localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
-        { context: this.camera?.getConfig() },
-      );
+      return renderMediaNotification({
+        title: localize(error === 'proxy' ? 'error.failed_proxy' : 'error.failed_sign'),
+        targetTitle: this.cameraTitle,
+      });
     }
     if (!this.camera?.getEndpoints()?.go2rtc) {
-      return renderNotificationBlockFromText(localize('error.live_camera_no_endpoint'), {
-        context: this.camera?.getConfig(),
+      return renderMediaNotification({
+        title: localize('error.configuration_error'),
+        detail: localize('error.live_camera_no_endpoint'),
+        targetTitle: this.cameraTitle,
       });
     }
     return html`${this._player}`;

src/components/live/providers/image.ts

@@ -9,6 +9,8 @@ import { customElement, property } from 'lit/decorators.js';
 import { createRef, ref, type Ref } from 'lit/directives/ref.js';
 
 import type { Camera } from '../../../camera-manager/camera.js';
+import type { MediaUnavailableIssueReason } from '../../../card-controller/issues/issues/media-unavailable.js';
+import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispatch-live-error.js';
 import type { HomeAssistant } from '../../../ha/types';
 import basicBlockStyle from '../../../scss/basic-block.scss';
 import type {
@@ -31,6 +33,10 @@ export class AdvancedCameraCardLiveImage extends LitElement implements MediaPlay
   @property({ attribute: false })
   public targetID?: string;
 
+  // The camera's title, shown in error messages to identify the camera.
+  @property({ attribute: false })
+  public cameraTitle?: string;
+
   private _refImage: Ref<MediaPlayerElement> = createRef();
 
   public async getMediaPlayerController(): Promise<MediaPlayerController | null> {
@@ -51,7 +57,11 @@ export class AdvancedCameraCardLiveImage extends LitElement implements MediaPlay
         .imageConfig=${cameraConfig.image}
         .cameraConfig=${cameraConfig}
         .targetID=${this.targetID}
+        .cameraTitle=${this.cameraTitle}
         .proxyConfig=${this.camera?.getLiveProxyConfig()}
+        @advanced-camera-card:image-updating-player:error=${(
+          ev: CustomEvent<MediaUnavailableIssueReason>,
+        ) => dispatchLiveErrorEvent(this, ev.detail)}
       >
       </advanced-camera-card-image-updating-player>
     `;

src/components/live/providers/jsmpeg.ts

@@ -14,7 +14,7 @@ import type { Camera } from '../../../camera-manager/camera.js';
 import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispatch-live-error.js';
 import { MediaLoadedInfoSourceController } from '../../../components-lib/media-loaded-info-source-controller.js';
 import { JSMPEGMediaPlayerController } from '../../../components-lib/media-player/jsmpeg.js';
-import { createNotificationFromText } from '../../../components-lib/notification/factory.js';
+import { createMediaNotification } from '../../../components-lib/notification/media.js';
 import type { Notification } from '../../../config/schema/actions/types.js';
 import type { CardWideConfig } from '../../../config/schema/types.js';
 import { homeAssistantGetSignedURLIfNecessary } from '../../../ha/sign-path.js';
@@ -22,13 +22,14 @@ import type { HomeAssistant } from '../../../ha/types.js';
 import { localize } from '../../../localize/localize.js';
 import liveJSMPEGStyle from '../../../scss/live-jsmpeg.scss';
 import type { MediaPlayer, MediaPlayerController } from '../../../types.js';
-import { convertHTTPAdressToWebsocket, errorToConsole } from '../../../utils/basic.js';
+import { errorToConsole } from '../../../utils/basic.js';
 import {
   createMediaLoadedInfo,
   dispatchMediaPauseEvent,
   dispatchMediaPlayEvent,
 } from '../../../utils/media-info.js';
 import { Timer } from '../../../utils/timer.js';
+import { convertToWebSocketURL } from '../../../utils/websocket-url.js';
 
 import '../../notification/block.js';
 
@@ -58,6 +59,10 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
   @property({ attribute: false })
   public cardWideConfig?: CardWideConfig;
 
+  // The camera's title, shown in error messages to identify the camera.
+  @property({ attribute: false })
+  public cameraTitle?: string;
+
   @state()
   private _notification: Notification | null = null;
 
@@ -193,10 +198,11 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
 
     const endpoint = this.camera?.getEndpoints()?.jsmpeg;
     if (!endpoint) {
-      this._notification = createNotificationFromText(
-        localize('error.live_camera_no_endpoint'),
-        { context: this.camera?.getConfig() },
-      );
+      this._notification = createMediaNotification({
+        title: localize('error.configuration_error'),
+        detail: localize('error.live_camera_no_endpoint'),
+        targetTitle: this.cameraTitle,
+      });
       dispatchLiveErrorEvent(this);
       return;
     }
@@ -211,11 +217,12 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
     } catch (e) {
       errorToConsole(e);
     }
-    const address = response ? convertHTTPAdressToWebsocket(response) : null;
+    const address = response ? convertToWebSocketURL(response) : null;
 
     if (!address) {
-      this._notification = createNotificationFromText(localize('error.failed_sign'), {
-        context: this.camera?.getConfig(),
+      this._notification = createMediaNotification({
+        title: localize('error.failed_sign'),
+        targetTitle: this.cameraTitle,
       });
       dispatchLiveErrorEvent(this);
       return;
@@ -238,10 +245,10 @@ export class AdvancedCameraCardLiveJSMPEG extends LitElement implements MediaPla
 
       if (!this._jsmpegVideoPlayer || !this._jsmpegCanvasElement) {
         if (!this._notification) {
-          this._notification = createNotificationFromText(
-            localize('error.jsmpeg_no_player'),
-            { context: this.camera?.getConfig() },
-          );
+          this._notification = createMediaNotification({
+            title: localize('error.jsmpeg_no_player'),
+            targetTitle: this.cameraTitle,
+          });
           dispatchLiveErrorEvent(this);
         }
         return;

src/components/live/providers/webrtc-card.ts

@@ -14,24 +14,19 @@ import { dispatchLiveErrorEvent } from '../../../components-lib/live/utils/dispa
 import { getTechnologyForVideoRTC } from '../../../components-lib/live/utils/get-technology-for-video-rtc.js';
 import { MediaLoadedInfoSourceController } from '../../../components-lib/media-loaded-info-source-controller.js';
 import { VideoMediaPlayerController } from '../../../components-lib/media-player/video.js';
-import { createNotificationFromText } from '../../../components-lib/notification/factory.js';
+import { createMediaNotification } from '../../../components-lib/notification/media.js';
 import type { Notification } from '../../../config/schema/actions/types.js';
 import type { CardWideConfig } from '../../../config/schema/types.js';
 import type { HomeAssistant } from '../../../ha/types.js';
 import { localize } from '../../../localize/localize.js';
 import liveWebRTCCardStyle from '../../../scss/live-webrtc-card.scss';
-import {
-  AdvancedCameraCardError,
-  type MediaPlayer,
-  type MediaPlayerController,
-} from '../../../types.js';
+import type { MediaPlayer, MediaPlayerController } from '../../../types.js';
 import { mayHaveAudio } from '../../../utils/audio.js';
 import {
   hideMediaControlsTemporarily,
   MEDIA_LOAD_CONTROLS_HIDE_SECONDS,
   setControlsOnVideo,
 } from '../../../utils/controls.js';
-import { getContextFromError } from '../../../utils/error-context.js';
 import {
   createMediaLoadedInfo,
   dispatchMediaPauseEvent,
@@ -63,6 +58,10 @@ export class AdvancedCameraCardLiveWebRTCCard extends LitElement implements Medi
   @property({ attribute: false })
   public cardWideConfig?: CardWideConfig;
 
+  // The camera's title, shown in error messages to identify the camera.
+  @property({ attribute: false })
+  public cameraTitle?: string;
+
   @property({ attribute: true, type: Boolean })
   public controls = false;
 
@@ -169,15 +168,11 @@ export class AdvancedCameraCardLiveWebRTCCard extends LitElement implements Medi
       try {
         webrtcElement = this._createWebRTC();
       } catch (e) {
-        const context = getContextFromError(e);
-        this._notification = createNotificationFromText(
-          e instanceof AdvancedCameraCardError
-            ? e.message
-            : localize('error.webrtc_card_reported_error') + ': ' + (e as Error).message,
-          {
-            ...(context && { context }),
-          },
-        );
+        this._notification = createMediaNotification({
+          title: localize('error.webrtc_card_reported_error'),
+          detail: e instanceof Error ? e.message : String(e),
+          targetTitle: this.cameraTitle,
+        });
         dispatchLiveErrorEvent(this);
         return;
       }