fix: Improve 2-way audio detection (#2293)

- For Frigate cameras, you must be running at least [integration
v5.12.0](https://github.com/blakeblackshear/frigate-hass-integration/releases/tag/v5.12.0).
 - Closes: #2191 

Includes a significant refactor of cameras, and the introduction of a
`2-way-audio` capability that is dynamically fetched from `go2rtc`. One
gotcha is if you previously had a substream with 2-way audio, you may
need to modify

```yaml
 - camera_entity: camera.foo
    capabilities:
      disable_except:
        - substream
```

... to ...

```yaml
 - camera_entity: camera.foo
    capabilities:
      disable_except:
        - substream
        - 2-way-audio
```

src/components/live/provider.ts

@@ -16,9 +16,8 @@ import { MicrophoneState } from '../../card-controller/types.js';
 import { LazyLoadController } from '../../components-lib/lazy-load-controller.js';
 import { dispatchLiveErrorEvent } from '../../components-lib/live/utils/dispatch-live-error.js';
 import { PartialZoomSettings } from '../../components-lib/zoom/types.js';
-import { LiveProvider } from '../../config/schema/cameras.js';
 import { LiveConfig } from '../../config/schema/live.js';
-import { CardWideConfig, configDefaults } from '../../config/schema/types.js';
+import { CardWideConfig } from '../../config/schema/types.js';
 import { STREAM_TROUBLESHOOTING_URL } from '../../const.js';
 import { HomeAssistant } from '../../ha/types.js';
 import { localize } from '../../localize/localize.js';
@@ -29,6 +28,7 @@ import {
   MediaPlayerController,
   MediaPlayerElement,
 } from '../../types.js';
+import { getResolvedLiveProvider } from '../../utils/live-provider.js';
 import { dispatchMediaUnloadedEvent } from '../../utils/media-info.js';
 import '../icon.js';
 import { renderMessage } from '../message.js';
@@ -101,25 +101,6 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
     return (await this._refProvider.value?.getMediaPlayerController()) ?? null;
   }
 
-  /**
-   * Get the fully resolved live provider.
-   * @returns A live provider (that is not 'auto').
-   */
-  protected _getResolvedProvider(): Omit<LiveProvider, 'auto'> {
-    const config = this.camera?.getConfig();
-    if (config?.live_provider === 'auto') {
-      if (config?.webrtc_card?.entity || config?.webrtc_card?.url) {
-        return 'webrtc-card';
-      } else if (config?.camera_entity) {
-        return 'ha';
-      } else if (config?.frigate.camera_name) {
-        return 'jsmpeg';
-      }
-      return configDefaults.cameras.live_provider;
-    }
-    return config?.live_provider || 'image';
-  }
-
   /**
    * Determine if a camera image should be shown in lieu of the real stream
    * whilst loading.
@@ -172,7 +153,7 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
     }
 
     if (changedProps.has('camera')) {
-      const provider = this._getResolvedProvider();
+      const provider = getResolvedLiveProvider(this.camera?.getConfig());
       if (provider === 'jsmpeg') {
         this._importPromises.push(import('./providers/jsmpeg.js'));
       } else if (provider === 'ha') {
@@ -248,7 +229,7 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
     this.title = this.label;
     this.ariaLabel = this.label;
 
-    const provider = this._getResolvedProvider();
+    const provider = getResolvedLiveProvider(this.camera?.getConfig());
 
     if (
       provider === 'ha' ||

src/components/live/providers/go2rtc/index.ts

@@ -15,11 +15,7 @@ import { VideoMediaPlayerController } from '../../../../components-lib/media-pla
 import { MicrophoneConfig } from '../../../../config/schema/live.js';
 import { homeAssistantSignPath } from '../../../../ha/sign-path.js';
 import { HomeAssistant } from '../../../../ha/types.js';
-import {
-  addDynamicProxyURL,
-  getWebProxiedURL,
-  shouldUseWebProxy,
-} from '../../../../ha/web-proxy.js';
+import { createProxiedEndpointIfNecessary } from '../../../../ha/web-proxy.js';
 import { localize } from '../../../../localize/localize.js';
 import liveGo2RTCStyle from '../../../../scss/live-go2rtc.scss';
 import { MediaPlayer, MediaPlayerController, Message } from '../../../../types.js';
@@ -100,12 +96,13 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
 
   protected async _getPlayerSource(): Promise<string | null> {
     const cameraConfig = this.camera?.getConfig();
+    const proxyConfig = this.camera?.getProxyConfig();
     if (!this.hass || !cameraConfig) {
       return null;
     }
 
-    const endpoint = this.cameraEndpoints?.go2rtc;
-    if (!endpoint) {
+    const streamEndpoint = this.cameraEndpoints?.go2rtc;
+    if (!streamEndpoint) {
       this._handleError({
         message: localize('error.live_camera_no_endpoint'),
         context: cameraConfig,
@@ -113,56 +110,49 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer
       return null;
     }
 
-    const proxyConfig = this.camera?.getProxyConfig();
-    let src: string | null = endpoint.endpoint;
-    let sign: boolean = endpoint.sign ?? false;
+    let result: string | null = null;
 
-    if (proxyConfig && shouldUseWebProxy(this.hass, proxyConfig, 'live')) {
-      if (proxyConfig.dynamic) {
-        try {
-          await addDynamicProxyURL(this.hass, endpoint.endpoint, {
-            proxyConfig,
-            ttl: GO2RTC_URL_SIGN_EXPIRY_SECONDS,
+    try {
+      const endpoint = await createProxiedEndpointIfNecessary(
+        this.hass,
+        streamEndpoint,
+        proxyConfig,
+        {
+          context: 'live',
+          ttl: GO2RTC_URL_SIGN_EXPIRY_SECONDS,
+          websocket: true,
 
-            // The link may need to be opened multiple times.
-            openLimit: 0,
-          });
-        } catch (e) {
-          this._handleError(
-            {
-              message: localize('error.failed_proxy'),
-              context: cameraConfig,
-            },
-            e as Error,
-          );
-          return null;
-        }
-      }
+          // The link may need to be opened multiple times.
+          openLimit: 0,
+        },
+      );
 
-      src = getWebProxiedURL(endpoint.endpoint, { websocket: true });
-      sign = true;
-    }
-
-    if (src && sign) {
-      try {
-        src = await homeAssistantSignPath(
+      if (endpoint.sign) {
+        result = await homeAssistantSignPath(
           this.hass,
-          src,
+          endpoint.endpoint,
           GO2RTC_URL_SIGN_EXPIRY_SECONDS,
         );
-      } catch (e) {
-        this._handleError(
-          {
+        if (!result) {
+          this._handleError({
             message: localize('error.failed_sign'),
             context: cameraConfig,
-          },
-          e as Error,
-        );
-        return null;
+          });
+        }
+      } else {
+        result = endpoint.endpoint;
       }
+    } catch (e) {
+      this._handleError(
+        {
+          message: localize('error.failed_proxy'),
+          context: cameraConfig,
+        },
+        e as Error,
+      );
     }
 
-    return src;
+    return result;
   }
 
   protected async _createPlayer(): Promise<void> {

src/components/live/providers/go2rtc/video-rtc.js

@@ -1,4 +1,8 @@
-import { mayHaveAudio } from '../../../../utils/audio';
+import {
+  addAudioTracksMuteStateListener,
+  has2WayAudio,
+  hasAudio,
+} from '../../../../utils/audio';
 import {
   hideMediaControlsTemporarily,
   MEDIA_LOAD_CONTROLS_HIDE_SECONDS,
@@ -172,6 +176,29 @@ export class VideoRTC extends HTMLElement {
      * @type {boolean}}
      */
     this.controls = true;
+
+    /**
+     * [internal] Cleanup function for audio track mute state listener.
+     * @type {Function | null}
+     */
+    this._audioTracksMuteStateCleanup = null;
+  }
+
+  /**
+   * Dispatch a media loaded event with current capabilities.
+   */
+  _dispatchMediaLoadedEvent() {
+    dispatchMediaLoadedEvent(this, this.video, {
+      ...(this.mediaPlayerController && {
+        mediaPlayerController: this.mediaPlayerController,
+      }),
+      capabilities: {
+        has2WayAudio: has2WayAudio(this.pc),
+        hasAudio: hasAudio(this.video, this.pc, this.mseCodecs),
+        supportsPause: true,
+      },
+      technology: getTechnologyForVideoRTC(this),
+    });
   }
 
   /**
@@ -352,21 +379,13 @@ export class VideoRTC extends HTMLElement {
       if (this.controls) {
         hideMediaControlsTemporarily(this.video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
       }
-      dispatchMediaLoadedEvent(this, this.video, {
-        ...(this.mediaPlayerController && {
-          mediaPlayerController: this.mediaPlayerController,
-        }),
-        capabilities: {
-          // 2-way audio is only supported on WebRTC connections. The state of
-          // `this.microphoneStream` is not taken into account here since
-          // that can be created after the fact -- this is purely saying that
-          // were a microphone stream available it could be used usefully.
-          supports2WayAudio: !!this.pc,
-          supportsPause: true,
-          hasAudio: mayHaveAudio(this.video),
-        },
-        technology: getTechnologyForVideoRTC(this),
-      });
+      this._dispatchMediaLoadedEvent();
+
+      // Listen for audio track mute/unmute changes and re-dispatch
+      this._audioTracksMuteStateCleanup?.();
+      this._audioTracksMuteStateCleanup = addAudioTracksMuteStateListener(this.pc, () =>
+        this._dispatchMediaLoadedEvent(),
+      );
     };
     this.video.onvolumechange = () => dispatchMediaVolumeChangeEvent(this);
     this.video.onplay = () => dispatchMediaPlayEvent(this);
@@ -415,6 +434,9 @@ export class VideoRTC extends HTMLElement {
 
     this.video.src = '';
     this.video.srcObject = null;
+
+    this._audioTracksMuteStateCleanup?.();
+    this._audioTracksMuteStateCleanup = null;
   }
 
   /**

src/components/viewer/provider.ts

@@ -22,11 +22,7 @@ import { isHARelativeURL } from '../../ha/is-ha-relative-url.js';
 import { ResolvedMediaCache, resolveMedia } from '../../ha/resolved-media.js';
 import { homeAssistantSignPath } from '../../ha/sign-path.js';
 import { HomeAssistant, ResolvedMedia } from '../../ha/types.js';
-import {
-  addDynamicProxyURL,
-  getWebProxiedURL,
-  shouldUseWebProxy,
-} from '../../ha/web-proxy.js';
+import { createProxiedEndpointIfNecessary } from '../../ha/web-proxy.js';
 import '../../patches/ha-hls-player.js';
 import viewerProviderStyle from '../../scss/viewer-provider.scss';
 import { MediaPlayer, MediaPlayerController, MediaPlayerElement } from '../../types.js';
@@ -153,34 +149,32 @@ export class AdvancedCameraCardViewerProvider extends LitElement implements Medi
     const camera = cameraID ? this.cameraManager?.getStore().getCamera(cameraID) : null;
     const proxyConfig = camera?.getProxyConfig();
 
-    if (proxyConfig && shouldUseWebProxy(this.hass, proxyConfig, 'media')) {
-      if (proxyConfig.dynamic) {
-        // Don't use URL() parsing, since that will strip the port number if
-        // it's the default, just need to strip any hash part of the URL.
-        const urlWithoutQSorHash = unsignedURL.split(/#/)[0];
-
-        try {
-          await addDynamicProxyURL(this.hass, urlWithoutQSorHash, {
-            proxyConfig,
-
-            // The link may need to be opened multiple times.
-            openLimit: 0,
-          });
-        } catch (e) {
-          errorToConsole(e as Error);
-        }
-      }
-
-      try {
-        this._url = await homeAssistantSignPath(
-          this.hass,
-          getWebProxiedURL(unsignedURL),
-        );
-      } catch (e) {
-        errorToConsole(e as Error);
-      }
-    } else {
+    if (!proxyConfig) {
       this._url = unsignedURL;
+      return;
+    }
+
+    try {
+      // Create endpoint from unsigned URL - it doesn't need signing initially
+      const unsignedEndpoint = { endpoint: unsignedURL, sign: false };
+      const proxiedEndpoint = await createProxiedEndpointIfNecessary(
+        this.hass,
+        unsignedEndpoint,
+        proxyConfig,
+        {
+          context: 'media',
+          // The link may need to be opened multiple times.
+          openLimit: 0,
+        },
+      );
+
+      if (proxiedEndpoint.sign) {
+        this._url = await homeAssistantSignPath(this.hass, proxiedEndpoint.endpoint);
+      } else {
+        this._url = proxiedEndpoint.endpoint;
+      }
+    } catch (e) {
+      errorToConsole(e as Error);
     }
   }