Merge pull request #1150 from dermotduffy/2-way-audio-is-a-property-of-loaded-media

Only show the microphone button if the media that is loaded supports 2-way audio

README.md

@@ -406,6 +406,8 @@ See the [fully expanded menu configuration example](#config-expanded-menu) for h
 | `button_size` | 40 | :white_check_mark: | The size of the menu buttons in pixels. Must be >= `20`.|
 | `buttons` | | :white_check_mark: | Whether to show or hide built-in buttons. See below. |
 
+<a name="menu-buttons"></a>
+
 #### Menu Options: Buttons
 
 All configuration is under:
@@ -433,6 +435,7 @@ menu:
 | `expand` | :white_check_mark: | The `expand` menu button: expand the card into a popup/dialog. |
 | `timeline` | :white_check_mark: | The `timeline` menu button: show the event timeline. |
 | `media_player` | :white_check_mark: | The `media_player` menu button: sends the visible media to a remote media player. Supports Frigate clips, snapshots and live camera (only for cameras that specify a `camera_entity` and only using the default HA stream (equivalent to the `ha` live provider). `jsmpeg` or `webrtc-card` are not supported, although live can still be played as long as `camera_entity` is specified. In the player list, a `tap` will send the media to the player, a `hold` will stop the media on the player. |
+| `microphone` | :white_check_mark: | The `microphone` button allows usage of 2-way audio in certain configurations. See [Using 2-way audio](#using-2-way-audio). |
 
 ##### Configuration on each button
 
@@ -3863,6 +3866,12 @@ with `go2rtc` not with the card. In this case, you could file an issue in [that
 repo](https://github.com/AlexxIT/go2rtc/issues) with debugging information as
 appropriate.
 
+### Microphone menu button does not appear
+
+The microphone menu button will only appear if both enabled (see [Menu Button Options](#menu-buttons))
+and if the media that is currently loaded supports 2-way audio. See
+[Using 2-way audio](#using-2-way-audio) for more information about the requirements that must be followed.
+
 ### Static image URL with credentials doesn't load
 
 Your browser will not allow a page/script (like this card) to pass credentials to a cross-origin (different host) image URL for security reasons. There is no way around this unless you could also control the webserver that is serving the image to specifically allow `crossorigin` requests (which is typically not the case for an image served from a camera, for example). The stock Home Assistant Picture Glance card has the same limitation, for the same reasons.

src/camera-manager/frigate/engine-frigate.ts

@@ -1081,7 +1081,6 @@ export class FrigateCameraManagerEngine
       supportsSnapshots: !isBirdseye,
       supportsRecordings: !isBirdseye,
       supportsTimeline: !isBirdseye,
-      supports2WayAudio: cameraConfig.live_provider === 'go2rtc',
     };
   }
 

src/camera-manager/generic/engine-generic.ts

@@ -182,7 +182,6 @@ export class GenericCameraManagerEngine implements CameraManagerEngine {
       supportsRecordings: false,
       supportsSnapshots: false,
       supportsTimeline: false,
-      supports2WayAudio: false,
     };
   }
 

src/camera-manager/manager.ts

@@ -755,7 +755,6 @@ export class CameraManager {
       supportsRecordings: perCameraCapabilities.some((cap) => cap?.supportsRecordings),
       supportsSnapshots: perCameraCapabilities.some((cap) => cap?.supportsSnapshots),
       supportsTimeline: perCameraCapabilities.some((cap) => cap?.supportsTimeline),
-      supports2WayAudio: perCameraCapabilities.some((cap) => cap?.supports2WayAudio),
     };
   }
 }

src/camera-manager/types.ts

@@ -100,7 +100,6 @@ interface BaseCapabilities {
   supportsRecordings: boolean;
   supportsSnapshots: boolean;
   supportsTimeline: boolean;
-  supports2WayAudio: boolean;
 }
 
 export type CameraManagerCapabilities = BaseCapabilities;

src/card.ts

@@ -619,7 +619,10 @@ class FrigateCard extends LitElement {
       });
     }
 
-    if (this._microphoneController && cameraCapabilities?.supports2WayAudio) {
+    if (
+      this._microphoneController &&
+      this._currentMediaLoadedInfo?.capabilities?.supports2WayAudio
+    ) {
       const muted = this._microphoneController.isMuted();
       const buttonType = this._getConfig().menu.buttons.microphone.type;
       buttons.push({

src/components/image.ts

@@ -11,27 +11,32 @@ import {
 import { customElement, property } from 'lit/decorators.js';
 import { live } from 'lit/directives/live.js';
 import { createRef, ref, Ref } from 'lit/directives/ref.js';
+import isEqual from 'lodash-es/isEqual';
 import { CachedValueController } from '../cached-value-controller.js';
 import defaultImage from '../images/frigate-bird-in-sky.jpg';
 import { localize } from '../localize/localize.js';
 import imageStyle from '../scss/image.scss';
-import { CameraConfig, ImageViewConfig, MediaLoadedInfo } from '../types.js';
+import {
+  CameraConfig,
+  FrigateCardMediaPlayer,
+  ImageViewConfig,
+  MediaLoadedInfo,
+} from '../types.js';
+import { contentsChanged } from '../utils/basic.js';
 import { isHassDifferent } from '../utils/ha';
-import { updateElementStyleFromMediaLayoutConfig } from '../utils/media-layout.js';
 import {
   createMediaLoadedInfo,
   dispatchExistingMediaLoadedInfoAsEvent,
 } from '../utils/media-info.js';
+import { updateElementStyleFromMediaLayoutConfig } from '../utils/media-layout.js';
 import { View } from '../view/view.js';
 import { dispatchErrorMessageEvent } from './message.js';
-import { contentsChanged } from '../utils/basic.js';
-import isEqual from 'lodash-es/isEqual';
 
 // See TOKEN_CHANGE_INTERVAL in https://github.com/home-assistant/core/blob/dev/homeassistant/components/camera/__init__.py .
 const HASS_REJECTION_CUTOFF_MS = 5 * 60 * 1000;
 
 @customElement('frigate-card-image')
-export class FrigateCardImage extends LitElement {
+export class FrigateCardImage extends LitElement implements FrigateCardMediaPlayer {
   @property({ attribute: false })
   public hass?: HomeAssistant;
 
@@ -54,6 +59,36 @@ export class FrigateCardImage extends LitElement {
 
   protected _mediaLoadedInfo: MediaLoadedInfo | null = null;
 
+  public async play(): Promise<void> {
+    this._cachedValueController?.startTimer();
+  }
+
+  public async pause(): Promise<void> {
+    this._cachedValueController?.stopTimer();
+  }
+
+  public async mute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public async unmute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public isMuted(): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async seek(_seconds: number): Promise<void> {
+    // Not implemented.
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async setControls(_controls: boolean): Promise<void> {
+    // Not implemented.
+  }
+
   /**
    * Get the camera entity for the current camera configuration.
    * @returns The entity or undefined if no camera entity is available.
@@ -250,7 +285,7 @@ export class FrigateCardImage extends LitElement {
           ${ref(this._refImage)}
           src=${live(src)}
           @load=${(ev: Event) => {
-            const mediaLoadedInfo = createMediaLoadedInfo(ev);
+            const mediaLoadedInfo = createMediaLoadedInfo(ev, { player: this });
             // Avoid the media being reported as repeatedly loading unless the
             // media info changes.
             if (mediaLoadedInfo && !isEqual(this._mediaLoadedInfo, mediaLoadedInfo)) {

src/components/live/live-go2rtc.ts

@@ -36,9 +36,11 @@ const GO2RTC_URL_SIGN_EXPIRY_SECONDS = 24 * 60 * 60;
 @customElement('frigate-card-live-go2rtc-player')
 class FrigateCardGo2RTCPlayer extends VideoRTC {
   protected _microphoneStream?: MediaStream;
+  protected _containingPlayer?: FrigateCardMediaPlayer;
 
-  constructor(microphoneStream?: MediaStream) {
+  constructor(containingPlayer: FrigateCardMediaPlayer, microphoneStream?: MediaStream) {
     super();
+    this._containingPlayer = containingPlayer;
     if (microphoneStream) {
       this._microphoneStream = microphoneStream;
     }
@@ -79,7 +81,16 @@ class FrigateCardGo2RTCPlayer extends VideoRTC {
           onloadeddata.call(this.video, e);
         }
         hideMediaControlsTemporarily(this.video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
-        dispatchMediaLoadedEvent(this, this.video);
+        dispatchMediaLoadedEvent(this, this.video, {
+          player: this._containingPlayer,
+          capabilities: {
+            // 2-way audio is only supported on WebRTC connections. The state of
+            // `this._microphoneStream` is not taken into account here since
+            // that can be created after the fact -- this is purely saying that
+            // were a microphone stream available it could be used usefully.
+            supports2WayAudio: !!this.pc,
+          },
+        });
       };
 
       // Always started muted. Media may be unmuted in accordance with user
@@ -256,7 +267,7 @@ export class FrigateCardGo2RTC extends LitElement implements FrigateCardMediaPla
       return;
     }
 
-    this._player = new FrigateCardGo2RTCPlayer(this.microphoneStream);
+    this._player = new FrigateCardGo2RTCPlayer(this, this.microphoneStream);
     this._player.src = address;
     this._player.visibilityCheck = false;
 

src/components/live/live-image.ts

@@ -1,10 +1,11 @@
 import { HomeAssistant } from 'custom-card-helpers';
 import { CSSResultGroup, html, LitElement, TemplateResult, unsafeCSS } from 'lit';
-import { customElement, property, state } from 'lit/decorators.js';
+import { customElement, property } from 'lit/decorators.js';
+import { createRef, ref, Ref } from 'lit/directives/ref.js';
 import liveImageStyle from '../../scss/live-image.scss';
 import { CameraConfig, FrigateCardMediaPlayer } from '../../types.js';
-import { getStateObjOrDispatchError } from './live.js';
 import '../image.js';
+import { getStateObjOrDispatchError } from './live.js';
 
 @customElement('frigate-card-live-image')
 export class FrigateCardLiveImage extends LitElement implements FrigateCardMediaPlayer {
@@ -14,37 +15,34 @@ export class FrigateCardLiveImage extends LitElement implements FrigateCardMedia
   @property({ attribute: false })
   public cameraConfig?: CameraConfig;
 
-  @state()
+  protected _refImage: Ref<Element & FrigateCardMediaPlayer> = createRef();
-  protected _playing = true;
 
   public async play(): Promise<void> {
-    this._playing = true;
+    await this._refImage.value?.play();
   }
 
   public async pause(): Promise<void> {
-    this._playing = false;
+    await this._refImage.value?.pause();
   }
 
   public async mute(): Promise<void> {
-    // Not implemented.
+    await this._refImage.value?.mute();
   }
 
   public async unmute(): Promise<void> {
-    // Not implemented.
+    await this._refImage.value?.unmute();
   }
 
   public isMuted(): boolean {
-    return true;
+    return !!this._refImage.value?.isMuted();
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async seek(seconds: number): Promise<void> {
-  public async seek(_seconds: number): Promise<void> {
+    await this._refImage.value?.seek(seconds);
-    // Not implemented.
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async setControls(controls: boolean): Promise<void> {
-  public async setControls(_controls: boolean): Promise<void> {
+    await this._refImage.value?.setControls(controls);
-    // Not implemented.
   }
 
   protected render(): TemplateResult | void {
@@ -55,16 +53,14 @@ export class FrigateCardLiveImage extends LitElement implements FrigateCardMedia
     getStateObjOrDispatchError(this, this.hass, this.cameraConfig);
 
     return html` <frigate-card-image
+      ${ref(this._refImage)}
       .imageConfig=${{
         mode: this.cameraConfig.image.url ? ('url' as const) : ('camera' as const),
-        refresh_seconds: this._playing ? this.cameraConfig.image.refresh_seconds : 0,
+        refresh_seconds: this.cameraConfig.image.refresh_seconds,
         url: this.cameraConfig.image.url,
 
-        // The live provider will take care of zoom.
+        // The live provider will take care of zoom and layout options.
         zoomable: false,
-
-        // Don't need to pass layout options as FrigateCardLiveProvider has
-        // already taken care of this for us.
       }}
       .hass=${this.hass}
       .cameraConfig=${this.cameraConfig}

src/components/live/live-jsmpeg.ts

@@ -111,7 +111,9 @@ export class FrigateCardLiveJSMPEG extends LitElement implements FrigateCardMedi
             // ignore any subsequent calls.
             if (!videoDecoded && this._jsmpegCanvasElement) {
               videoDecoded = true;
-              dispatchMediaLoadedEvent(this, this._jsmpegCanvasElement);
+              dispatchMediaLoadedEvent(this, this._jsmpegCanvasElement, {
+                player: this,
+              });
               resolve(player);
             }
           },

src/components/live/live-webrtc-card.ts

@@ -185,7 +185,7 @@ export class FrigateCardLiveWebRTCCard
             onloadeddata.call(video, e);
           }
           hideMediaControlsTemporarily(video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
-          dispatchMediaLoadedEvent(this, video);
+          dispatchMediaLoadedEvent(this, video, { player: this });
         };
       }
     });

src/components/viewer.ts

@@ -750,7 +750,7 @@ export class FrigateCardViewerProvider
                   }
                 }}
                 @loadeddata=${(ev: Event) => {
-                  dispatchMediaLoadedEvent(this, ev);
+                  dispatchMediaLoadedEvent(this, ev, { player: this });
                 }}
               >
                 <source
@@ -768,8 +768,8 @@ export class FrigateCardViewerProvider
                 this._dispatchRelatedClipView();
               }
             }}
-            @load=${(e: Event) => {
+            @load=${(ev: Event) => {
-              dispatchMediaLoadedEvent(this, e);
+              dispatchMediaLoadedEvent(this, ev, { player: this });
             }}
           />`}
     `);

src/patches/ha-camera-stream.ts

@@ -90,7 +90,7 @@ customElements.whenDefined('ha-camera-stream').then(() => {
         return html`
           <img
             @load=${(ev: Event) => {
-              dispatchMediaLoadedEvent(this, ev);
+              dispatchMediaLoadedEvent(this, ev, { player: this });
             }}
             .src=${typeof this._connected == 'undefined' || this._connected
               ? computeMJPEGStreamUrl(this.stateObj)

src/patches/ha-hls-player.ts

@@ -97,7 +97,7 @@ customElements.whenDefined('ha-hls-player').then(() => {
             hideMediaControlsTemporarily(this._video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
           }}
           @loadeddata=${(e) => {
-            dispatchMediaLoadedEvent(this, e);
+            dispatchMediaLoadedEvent(this, e, { player: this });
           }}
         ></video>
       `;

src/patches/ha-web-rtc-player.ts

@@ -93,7 +93,7 @@ customElements.whenDefined('ha-web-rtc-player').then(() => {
             hideMediaControlsTemporarily(this._video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
           }}
           @loadeddata=${(e) => {
-            dispatchMediaLoadedEvent(this, e);
+            dispatchMediaLoadedEvent(this, e, { player: this });
           }}
         ></video>
       `;

src/types.ts

@@ -1471,9 +1471,15 @@ export interface ExtendedHomeAssistant extends HomeAssistant {
   };
 }
 
+export interface MediaLoadedCapabilities {
+  supports2WayAudio?: boolean;
+}
+
 export interface MediaLoadedInfo {
   width: number;
   height: number;
+  player?: FrigateCardMediaPlayer;
+  capabilities?: MediaLoadedCapabilities;
 }
 
 export const MESSAGE_TYPE_PRIORITIES = {

src/utils/media-info.ts

@@ -1,4 +1,8 @@
-import { MediaLoadedInfo } from '../types.js';
+import {
+  FrigateCardMediaPlayer,
+  MediaLoadedCapabilities,
+  MediaLoadedInfo,
+} from '../types.js';
 import { dispatchFrigateCardEvent } from './basic.js';
 
 const MEDIA_INFO_HEIGHT_CUTOFF = 50;
@@ -11,6 +15,10 @@ const MEDIA_INFO_WIDTH_CUTOFF = MEDIA_INFO_HEIGHT_CUTOFF;
  */
 export function createMediaLoadedInfo(
   source: Event | HTMLElement,
+  options?: {
+    player?: FrigateCardMediaPlayer;
+    capabilities?: MediaLoadedCapabilities;
+  },
 ): MediaLoadedInfo | null {
   let target: HTMLElement | EventTarget;
   if (source instanceof Event) {
@@ -23,16 +31,20 @@ export function createMediaLoadedInfo(
     return {
       width: (target as HTMLImageElement).naturalWidth,
       height: (target as HTMLImageElement).naturalHeight,
+      ...options,
     };
   } else if (target instanceof HTMLVideoElement) {
     return {
       width: (target as HTMLVideoElement).videoWidth,
       height: (target as HTMLVideoElement).videoHeight,
+      ...options,
     };
   } else if (target instanceof HTMLCanvasElement) {
     return {
       width: (target as HTMLCanvasElement).width,
       height: (target as HTMLCanvasElement).height,
+      player: options?.player,
+      ...options,
     };
   }
   return null;
@@ -46,8 +58,12 @@ export function createMediaLoadedInfo(
 export function dispatchMediaLoadedEvent(
   target: HTMLElement,
   source: Event | HTMLElement,
+  options?: {
+    player?: FrigateCardMediaPlayer;
+    capabilities?: MediaLoadedCapabilities;
+  },
 ): void {
-  const mediaLoadedInfo = createMediaLoadedInfo(source);
+  const mediaLoadedInfo = createMediaLoadedInfo(source, options);
   if (mediaLoadedInfo) {
     dispatchExistingMediaLoadedInfoAsEvent(target, mediaLoadedInfo);
   }