fix: Increase reliability of media auto actions (e.g. play) (#1934)

Refactors media player handling entirely to make the code more
consistent and less boilerplate. Add testing for media player actions.

- Closes #1921
- Closes #1916

src/components-lib/media-actions-controller.ts

@@ -5,7 +5,7 @@ import {
   AutoPlayCondition,
   AutoUnmuteCondition,
 } from '../config/types.js';
-import { AdvancedCameraCardMediaPlayer } from '../types.js';
+import { MediaPlayerElement } from '../types.js';
 import { AdvancedCameraCardMediaLoadedEventTarget } from '../utils/media-info.js';
 import { Timer } from '../utils/timer.js';
 
@@ -22,7 +22,6 @@ export interface MediaActionsControllerOptions {
 }
 
 type RenderRoot = HTMLElement & AdvancedCameraCardMediaLoadedEventTarget;
-type PlayerElement = HTMLElement & AdvancedCameraCardMediaPlayer;
 
 /**
  * General note: Always unmute before playing, since Chrome may pause a piece of
@@ -43,13 +42,17 @@ export class MediaActionsController {
   protected _root: RenderRoot | null = null;
 
   protected _eventListeners = new Map<HTMLElement, () => void>();
-  protected _children: PlayerElement[] = [];
+  protected _children: MediaPlayerElement[] = [];
   protected _target: MediaActionsTarget | null = null;
   protected _mutationObserver = new MutationObserver(this._mutationHandler.bind(this));
   protected _intersectionObserver = new IntersectionObserver(
     this._intersectionHandler.bind(this),
   );
 
+  constructor() {
+    document.addEventListener('visibilitychange', this._visibilityHandler);
+  }
+
   public setOptions(options: MediaActionsControllerOptions): void {
     if (this._options?.microphoneState !== options.microphoneState) {
       this._microphoneStateChangeHandler(
@@ -116,7 +119,7 @@ export class MediaActionsController {
     }
   }
   protected async _play(index: number): Promise<void> {
-    await this._children[index]?.play();
+    await (await this._children[index]?.getMediaPlayerController())?.play();
   }
   protected async _unmuteTargetIfConfigured(
     condition: AutoUnmuteCondition,
@@ -129,7 +132,7 @@ export class MediaActionsController {
     }
   }
   protected async _unmute(index: number): Promise<void> {
-    await this._children[index]?.unmute();
+    await (await this._children[index]?.getMediaPlayerController())?.unmute();
   }
 
   protected async _pauseAllIfConfigured(condition: AutoPauseCondition): Promise<void> {
@@ -150,7 +153,7 @@ export class MediaActionsController {
     }
   }
   protected async _pause(index: number): Promise<void> {
-    await this._children[index]?.pause();
+    await (await this._children[index]?.getMediaPlayerController())?.pause();
   }
 
   protected async _muteAllIfConfigured(condition: AutoMuteCondition): Promise<void> {
@@ -169,7 +172,7 @@ export class MediaActionsController {
     }
   }
   protected async _mute(index: number): Promise<void> {
-    await this._children[index]?.mute();
+    await (await this._children[index]?.getMediaPlayerController())?.mute();
   }
 
   protected _mutationHandler(
@@ -196,17 +199,21 @@ export class MediaActionsController {
     this._eventListeners.clear();
   }
 
-  public initialize(root: RenderRoot): void {
+  public setRoot(root: RenderRoot): boolean {
+    if (root === this._root) {
+      return false;
+    }
+
+    this._target = null;
     this._root = root;
     this._initializeRoot();
 
-    document.addEventListener('visibilitychange', this._visibilityHandler);
-
     this._intersectionObserver.disconnect();
-    this._intersectionObserver.observe(root);
+    this._intersectionObserver.observe(this._root);
 
     this._mutationObserver.disconnect();
     this._mutationObserver.observe(this._root, { childList: true, subtree: true });
+    return true;
   }
 
   protected _initializeRoot(): void {
@@ -217,7 +224,7 @@ export class MediaActionsController {
     this._removeChildHandlers();
 
     this._children = [
-      ...this._root.querySelectorAll<PlayerElement>(this._options.playerSelector),
+      ...this._root.querySelectorAll<MediaPlayerElement>(this._options.playerSelector),
     ];
 
     for (const [index, child] of this._children.entries()) {

src/components-lib/media-player/image.ts

@@ -0,0 +1,61 @@
+import { LitElement } from 'lit';
+import { FullscreenElement, MediaPlayerController } from '../../types';
+import { screenshotImage } from '../../utils/screenshot';
+
+export class ImageMediaPlayerController implements MediaPlayerController {
+  private _host: LitElement;
+  private _getImageCallback: () => HTMLImageElement | null;
+
+  constructor(host: LitElement, getImageCallback: () => HTMLImageElement | null) {
+    this._host = host;
+    this._getImageCallback = getImageCallback;
+  }
+
+  public async play(): Promise<void> {
+    // Not implemented.
+  }
+
+  public async pause(): Promise<void> {
+    // Not implemented.
+  }
+
+  public async mute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public async unmute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public isMuted(): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async seek(_seconds: number): Promise<void> {
+    // Not implemented.
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async setControls(_controls: boolean): Promise<void> {
+    // Not implemented.
+  }
+
+  public isPaused(): boolean {
+    // The image could be an MJPEG, so it is always reported unpaused.
+    return false;
+  }
+
+  public async getScreenshotURL(): Promise<string | null> {
+    await this._host.updateComplete;
+
+    const image = this._getImageCallback();
+
+    // It might an MJPEG so still need to screenshot it.
+    return image ? screenshotImage(image) : null;
+  }
+
+  public getFullscreenElement(): FullscreenElement | null {
+    return this._getImageCallback() ?? null;
+  }
+}

src/components-lib/media-player/jsmpeg.ts

@@ -0,0 +1,73 @@
+import JSMpeg from '@cycjimmy/jsmpeg-player';
+import { LitElement } from 'lit';
+import { FullscreenElement, MediaPlayerController } from '../../types';
+
+export class JSMPEGMediaPlayerController implements MediaPlayerController {
+  private _host: LitElement;
+  private _getJSMPEGVideoElementCallback: () => JSMpeg.VideoElement | null;
+  private _getCanvasElementCallback: () => HTMLCanvasElement | null;
+
+  constructor(
+    host: LitElement,
+    _getJSMPEGVideoElementCallback: () => JSMpeg.VideoElement | null,
+    _getCanvasElementCallback: () => HTMLCanvasElement | null,
+  ) {
+    this._host = host;
+    this._getJSMPEGVideoElementCallback = _getJSMPEGVideoElementCallback;
+    this._getCanvasElementCallback = _getCanvasElementCallback;
+  }
+
+  public async play(): Promise<void> {
+    await this._host.updateComplete;
+    return this._getJSMPEGVideoElementCallback()?.play();
+  }
+
+  public async pause(): Promise<void> {
+    await this._host.updateComplete;
+    return this._getJSMPEGVideoElementCallback()?.stop();
+  }
+
+  public async mute(): Promise<void> {
+    await this._host.updateComplete;
+    const player = this._getJSMPEGVideoElementCallback()?.player;
+    if (player) {
+      player.volume = 0;
+    }
+  }
+
+  public async unmute(): Promise<void> {
+    await this._host.updateComplete;
+    const player = this._getJSMPEGVideoElementCallback()?.player;
+    if (player) {
+      player.volume = 1;
+    }
+  }
+
+  public isMuted(): boolean {
+    const player = this._getJSMPEGVideoElementCallback()?.player;
+    return player ? player.volume === 0 : true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async seek(_seconds: number): Promise<void> {
+    // JSMPEG does not support seeking.
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async setControls(_controls: boolean): Promise<void> {
+    // Not implemented.
+  }
+
+  public isPaused(): boolean {
+    return this._getJSMPEGVideoElementCallback()?.player?.paused ?? true;
+  }
+
+  public async getScreenshotURL(): Promise<string | null> {
+    await this._host.updateComplete;
+    return this._getCanvasElementCallback()?.toDataURL('image/jpeg') ?? null;
+  }
+
+  public getFullscreenElement(): FullscreenElement | null {
+    return this._getCanvasElementCallback() ?? null;
+  }
+}

src/components-lib/media-player/updating-image.ts

@@ -0,0 +1,64 @@
+import { LitElement } from 'lit';
+import { FullscreenElement, MediaPlayerController } from '../../types';
+import { CachedValueController } from '../cached-value-controller';
+
+export class UpdatingImageMediaPlayerController implements MediaPlayerController {
+  private _host: LitElement;
+  private _getImageCallback: () => HTMLImageElement | null;
+  private _getCachedValueController: () => CachedValueController<string> | null;
+
+  constructor(
+    host: LitElement,
+    getImageCallback: () => HTMLImageElement | null,
+    getCachedValueController: () => CachedValueController<string> | null,
+  ) {
+    this._host = host;
+    this._getImageCallback = getImageCallback;
+    this._getCachedValueController = getCachedValueController;
+  }
+
+  public async play(): Promise<void> {
+    await this._host.updateComplete;
+    this._getCachedValueController()?.startTimer();
+  }
+
+  public async pause(): Promise<void> {
+    await this._host.updateComplete;
+    this._getCachedValueController()?.stopTimer();
+  }
+
+  public async mute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public async unmute(): Promise<void> {
+    // Not implemented.
+  }
+
+  public isMuted(): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async seek(_seconds: number): Promise<void> {
+    // Not implemented.
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public async setControls(_controls: boolean): Promise<void> {
+    // Not implemented.
+  }
+
+  public isPaused(): boolean {
+    return !this._getCachedValueController()?.hasTimer();
+  }
+
+  public async getScreenshotURL(): Promise<string | null> {
+    await this._host.updateComplete;
+    return this._getCachedValueController()?.value ?? null;
+  }
+
+  public getFullscreenElement(): FullscreenElement | null {
+    return this._getImageCallback() ?? null;
+  }
+}

src/components-lib/media-player/video.ts

@@ -0,0 +1,109 @@
+import { LitElement } from 'lit';
+import { FullscreenElement, MediaPlayerController } from '../../types';
+import { hideMediaControlsTemporarily, setControlsOnVideo } from '../../utils/controls';
+import { screenshotVideo } from '../../utils/screenshot';
+
+export class VideoMediaPlayerController implements MediaPlayerController {
+  private _host: LitElement;
+  private _getVideoCallback: () => HTMLVideoElement | null;
+  private _getControlsDefaultCallback: (() => boolean) | null;
+
+  constructor(
+    host: LitElement,
+    getVideoCallback: () => HTMLVideoElement | null,
+    getControlsDefaultCallback?: () => boolean,
+  ) {
+    this._host = host;
+    this._getVideoCallback = getVideoCallback;
+    this._getControlsDefaultCallback = getControlsDefaultCallback ?? null;
+  }
+
+  public async play(): Promise<void> {
+    await this._host.updateComplete;
+
+    const video = this._getVideoCallback();
+    if (!video?.play) {
+      return;
+    }
+
+    // If the play call fails, and the media is not already muted, mute it first
+    // and then try again. This works around some browsers that prevent
+    // auto-play unless the video is muted.
+    try {
+      await video.play();
+    } catch (err: unknown) {
+      if ((err as Error).name === 'NotAllowedError' && !this.isMuted()) {
+        await this.mute();
+        try {
+          await video.play();
+        } catch (_) {
+          // Pass.
+        }
+      }
+    }
+  }
+
+  public async pause(): Promise<void> {
+    await this._host.updateComplete;
+    this._getVideoCallback()?.pause();
+  }
+
+  public async mute(): Promise<void> {
+    await this._host.updateComplete;
+
+    // The muted property is only for the initial muted state. Must explicitly
+    // set the muted on the video player to make the change dynamic.
+    const video = this._getVideoCallback();
+    if (video) {
+      video.muted = true;
+    }
+  }
+
+  public async unmute(): Promise<void> {
+    await this._host.updateComplete;
+
+    const video = this._getVideoCallback();
+    if (video) {
+      video.muted = false;
+    }
+  }
+
+  public isMuted(): boolean {
+    return this._getVideoCallback()?.muted ?? true;
+  }
+
+  public async seek(seconds: number): Promise<void> {
+    await this._host.updateComplete;
+
+    const video = this._getVideoCallback();
+    if (video) {
+      hideMediaControlsTemporarily(video);
+      video.currentTime = seconds;
+    }
+  }
+
+  public async setControls(controls?: boolean): Promise<void> {
+    await this._host.updateComplete;
+
+    const video = this._getVideoCallback();
+    const value = controls ?? this._getControlsDefaultCallback?.();
+    if (video && value !== undefined) {
+      setControlsOnVideo(video, value);
+    }
+  }
+
+  public isPaused(): boolean {
+    return this._getVideoCallback()?.paused ?? true;
+  }
+
+  public async getScreenshotURL(): Promise<string | null> {
+    await this._host.updateComplete;
+
+    const video = this._getVideoCallback();
+    return video ? screenshotVideo(video) : null;
+  }
+
+  public getFullscreenElement(): FullscreenElement | null {
+    return this._getVideoCallback() ?? null;
+  }
+}

src/components-lib/menu-button-controller.ts

@@ -489,10 +489,10 @@ export class MenuButtonController {
   ): MenuItem | null {
     if (
       currentMediaLoadedInfo &&
-      currentMediaLoadedInfo.player &&
+      currentMediaLoadedInfo.mediaPlayerController &&
       currentMediaLoadedInfo.capabilities?.supportsPause
     ) {
-      const paused = currentMediaLoadedInfo.player.isPaused();
+      const paused = currentMediaLoadedInfo.mediaPlayerController?.isPaused();
       return {
         icon: paused ? 'mdi:play' : 'mdi:pause',
         ...config.menu.buttons.play,
@@ -512,10 +512,10 @@ export class MenuButtonController {
   ): MenuItem | null {
     if (
       currentMediaLoadedInfo &&
-      currentMediaLoadedInfo.player &&
+      currentMediaLoadedInfo.mediaPlayerController &&
       currentMediaLoadedInfo?.capabilities?.hasAudio
     ) {
-      const muted = currentMediaLoadedInfo.player.isMuted();
+      const muted = currentMediaLoadedInfo.mediaPlayerController?.isMuted();
       return {
         icon: muted ? 'mdi:volume-off' : 'mdi:volume-high',
         ...config.menu.buttons.mute,
@@ -533,7 +533,7 @@ export class MenuButtonController {
     config: AdvancedCameraCardConfig,
     currentMediaLoadedInfo?: MediaLoadedInfo | null,
   ): MenuItem | null {
-    if (currentMediaLoadedInfo && currentMediaLoadedInfo.player) {
+    if (currentMediaLoadedInfo && currentMediaLoadedInfo.mediaPlayerController) {
       return {
         icon: 'mdi:monitor-screenshot',
         ...config.menu.buttons.screenshot,