feat: Automatically recover from frozen live streams (#2569)

- Closes: #2099

tests/components-lib/media-player/frame-stall-watchdog.test.ts

@@ -0,0 +1,264 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import {
+  FRAME_STALL_SECONDS,
+  FrameStallWatchdog,
+  type FrameStallWatchdogConfig,
+} from '../../../src/components-lib/media-player/frame-stall-watchdog';
+
+const STALL_MS = FRAME_STALL_SECONDS * 1000;
+
+const createConfig = (
+  overrides?: Partial<FrameStallWatchdogConfig>,
+): FrameStallWatchdogConfig => ({
+  isPlaybackExpected: vi.fn().mockReturnValue(true),
+  startSource: vi.fn().mockReturnValue(true),
+  stopSource: vi.fn(),
+  ...overrides,
+});
+
+// @vitest-environment jsdom
+describe('FrameStallWatchdog', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  describe('source lifecycle', () => {
+    it('should start the source only on the first subscriber', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+
+      watchdog.subscribe(vi.fn());
+      watchdog.subscribe(vi.fn());
+
+      expect(config.startSource).toHaveBeenCalledTimes(1);
+    });
+
+    it('should stop the source only on the last unsubscribe', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+
+      const unsubscribeFirst = watchdog.subscribe(vi.fn());
+      const unsubscribeSecond = watchdog.subscribe(vi.fn());
+
+      unsubscribeFirst();
+      expect(config.stopSource).not.toHaveBeenCalled();
+
+      unsubscribeSecond();
+      expect(config.stopSource).toHaveBeenCalledTimes(1);
+    });
+
+    it('should default to an always-available source needing no teardown', () => {
+      const watchdog = new FrameStallWatchdog({ isPlaybackExpected: () => true });
+      const callback = vi.fn();
+
+      const unsubscribe = watchdog.subscribe(callback);
+      vi.advanceTimersByTime(STALL_MS);
+      unsubscribe(); // no stopSource configured -> no throw
+
+      expect(callback).toHaveBeenCalledWith(false);
+    });
+  });
+
+  describe('stall detection', () => {
+    it('should report stalled when no frame arrives within the window', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(false);
+    });
+
+    it('should stay live while frames keep arriving within the window', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      for (let i = 0; i < 5; i++) {
+        vi.advanceTimersByTime(STALL_MS - 1000);
+        watchdog.notifyFrame();
+      }
+      vi.advanceTimersByTime(STALL_MS - 1000);
+
+      // Confirmed live on the first frame, and never stalled thereafter.
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(true);
+    });
+
+    it('should report live again once a frame arrives after a stall', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      vi.advanceTimersByTime(STALL_MS);
+      expect(callback).toHaveBeenNthCalledWith(1, false);
+
+      watchdog.notifyFrame();
+      expect(callback).toHaveBeenNthCalledWith(2, true);
+    });
+
+    it('should not report stalled while the source is legitimately idle', () => {
+      const config = createConfig({
+        isPlaybackExpected: vi.fn().mockReturnValue(false),
+      });
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should re-arm while idle so a freeze that becomes actionable later is caught', () => {
+      const isPlaybackExpected = vi.fn().mockReturnValue(false);
+      const config = createConfig({ isPlaybackExpected });
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+
+      // Idle window: no stall reported, but the watchdog re-arms rather than
+      // stopping.
+      vi.advanceTimersByTime(STALL_MS);
+      expect(callback).not.toHaveBeenCalled();
+
+      // The source becomes actionable while still frozen (holding a frame), with
+      // no new frame to kick the timer. The re-armed timer catches the freeze on
+      // the next window.
+      isPlaybackExpected.mockReturnValue(true);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(false);
+    });
+
+    it('should broadcast a stall to every subscriber', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const first = vi.fn();
+      const second = vi.fn();
+
+      watchdog.subscribe(first);
+      watchdog.subscribe(second);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(first).toHaveBeenCalledTimes(1);
+      expect(first).toHaveBeenCalledWith(false);
+      expect(second).toHaveBeenCalledTimes(1);
+      expect(second).toHaveBeenCalledWith(false);
+    });
+
+    it('should not report live merely on subscribing before any frame', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      // Unconfirmed until a real frame arrives: subscribing alone does not
+      // report live.
+      watchdog.subscribe(callback);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should confirm live on the first frame but not re-notify on the next', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      watchdog.notifyFrame();
+      watchdog.notifyFrame();
+
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(true);
+    });
+  });
+
+  describe('no available source', () => {
+    it('should never report stalled when the source is unavailable', () => {
+      const config = createConfig({ startSource: vi.fn().mockReturnValue(false) });
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+      expect(config.isPlaybackExpected).not.toHaveBeenCalled();
+    });
+
+    it('should ignore notifyFrame when the source is unavailable', () => {
+      const config = createConfig({ startSource: vi.fn().mockReturnValue(false) });
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      watchdog.subscribe(callback);
+      watchdog.notifyFrame();
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should not stop a source that was never started on unsubscribe', () => {
+      const config = createConfig({ startSource: vi.fn().mockReturnValue(false) });
+      const watchdog = new FrameStallWatchdog(config);
+
+      const unsubscribe = watchdog.subscribe(vi.fn());
+      unsubscribe();
+
+      expect(config.stopSource).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('teardown', () => {
+    it('should not report a stall after the last subscriber leaves', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      const callback = vi.fn();
+
+      const unsubscribe = watchdog.subscribe(callback);
+      unsubscribe();
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should stop cleanly if the last subscriber unsubscribes during a recovery notification', () => {
+      const config = createConfig();
+      const watchdog = new FrameStallWatchdog(config);
+      let unsubscribe: () => void = () => {};
+      const callback = vi.fn((isLive: boolean) => {
+        if (isLive) {
+          unsubscribe();
+        }
+      });
+
+      unsubscribe = watchdog.subscribe(callback);
+
+      // Stall -> callback(false)
+      vi.advanceTimersByTime(STALL_MS);
+
+      // Recovery -> callback(true) -> unsubscribe -> stop
+      watchdog.notifyFrame();
+
+      expect(config.stopSource).toHaveBeenCalledTimes(1);
+
+      // The timer armed by notifyFrame must have been stopped, so no further
+      // stall fires with no subscribers.
+      callback.mockClear();
+      vi.advanceTimersByTime(STALL_MS);
+      expect(callback).not.toHaveBeenCalled();
+    });
+  });
+});

tests/components-lib/media-player/jsmpeg.test.ts

@@ -1,7 +1,8 @@
 import type JSMpeg from '@cycjimmy/jsmpeg-player';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { mock } from 'vitest-mock-extended';
 
+import { FRAME_STALL_SECONDS } from '../../../src/components-lib/media-player/frame-stall-watchdog';
 import { JSMPEGMediaPlayerController } from '../../../src/components-lib/media-player/jsmpeg';
 import { createLitElement } from '../../test-utils';
 
@@ -260,4 +261,64 @@ describe('JSMPEGMediaPlayerController', () => {
 
     expect(controller.getPIPElement()).toBeNull();
   });
+
+  describe('subscribeLiveness', () => {
+    const STALL_MS = FRAME_STALL_SECONDS * 1000;
+
+    const setup = (options?: { paused?: boolean; hasPlayer?: boolean }) => {
+      let videoElement: JSMpeg.VideoElement | null = null;
+      if (options?.hasPlayer !== false) {
+        videoElement = mock<JSMpeg.VideoElement>();
+        videoElement.player = mock<JSMpeg.Player>();
+        videoElement.player.paused = options?.paused ?? false;
+      }
+      const controller = new JSMPEGMediaPlayerController(
+        createLitElement(),
+        () => videoElement,
+        () => mock<HTMLCanvasElement>(),
+      );
+      return { controller };
+    };
+
+    beforeEach(() => {
+      vi.useFakeTimers();
+    });
+
+    afterEach(() => {
+      vi.useRealTimers();
+    });
+
+    it('should report a stall when frame decodes stop', () => {
+      const { controller } = setup();
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      controller.notifyFrameDecoded();
+      vi.advanceTimersByTime(STALL_MS);
+
+      // The first decoded frame confirms live, then decodes stop -> stall.
+      expect(callback).toHaveBeenNthCalledWith(1, true);
+      expect(callback).toHaveBeenNthCalledWith(2, false);
+    });
+
+    it('should not report a stall while paused', () => {
+      const { controller } = setup({ paused: true });
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should not report a stall without a player', () => {
+      const { controller } = setup({ hasPlayer: false });
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+  });
 });

tests/components-lib/media-player/video.test.ts

@@ -1,6 +1,7 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { mock } from 'vitest-mock-extended';
 
+import { FRAME_STALL_SECONDS } from '../../../src/components-lib/media-player/frame-stall-watchdog';
 import { VideoMediaPlayerController } from '../../../src/components-lib/media-player/video';
 import {
   hideMediaControlsTemporarily,
@@ -16,6 +17,48 @@ class NotAllowedError extends Error {
   name = 'NotAllowedError';
 }
 
+const STALL_MS = FRAME_STALL_SECONDS * 1000;
+
+// A real jsdom <video> (so `'requestVideoFrameCallback' in video` is genuinely
+// false unless we add it) with controllable properties and a driveable frame
+// callback.
+const createVideo = (options?: {
+  readyState?: number;
+  paused?: boolean;
+  seeking?: boolean;
+  ended?: boolean;
+  rvfc?: boolean;
+}): {
+  video: HTMLVideoElement;
+  deliverFrame: () => void;
+  cancel: ReturnType<typeof vi.fn>;
+} => {
+  const video = document.createElement('video');
+  const define = (prop: string, value: unknown): void => {
+    Object.defineProperty(video, prop, { value, configurable: true });
+  };
+  define('readyState', options?.readyState ?? HTMLMediaElement.HAVE_CURRENT_DATA);
+  define('paused', options?.paused ?? false);
+  define('seeking', options?.seeking ?? false);
+  define('ended', options?.ended ?? false);
+
+  let frameCallback: (() => void) | null = null;
+  const cancel = vi.fn();
+  if (options?.rvfc !== false) {
+    let handle = 0;
+    define(
+      'requestVideoFrameCallback',
+      vi.fn((cb: () => void) => {
+        frameCallback = cb;
+        return ++handle;
+      }),
+    );
+    define('cancelVideoFrameCallback', cancel);
+  }
+
+  return { video, deliverFrame: () => frameCallback?.(), cancel };
+};
+
 // @vitest-environment jsdom
 describe('VideoMediaPlayerController', () => {
   beforeEach(() => {
@@ -287,4 +330,145 @@ describe('VideoMediaPlayerController', () => {
       expect(controller.getPIPElement()).toBeNull();
     });
   });
+
+  describe('subscribeLiveness', () => {
+    beforeEach(() => {
+      vi.useFakeTimers();
+    });
+
+    afterEach(() => {
+      vi.useRealTimers();
+    });
+
+    it('should report a stall when frames stop arriving', () => {
+      const { video, deliverFrame } = createVideo();
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      deliverFrame();
+      vi.advanceTimersByTime(STALL_MS);
+
+      // The first frame confirms live, then no further frame arrives -> stall.
+      expect(callback).toHaveBeenNthCalledWith(1, true);
+      expect(callback).toHaveBeenNthCalledWith(2, false);
+    });
+
+    it('should not report a stall while a paused video still holds a frame', () => {
+      // A genuine user pause: paused with a current frame is idle, never
+      // reported -- not even after the stall window.
+      const { video } = createVideo({ paused: true });
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should report a video stuck paused with no current frame', () => {
+      // The observed go2rtc failure: its internal reconnect leaves the <video>
+      // paused at readyState 0 (spinning). That is not a genuine pause -- there
+      // is no frame to pause on -- so playback is still expected and a missing
+      // frame is a stall.
+      const { video } = createVideo({
+        paused: true,
+        readyState: HTMLMediaElement.HAVE_NOTHING,
+      });
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(false);
+    });
+
+    it('should report no stall when requestVideoFrameCallback is unavailable', () => {
+      const { video } = createVideo({ rvfc: false });
+      expect('requestVideoFrameCallback' in video).toBe(false);
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should stop watching and cancel the frame callback on unsubscribe', () => {
+      const { video, cancel } = createVideo();
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      const callback = vi.fn();
+
+      const unsubscribe = controller.subscribeLiveness(callback);
+      unsubscribe();
+      vi.advanceTimersByTime(STALL_MS);
+
+      expect(cancel).toHaveBeenCalled();
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('should not re-arm when the last subscriber unsubscribes during recovery', () => {
+      const { video, deliverFrame, cancel } = createVideo();
+      const controller = new VideoMediaPlayerController(createLitElement(), () => video);
+      let unsubscribe: () => void = () => {};
+      const callback = vi.fn((isLive: boolean) => {
+        if (isLive) {
+          unsubscribe();
+        }
+      });
+
+      unsubscribe = controller.subscribeLiveness(callback);
+
+      // Stall -> callback(false)
+      vi.advanceTimersByTime(STALL_MS);
+
+      // Recovery -> callback(true) -> unsubscribe -> source stopped
+      deliverFrame();
+
+      expect(cancel).toHaveBeenCalled();
+    });
+
+    it('should skip re-arming and report no stall when the video is gone between frames', () => {
+      const { video, deliverFrame } = createVideo();
+      let currentVideo: HTMLVideoElement | null = video;
+      const controller = new VideoMediaPlayerController(
+        createLitElement(),
+        () => currentVideo,
+      );
+      const callback = vi.fn();
+
+      controller.subscribeLiveness(callback);
+      deliverFrame();
+
+      // Remove video
+      currentVideo = null;
+
+      // Re-arm skipped: no video
+      deliverFrame();
+
+      // The first frame reported live; the stall timer then fires with no
+      // video, so no stall is ever reported.
+      vi.advanceTimersByTime(STALL_MS);
+      expect(callback).not.toHaveBeenCalledWith(false);
+    });
+
+    it('should cancel nothing on unsubscribe when the video is already gone', () => {
+      const { video, cancel } = createVideo();
+      let currentVideo: HTMLVideoElement | null = video;
+      const controller = new VideoMediaPlayerController(
+        createLitElement(),
+        () => currentVideo,
+      );
+
+      const unsubscribe = controller.subscribeLiveness(vi.fn());
+      currentVideo = null;
+      unsubscribe();
+
+      expect(cancel).not.toHaveBeenCalled();
+    });
+  });
 });