fix: Give media that is still loading a grace period before rebuilding (#2739)

- Related: #2718

tests/card-controller/issues/issue-manager.test.ts

@@ -841,8 +841,8 @@ describe('IssueManager', () => {
     });
   });
 
-  describe('in-flight retry', () => {
-    it('should hold the backoff and not arm a timer while a retry is in flight', () => {
+  describe('when an issue cannot retry yet', () => {
+    it('should not retry while one is in flight without exponentially backing off', () => {
       vi.spyOn(Math, 'random').mockReturnValue(0.5);
       const api = createCardAPI();
       const config = createConfig();
@@ -872,8 +872,8 @@ describe('IssueManager', () => {
       expect(issue.retry).toHaveBeenCalledTimes(1);
 
       // The attempt is now in flight: the problem is still unresolved
-      // (needsRetry) but cannot be retried right now (canRetryNow). The running
-      // timer is canceled and no further attempt fires, however long we wait.
+      // (needsRetry) but cannot be retried right now (canRetryNow). No further
+      // attempt fires, however long we wait.
       canRetryNow.mockReturnValue(false);
       manager.evaluate();
       vi.advanceTimersByTime(RETRY_EXPONENTIAL_MAX_SECONDS * 1000);
@@ -890,6 +890,39 @@ describe('IssueManager', () => {
       vi.advanceTimersByTime(RETRY_EXPONENTIAL_BASE_SECONDS * 2 * 0.75 * 1000);
       expect(issue.retry).toHaveBeenCalledTimes(2);
     });
+
+    it('should retry on its own once the issue becomes retryable', () => {
+      const api = createCardAPI();
+      const config = createConfig();
+      vi.mocked(api.getConfigManager().getConfig).mockReturnValue({
+        ...config,
+        view: {
+          ...config.view,
+          issues: { interaction_mode: 'all', retry_seconds: 'auto' },
+        },
+      });
+      const manager = new IssueManager(api);
+
+      const canRetryNow = vi.fn().mockReturnValue(false);
+      const issue = createIssue('media_unavailable', {
+        hasIssue: vi.fn().mockReturnValue(true),
+        needsRetry: vi.fn().mockReturnValue(true),
+        canRetryNow,
+        retry: vi.fn().mockReturnValue(false),
+      });
+      manager.addIssue(issue);
+
+      manager.evaluate();
+
+      vi.advanceTimersByTime(RETRY_EXPONENTIAL_MAX_SECONDS * 1000);
+      expect(issue.retry).not.toHaveBeenCalled();
+
+      canRetryNow.mockReturnValue(true);
+
+      // Deliberately no evaluate() here: nothing on the card has changed.
+      vi.advanceTimersByTime(RETRY_EXPONENTIAL_BASE_SECONDS * 1000);
+      expect(issue.retry).toHaveBeenCalledTimes(1);
+    });
   });
 
   describe('reset', () => {

tests/card-controller/issues/issues/media-unavailable.browser.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, assert, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { RETRY_EXPONENTIAL_BASE_SECONDS } from '../../../../src/card-controller/issues/issue-manager';
+import { MEDIA_UNAVAILABLE_REASONS } from '../../../../src/card-controller/issues/issues/media-unavailable';
 import { LIVENESS_ENTITY_UNAVAILABLE_GRACE_SECONDS } from '../../../../src/components-lib/live/liveness/detectors/entity-availability';
 import { MEDIA_LOADING_TIMEOUT_SECONDS } from '../../../../src/components-lib/media-load-watchdog-controller';
 import { FRAME_STALL_SECONDS } from '../../../../src/components-lib/media-player/frame-stall-watchdog';
@@ -437,6 +438,31 @@ describe('MediaUnavailableIssue', () => {
     expect(isIssueReported(card)).toBe(true);
   });
 
+  it('should wait out the grace period before rebuilding a slow camera', async () => {
+    const retrySeconds = 5;
+    const mediaURL = createUnansweredMediaURL();
+    const card = await mountCard({
+      view: { issues: { retry_seconds: retrySeconds } },
+      cameras: [createStillImageCameraConfig(CAMERA_ENTITY, mediaURL)],
+    });
+
+    await card.waitForSelector('img');
+    await card.advanceSeconds(MEDIA_LOADING_TIMEOUT_SECONDS);
+    expect(isIssueReported(card)).toBe(true);
+
+    // Several retries fall due during the grace period, but rebuilding would
+    // discard a load that may still complete.
+    await card.advanceSeconds(
+      MEDIA_UNAVAILABLE_REASONS.not_loading.rebuildGraceSeconds - 1,
+    );
+    expect(getTestMediaRequestCount(mediaURL)).toBe(1);
+
+    // Past the grace period the attempt has had long enough, and the next retry
+    // replaces it.
+    await card.advanceSeconds(retrySeconds + 1);
+    expect(getTestMediaRequestCount(mediaURL)).toBeGreaterThan(1);
+  });
+
   it('should keep retrying a camera that is still broken', async () => {
     // Must use the real clock: fake time moves the card's timers instantly, so
     // a request would never get a chance to answer between one retry and the
@@ -470,12 +496,14 @@ describe('MediaUnavailableIssue', () => {
     expect(isLiveMediaShowing(card.card)).toBe(true);
 
     // Exactly the two attempts that failed, so the retry ran once rather than
-    // spinning until something happened to work.
+    // spinning until something happened to work. A refused request is reported
+    // as a server error rather than a slow load, which is what allows it to be
+    // retried without waiting.
     expect(
       card.events
         .getEntries('advanced-camera-card:issue:trigger')
         .map((entry) => getIssueReason(entry.detail)),
-    ).toEqual(['not_loading', 'not_loading']);
+    ).toEqual(['server_error', 'server_error']);
   });
 
   it('should re-attempt when the retry control is used', async () => {

tests/card-controller/issues/issues/media-unavailable.test.ts

@@ -1,7 +1,10 @@
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import type { CardController } from '../../../../src/card-controller/controller';
-import { MediaUnavailableIssue } from '../../../../src/card-controller/issues/issues/media-unavailable';
+import {
+  MEDIA_UNAVAILABLE_REASONS,
+  MediaUnavailableIssue,
+} from '../../../../src/card-controller/issues/issues/media-unavailable';
 import type { InternalCallbackActionConfig } from '../../../../src/config/schema/actions/custom/internal';
 import { IMAGE_VIEW_TARGET_ID_SENTINEL } from '../../../../src/view/target-id';
 import type { View } from '../../../../src/view/view';
@@ -13,6 +16,9 @@ import {
 import { createCardAPI } from '../../../test-utils';
 import { createView } from '../../../view/test-utils';
 
+// Read from the policy itself rather than restated here.
+const LOADING_GRACE_SECONDS = MEDIA_UNAVAILABLE_REASONS.not_loading.rebuildGraceSeconds;
+
 const createAPIWithView = (view: View | null): CardController => {
   const api = createCardAPI();
   vi.mocked(api.getViewManager().getView).mockReturnValue(view);
@@ -339,7 +345,7 @@ describe('MediaUnavailableIssue', () => {
       ['entity_unavailable' as const, 'Camera entity unavailable', 'mdi:cctv-off'],
       ['not_loading' as const, 'Media not loading', 'mdi:progress-helper'],
       ['playback_error' as const, 'Playback error', 'mdi:alert-circle'],
-      ['server_error' as const, 'Streaming server error', 'mdi:server-network-off'],
+      ['server_error' as const, 'Media server error', 'mdi:server-network-off'],
       ['stalled' as const, 'Stream stalled', 'mdi:motion-pause'],
       ['unsupported' as const, 'Stream not supported', 'mdi:video-off-outline'],
     ])('should give the %s cause its own text and icon', (reason, text, icon) => {
@@ -485,6 +491,154 @@ describe('MediaUnavailableIssue', () => {
     });
   });
 
+  describe('rebuilding a load that has not arrived', () => {
+    beforeEach(() => {
+      vi.useFakeTimers();
+    });
+
+    afterEach(() => {
+      vi.useRealTimers();
+    });
+
+    it('should not rebuild media that is still loading', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 1000 - 1);
+
+      expect(issue.retry()).toBe(false);
+      expect(api.getViewManager().setViewWithMergedContext).not.toHaveBeenCalled();
+    });
+
+    it('should rebuild media that has still not arrived once the grace period has passed', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 1000);
+
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-1': 1 },
+      });
+    });
+
+    it('should rebuild media that has failed at once', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'stalled' });
+
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-1': 1 },
+      });
+    });
+
+    it('should rebuild media that is still loading when the user asks', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+
+      issue.retry(true);
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-1': 1 },
+      });
+    });
+
+    it('should only rebuild media that is ready for a retry', () => {
+      const api = createAPIDisplaying('camera-1', 'camera-2');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      issue.trigger({ targetID: 'camera-2', reason: 'stalled' });
+
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-2': 1 },
+      });
+    });
+
+    it('should not let repeated reports of one failure postpone the rebuild', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 0.5 * 1000);
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 0.5 * 1000);
+
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-1': 1 },
+      });
+    });
+
+    it('should rebuild at once when media that was still loading is reported as failed', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      issue.trigger({ targetID: 'camera-1', reason: 'stalled' });
+
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).toHaveBeenCalledWith({
+        mediaEpoch: { 'camera-1': 1 },
+      });
+    });
+
+    it('should not allow a retry while everything on screen is still loading', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+
+      expect(issue.canRetryNow()).toBe(false);
+
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 1000);
+
+      expect(issue.canRetryNow()).toBe(true);
+    });
+
+    it('should allow a retry when any media has failed', () => {
+      const api = createAPIDisplaying('camera-1', 'camera-2');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      issue.trigger({ targetID: 'camera-2', reason: 'stalled' });
+
+      expect(issue.canRetryNow()).toBe(true);
+    });
+
+    it('should give a rebuilt load a fresh grace period', () => {
+      const api = createAPIDisplaying('camera-1');
+      const issue = new MediaUnavailableIssue(api);
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 1000);
+      issue.retry();
+
+      vi.mocked(api.getViewManager().setViewWithMergedContext).mockClear();
+
+      issue.trigger({ targetID: 'camera-1', reason: 'not_loading' });
+      vi.advanceTimersByTime(LOADING_GRACE_SECONDS * 1000 - 1);
+      issue.retry();
+
+      expect(api.getViewManager().setViewWithMergedContext).not.toHaveBeenCalled();
+    });
+  });
+
   describe('reset', () => {
     it('should forget every failure', () => {
       const issue = new MediaUnavailableIssue(createAPIDisplaying('camera-1'));