fix: Stop the live media frame collapsing while a stream loads (#2671)

- Closes: #2574

src/components-lib/live/liveness/stream-liveness-controller.ts

@@ -55,7 +55,7 @@ export type LivenessVerdict =
 // nothing. Modeled as the failure rather than a `live` flag because a stream
 // that is merely still connecting is not a failure yet is not playing either, so
 // a positive `live` boolean would misleadingly read as "media is playing".
-interface StreamFailure {
+export interface StreamFailure {
   reason: MediaUnavailableIssueReason;
 
   // Free text naming the specific failure, when known.

src/components-lib/media-dimensions-container-controller.ts

@@ -3,7 +3,11 @@ import { debounce } from 'lodash-es';
 
 import type { CameraDimensionsConfig } from '../config/schema/cameras';
 import type { MediaLoadedInfoEventDetail } from '../types';
-import { aspectRatioToString, setOrRemoveStyleProperty } from '../utils/basic';
+import {
+  aspectRatioToString,
+  isValidAspectRatio,
+  setOrRemoveStyleProperty,
+} from '../utils/basic';
 import { updateElementStyleFromMediaLayoutConfig } from '../utils/media-layout';
 
 const ROTATED_ATTRIBUTE = 'rotated';
@@ -130,7 +134,7 @@ export class MediaDimensionsContainerController implements ReactiveController {
   }
 
   private _hasFixedAspectRatio(): boolean {
-    return this._dimensionsConfig?.aspect_ratio?.length === 2;
+    return isValidAspectRatio(this._dimensionsConfig?.aspect_ratio);
   }
 
   private _requiresRotation(): boolean {

src/components/live/provider.ts

@@ -16,7 +16,10 @@ import type { StateWatcherSubscriptionInterface } from '../../card-controller/ha
 import { MEDIA_UNAVAILABLE_REASONS } from '../../card-controller/issues/issues/media-unavailable.js';
 import { LazyLoadController } from '../../components-lib/lazy-load-controller.js';
 import { isAudioIntendedOnLoad } from '../../components-lib/live/audio-intent.js';
-import { StreamLivenessController } from '../../components-lib/live/liveness/stream-liveness-controller.js';
+import {
+  StreamLivenessController,
+  type StreamFailure,
+} from '../../components-lib/live/liveness/stream-liveness-controller.js';
 import { MediaLoadWatchdogController } from '../../components-lib/media-load-watchdog-controller.js';
 import { MediaLoadedInfoSinkController } from '../../components-lib/media-loaded-info-sink-controller.js';
 import type { PartialZoomSettings } from '../../components-lib/zoom/types.js';
@@ -26,10 +29,13 @@ import type { HomeAssistant } from '../../ha/types.js';
 import { localize } from '../../localize/localize.js';
 import liveProviderStyle from '../../scss/live-provider.scss?inline';
 import type {
+  MediaLoadedInfoEventDetail,
   MediaPlayer,
   MediaPlayerController,
   MediaPlayerElement,
 } from '../../types.js';
+import { onAbort } from '../../utils/abort-signal.js';
+import { isValidAspectRatio } from '../../utils/basic.js';
 import { fireAdvancedCameraCardEvent } from '../../utils/fire-advanced-camera-card-event.js';
 import { getResolvedLiveProvider } from '../../utils/live-provider.js';
 
@@ -107,6 +113,9 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
   @state()
   private _zoomed = false;
 
+  @state()
+  private _loadingImageLoaded = false;
+
   private _refProvider: Ref<MediaPlayerElement> = createRef();
 
   private _lazyLoadController: LazyLoadController = new LazyLoadController(this);
@@ -182,6 +191,7 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
 
     if (changedProps.has('camera')) {
       this._streamLivenessController.reset();
+      this._loadingImageLoaded = false;
 
       const provider = getResolvedLiveProvider(this.camera?.getConfig());
       if (provider === 'jsmpeg') {
@@ -277,24 +287,13 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
       : intermediateTemplate}`;
   }
 
-  protected render(): TemplateResult | void {
-    const cameraConfig = this.camera?.getConfig();
-    if (
-      !this._shouldLoad() ||
-      !this.hass ||
-      !this.liveConfig ||
-      !this.camera ||
-      !cameraConfig
-    ) {
-      return;
-    }
-
+  private _getNotification(failure: StreamFailure | null): TemplateResult | null {
     // If a card *re*-initializes (e.g. was already initialized and then there's
     // a use of the editor to change the config), cameras will re-initialize in
     // place, which means they might be asked to render (here) whilst not yet
     // being initialized. This can cause spurious errors (e.g. lack of resolved
     // endpoints). Instead, simply never render uninitialized cameras.
-    if (!this.camera.isInitialized()) {
+    if (!this.camera?.isInitialized()) {
       return renderMediaNotification({
         icon: 'mdi:progress-helper',
         title: localize('error.awaiting_live'),
@@ -302,12 +301,6 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
       });
     }
 
-    // Set title and ariaLabel from the provided label property.
-    this.title = this.cameraTitle ?? '';
-    this.ariaLabel = this.cameraTitle ?? '';
-
-    const provider = getResolvedLiveProvider(this.camera?.getConfig());
-
     const configurationError = this._getConfigurationError();
     if (configurationError) {
       return renderMediaNotification({
@@ -318,8 +311,6 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
       });
     }
 
-    const failure = this._streamLivenessController.getFailure();
-
     // A detector reports the stream is silently lost (the camera entity is
     // unavailable, or the stream stalled): render a reconnecting placeholder,
     // which unmounts the provider and unloads it via the existing media-loaded
@@ -334,21 +325,60 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
       });
     }
 
-    const showImageDuringLoading = this._shouldShowImageDuringLoading();
-    const mediaLoaded = this._mediaLoadedInfoSinkController.has();
+    return null;
+  }
 
-    // Loaded media or a snapshot gives the frame a size; mark the host `sized`
-    // when one is present. In its absence CSS reserves an aspect ratio so the
-    // frame (whose loading/error fill is absolutely positioned) doesn't
-    // collapse.
-    this.toggleAttribute('sized', mediaLoaded || showImageDuringLoading);
+  protected render(): TemplateResult | void {
+    const cameraConfig = this.camera?.getConfig();
+    if (
+      !this._shouldLoad() ||
+      !this.hass ||
+      !this.liveConfig ||
+      !this.camera ||
+      !cameraConfig
+    ) {
+      // Nothing is drawn at all, so nothing is filling the frame.
+      this.toggleAttribute('sized', false);
+      return;
+    }
+
+    // Set title and ariaLabel from the provided label property.
+    this.title = this.cameraTitle ?? '';
+    this.ariaLabel = this.cameraTitle ?? '';
+
+    const failure = this._streamLivenessController.getFailure();
+    const notification = this._getNotification(failure);
+
+    const shouldShowImageDuringLoading = this._shouldShowImageDuringLoading();
+    const mediaLoaded = this._mediaLoadedInfoSinkController.has();
+    const loadingImageLoaded = shouldShowImageDuringLoading && this._loadingImageLoaded;
+
+    // Mark the host `sized` when something gives the frame a size: loaded
+    // media, a loaded snapshot ("loading image"), or a camera aspect ratio that
+    // the media dimensions container applies to the media itself. A
+    // notification takes the place of the media, so it supplies no size at all.
+    // Absent a size CSS reserves a default aspect ratio so the frame (whose
+    // loading/error fill is absolutely positioned) doesn't collapse.
+    this.toggleAttribute(
+      'sized',
+      !notification &&
+        (mediaLoaded ||
+          loadingImageLoaded ||
+          isValidAspectRatio(cameraConfig.dimensions?.aspect_ratio)),
+    );
+
+    if (notification) {
+      return notification;
+    }
+
+    const provider = getResolvedLiveProvider(cameraConfig);
 
     const classes = {
-      hidden: showImageDuringLoading,
+      hidden: shouldShowImageDuringLoading,
     };
 
     return html`${this._renderContainer(html`
-      ${showImageDuringLoading || provider === 'image'
+      ${shouldShowImageDuringLoading || provider === 'image'
         ? html` <advanced-camera-card-live-image
             ${ref(this._refProvider)}
             .hass=${this.hass}
@@ -361,13 +391,19 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
               // so it should not be hidden.
               hidden: false,
             })}
-            @advanced-camera-card:media:loaded=${(ev: Event) => {
+            @advanced-camera-card:media:loaded=${(
+              ev: CustomEvent<MediaLoadedInfoEventDetail>,
+            ) => {
               // When the image is rendered as a placeholder behind another
               // provider, suppress its load event so it doesn't reach the
               // card-root listener and clobber the real provider's
               // registration. The real provider's load event will arrive
               // afterwards.
               if (provider !== 'image') {
+                this._loadingImageLoaded = true;
+
+                // Should the image unload, the provider returns to unsized.
+                onAbort(ev.detail.signal, () => (this._loadingImageLoaded = false));
                 ev.stopPropagation();
               }
             }}
@@ -438,7 +474,9 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP
                   </advanced-camera-card-live-jsmpeg>`
                 : html``}
     `)}
-    ${failure || mediaLoaded ? '' : this._renderLoadingOverlay(showImageDuringLoading)}`;
+    ${failure || mediaLoaded
+      ? ''
+      : this._renderLoadingOverlay(shouldShowImageDuringLoading)}`;
   }
 
   // The loading status drawn on top of the mounted provider while its media has

src/scss/live-provider.scss

@@ -4,10 +4,9 @@
   position: relative;
 }
 
-// Until loaded media or a snapshot sizes the frame (the `sized` attribute),
-// nothing gives it a size, so reserve a default ratio to stop it collapsing.
-// Real media is never letterboxed into this ratio: the attribute is set the
-// moment it sizes the frame.
+// Until something sizes the frame (the `sized` attribute), reserve a default
+// ratio to stop it collapsing. Media is never letterboxed into this ratio: the
+// attribute is set the moment anything else sizes the frame.
 :host(:not([sized])) {
   aspect-ratio: 16 / 9;
 }

src/utils/basic.ts

@@ -283,12 +283,17 @@ export const recursivelyMergeObjectsConcatenatingArraysUniquely = <T>(
   );
 };
 
+export const isValidAspectRatio = (ratio?: number[] | null): ratio is number[] =>
+  ratio?.length === 2 &&
+  ratio.every((dimension) => Number.isFinite(dimension) && dimension > 0);
+
 export const aspectRatioToString = (options?: {
   ratio?: number[] | null;
   defaultStatic?: boolean;
 }): string => {
-  if (options?.ratio && options.ratio.length === 2) {
-    return `${options.ratio[0]} / ${options.ratio[1]}`;
+  const ratio = options?.ratio;
+  if (isValidAspectRatio(ratio)) {
+    return `${ratio[0]} / ${ratio[1]}`;
   } else if (options?.defaultStatic) {
     return '16 / 9';
   } else {

tests/components/image-updating-player.browser.test.ts

@@ -1,3 +1,4 @@
+import type { LitElement } from 'lit';
 import { describe, expect, it } from 'vitest';
 
 import type { MediaLoadedInfoEventDetail } from '../../src/types';
@@ -19,10 +20,6 @@ const IMAGE_ERROR_EVENT = 'advanced-camera-card:image-updating-player:error';
 const FIXTURE_WIDTH = 320;
 const FIXTURE_HEIGHT = 180;
 
-interface RenderedElement extends Element {
-  updateComplete: Promise<boolean>;
-}
-
 const mount = async (): Promise<MountedCard> => {
   const hass = createGenericCameraHASS({ entities: { [UNRELATED_ENTITY]: 'off' } });
   return await MountedCardFactory.createFromSource(createStillImageCardConfig(), hass);
@@ -38,7 +35,7 @@ describe('AdvancedCameraCardImageUpdatingPlayer', () => {
   it('should announce the media load once even when the card re-renders', async () => {
     const card = await mount();
 
-    const player = await card.waitForSelector<RenderedElement>(
+    const player = await card.waitForSelector<LitElement>(
       'advanced-camera-card-image-updating-player',
     );
     await card.events.waitForFirst('advanced-camera-card:media:loaded');

tests/components/live/provider.browser.test.ts

@@ -0,0 +1,181 @@
+import type { LitElement } from 'lit';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { LIVENESS_ENTITY_UNAVAILABLE_GRACE_SECONDS } from '../../../src/components-lib/live/liveness/detectors/entity-availability';
+import type { RawAdvancedCameraCardConfig } from '../../../src/config/types';
+import { aspectRatioToString } from '../../../src/utils/basic';
+import { deepQuery } from '../../browser/dom';
+import { createFixtureURL, SNAPSHOT_FIXTURE_FILENAME } from '../../browser/fixtures';
+import { MountedCardFactory, type MountedCard } from '../../browser/mounted-card';
+import { createUnansweredMediaURL, useTestMedia } from '../../browser/test-media';
+import {
+  CAMERA_ENTITY,
+  createGenericCameraHASS,
+  createStillImageCardConfig,
+} from '../../browser/test-utils';
+
+useTestMedia();
+
+const IMAGE_URL = createFixtureURL(SNAPSHOT_FIXTURE_FILENAME);
+
+const DEFAULT_RATIO = [16, 9];
+const DEFAULT_RATIO_STYLE = aspectRatioToString({ ratio: DEFAULT_RATIO });
+
+const PORTRAIT_RATIO = [9, 16];
+const PORTRAIT_RATIO_STYLE = aspectRatioToString({ ratio: PORTRAIT_RATIO });
+
+const mount = async (
+  camera: RawAdvancedCameraCardConfig,
+  live?: RawAdvancedCameraCardConfig,
+): Promise<MountedCard> =>
+  await MountedCardFactory.createFromSource(
+    createStillImageCardConfig({
+      cameras: [{ camera_entity: CAMERA_ENTITY, ...camera }],
+      live: { show_image_during_load: false, ...live },
+    }),
+    createGenericCameraHASS(),
+  );
+
+// The provider writes `sized` during its own render, so its render is what a
+// test has to wait for rather than that of the card that drew it.
+const getProvider = async (card: MountedCard): Promise<LitElement> => {
+  const provider = await card.waitForSelector<LitElement>(
+    'advanced-camera-card-live-provider',
+  );
+  await provider.updateComplete;
+  return provider;
+};
+
+const getReservedAspectRatio = (provider: Element): string =>
+  getComputedStyle(provider).aspectRatio;
+
+const getMeasuredAspectRatio = (element: Element): number => {
+  const { width, height } = element.getBoundingClientRect();
+  return width / height;
+};
+
+// A browser lays a box out in fractional pixels, so a measurement never divides
+// back to exactly the ratio it was given.
+const expectMeasuredAspectRatio = (element: Element, ratio: number[]): void =>
+  expect(getMeasuredAspectRatio(element)).toBeCloseTo(ratio[0] / ratio[1], 2);
+
+const waitForMeasuredAspectRatio = async (
+  card: MountedCard,
+  ratio: number[],
+): Promise<void> => {
+  await card.waitForRender(
+    () => {
+      try {
+        expectMeasuredAspectRatio(card.card, ratio);
+        return true;
+      } catch {
+        return null;
+      }
+    },
+    `the card being measured as ${aspectRatioToString({ ratio })}`,
+  );
+};
+
+describe('AdvancedCameraCardLiveProvider', () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  describe('should reserve an aspect ratio until the frame has a size', () => {
+    it('should set default absent other information', async () => {
+      const card = await mount({
+        live_provider: 'image',
+        image: { mode: 'url', url: createUnansweredMediaURL(), refresh_seconds: 0 },
+      });
+
+      const provider = await getProvider(card);
+
+      expect(provider.hasAttribute('sized')).toBe(false);
+      expect(getReservedAspectRatio(provider)).toBe(DEFAULT_RATIO_STYLE);
+
+      // The reservation is what keeps the card off its 100px minimum height.
+      expectMeasuredAspectRatio(card.card, DEFAULT_RATIO);
+    });
+
+    it('should reserve an aspect ratio when the placeholder snapshot is allowed but has not loaded', async () => {
+      const card = await mount(
+        {
+          live_provider: 'image',
+          image: { mode: 'url', url: createUnansweredMediaURL(), refresh_seconds: 0 },
+        },
+        { show_image_during_load: true },
+      );
+
+      const provider = await getProvider(card);
+
+      // The image loading snapshot is on screen and is what would size the
+      // frame, so its absence here is the image never being decoded rather than
+      // never being asked for.
+      expect(deepQuery(provider, 'advanced-camera-card-live-image')).not.toBeNull();
+      expect(provider.hasAttribute('sized')).toBe(false);
+      expect(getReservedAspectRatio(provider)).toBe(DEFAULT_RATIO_STYLE);
+      expectMeasuredAspectRatio(card.card, DEFAULT_RATIO);
+    });
+
+    it('should size to the snapshot once it has loaded', async () => {
+      const card = await mount(
+        {
+          live_provider: 'image',
+          image: { mode: 'url', url: IMAGE_URL, refresh_seconds: 0 },
+        },
+        { show_image_during_load: true },
+      );
+
+      const provider = await getProvider(card);
+
+      await card.waitForRender(
+        () => provider.hasAttribute('sized') || null,
+        'the frame being sized by the loaded snapshot',
+      );
+
+      expect(getReservedAspectRatio(provider)).toBe('auto');
+    });
+
+    it('should size to a configured camera aspect ratio', async () => {
+      const card = await mount({
+        live_provider: 'image',
+        image: { mode: 'url', url: createUnansweredMediaURL(), refresh_seconds: 0 },
+        dimensions: { aspect_ratio: PORTRAIT_RATIO_STYLE },
+      });
+
+      const provider = await getProvider(card);
+
+      // The camera declares the shape of its media, so the media itself is
+      // sized and must not be letterboxed into the reserved ratio.
+      expect(provider.hasAttribute('sized')).toBe(true);
+      expect(getReservedAspectRatio(provider)).toBe('auto');
+      await waitForMeasuredAspectRatio(card, PORTRAIT_RATIO);
+    });
+
+    it('should reserve an aspect ratio again once a camera goes unavailable', async () => {
+      const card = await mount({
+        live_provider: 'image',
+        image: { mode: 'url', url: IMAGE_URL, refresh_seconds: 0 },
+        dimensions: { aspect_ratio: PORTRAIT_RATIO_STYLE },
+      });
+
+      const provider = await getProvider(card);
+      await waitForMeasuredAspectRatio(card, PORTRAIT_RATIO);
+      expect(provider.hasAttribute('sized')).toBe(true);
+
+      vi.useFakeTimers();
+
+      // This triggers a notification to be rendered in the default aspect
+      // ratio.
+      card.setEntityState(CAMERA_ENTITY, 'unavailable');
+      await card.advanceSeconds(LIVENESS_ENTITY_UNAVAILABLE_GRACE_SECONDS);
+
+      await card.waitForRender(
+        () => !provider.hasAttribute('sized') || null,
+        'the frame giving up its size',
+      );
+      expect(getReservedAspectRatio(provider)).toBe(DEFAULT_RATIO_STYLE);
+      expectMeasuredAspectRatio(card.card, DEFAULT_RATIO);
+    });
+  });
+});

tests/utils/basic.test.ts

@@ -440,6 +440,12 @@ describe('aspectRatioToStyle', () => {
   it('invalid ratio', () => {
     expect(aspectRatioToStyle({ ratio: [4] })).toEqual({ 'aspect-ratio': 'auto' });
   });
+  it.each([[[0, 0]], [[-4, 3]], [[Infinity, 3]]])(
+    'should ignore a ratio with a side that describes no shape: %s',
+    (ratio: number[]) => {
+      expect(aspectRatioToStyle({ ratio })).toEqual({ 'aspect-ratio': 'auto' });
+    },
+  );
 });
 
 describe('desparsifyArrays', () => {