Merge pull request #72 from dermotduffy/jsmpeg

Support live JSMPEG view [requires integration upgrade]

README.md

@@ -60,6 +60,7 @@ lovelace:
 | Option           | Default | Description                                         |
 | ------------- | --------------------------------------------- | - |
 | `frigate_camera_name` | The string after the "camera." in the `camera_entity` option (above). | This parameter allows the camera name heuristic to be overriden for cases where the entity name does not cleanly map to the Frigate camera name (e.g. when the Frigate camera name is capitalized, but the entity name is lower case). This camera name is used for communicating with the Frigate backend, e.g. for fetching events. |
+| `live_provider` | `frigate` | Whether `frigate` (the default Frigate camera in Home Assistant which uses an RTMP stream), `frigate-jsmpeg` (JSMPEG stream proxied from the Frigate backend) or `webrtc` should provide the live camera view. See [note below on the required integration version](#jsmpeg-troubleshooting) for `frigate-jsmpeg` to function.|
 | `view_default` | `live` | The view to show by default. See [views](#views) below.|
 | `frigate_client_id` | `frigate` | The Frigate client id to use. If this Home Assistant server has multiple Frigate server backends configured, this selects which server should be used. It should be set to the MQTT client id configured for this server, see [Frigate Integration Multiple Instance Support](https://blakeblackshear.github.io/frigate/usage/home-assistant/#multiple-instance-support).|
 | `view_timeout` | | A numbers of seconds of inactivity after which the card will reset to the default configured view. Inactivity is defined as lack of interaction with the Frigate menu.|
@@ -93,7 +94,6 @@ events/snapshots/UI. A perfect combination!
 
 | Option           | Default | Description                                         |
 | ------------- | - | -------------------------------------------- |
-| `live_provider` | | Whether `frigate` or `webrtc` should provide the live camera view.|
 | `webrtc.entity` | | The RTSP entity to use with WebRTC.|
 | `webrtc.*`| | Any other options in a `webrtc:` YAML dictionary are silently passed through to WebRTC. See [WebRTC Configuration](https://github.com/AlexxIT/WebRTC#configuration) for full details this external card provides.|
 
@@ -242,6 +242,16 @@ This card supports full editing via the Lovelace card editor. Additional arbitra
 
 ## Troubleshooting
 
+<a name="jsmpeg-troubleshooting"></a>
+
+### JSMPEG live camera only shows a 'spinner'
+
+**Note:** As of 2021-09-12, no released version of the [Frigate
+integration](https://github.com/blakeblackshear/frigate-hass-integration)
+supports JSMPEG proxying. That functionality is already merged, and will be in
+the release *after* (not including) v2.0.0. The `frigate-jsmpeg` live provider
+will not work with earlier integration versions.
+
 ### Failed to fetch
 
 **Note:** This error should no longer be possible >= v0.1.5 .

package.json

@@ -14,6 +14,7 @@
   "author": "Dermot Duffy <dermot.duffy@gmail.com>",
   "license": "MIT",
   "dependencies": {
+    "@cycjimmy/jsmpeg-player": "^5.0.1",
     "@material/image-list": "^12.0.0",
     "custom-card-helpers": "^1.7.2",
     "dayjs": "^1.10.6",

src/editor.ts

@@ -126,6 +126,7 @@ export class FrigateCardEditor extends LitElement implements LovelaceCardEditor
     const liveProvider = {
       '': '',
       frigate: localize('live_provider.frigate'),
+      'frigate-jsmpeg': localize('live_provider.frigate-jsmpeg'),
       webrtc: localize('live_provider.webrtc'),
     };
 
@@ -182,6 +183,24 @@ export class FrigateCardEditor extends LitElement implements LovelaceCardEditor
                 .configValue=${'frigate_camera_name'}
                 @value-changed=${this._valueChanged}
               ></paper-input>
+              <paper-dropdown-menu
+                .label=${localize('editor.live_provider')}
+                @value-changed=${this._valueChanged}
+                .configValue=${'live_provider'}
+              >
+                <paper-listbox
+                  slot="dropdown-content"
+                  .selected=${Object.keys(liveProvider).indexOf(
+                    this._config?.live_provider || '',
+                  )}
+                >
+                  ${Object.keys(liveProvider).map((key) => {
+                    return html`
+                      <paper-item .label="${key}">${liveProvider[key]} </paper-item>
+                    `;
+                  })}
+                </paper-listbox>
+              </paper-dropdown-menu>
               <paper-dropdown-menu
                 label=${localize('editor.default_view')}
                 @value-changed=${this._valueChanged}
@@ -368,24 +387,6 @@ export class FrigateCardEditor extends LitElement implements LovelaceCardEditor
         </div>
         ${options.webrtc.show
           ? html` <div class="values">
-              <paper-dropdown-menu
-                .label=${localize('editor.live_provider')}
-                @value-changed=${this._valueChanged}
-                .configValue=${'live_provider'}
-              >
-                <paper-listbox
-                  slot="dropdown-content"
-                  .selected=${Object.keys(liveProvider).indexOf(
-                    this._config?.live_provider || '',
-                  )}
-                >
-                  ${Object.keys(liveProvider).map((key) => {
-                    return html`
-                      <paper-item .label="${key}">${liveProvider[key]} </paper-item>
-                    `;
-                  })}
-                </paper-listbox>
-              </paper-dropdown-menu>
               <paper-dropdown-menu
                 .label=${localize('webrtc.entity')}
                 @value-changed=${this._valueChanged}

src/frigate-hass-card.ts

@@ -29,10 +29,12 @@ import {
   browseMediaSourceSchema,
   frigateCardConfigSchema,
   resolvedMediaSchema,
+  signedPathSchema,
 } from './types';
 import type {
   BrowseMediaNeighbors,
   BrowseMediaSource,
+  ExtendedHomeAssistant,
   FrigateCardConfig,
   FrigateCardView,
   FrigateMenuMode,
@@ -46,6 +48,8 @@ import dayjs_custom_parse_format from 'dayjs/plugin/customParseFormat';
 import { ZodSchema, z } from 'zod';
 import { MessageBase } from 'home-assistant-js-websocket';
 
+import JSMpeg from '@cycjimmy/jsmpeg-player';
+
 const URL_TROUBLESHOOTING =
   'https://github.com/dermotduffy/frigate-hass-card#troubleshooting';
 
@@ -257,7 +261,7 @@ export class FrigateCard extends LitElement {
   public static getStubConfig(): Record<string, string> {
     return {};
   }
-  set hass(hass: HomeAssistant) {
+  set hass(hass: HomeAssistant & ExtendedHomeAssistant) {
     if (this._webrtcElement) {
       this._webrtcElement.hass = hass;
     }
@@ -266,12 +270,14 @@ export class FrigateCard extends LitElement {
   }
 
   @property({ attribute: false })
-  protected _hass: HomeAssistant | null = null;
+  protected _hass: (HomeAssistant & ExtendedHomeAssistant) | null = null;
 
   @state()
   public config!: FrigateCardConfig;
 
   protected _interactionTimerID: number | null = null;
+  protected _jsmpegCanvasElement: any | null = null;
+  protected _jsmpegPlayer: any | null = null;
   protected _webrtcElement: any | null = null;
 
   @property({ attribute: false })
@@ -406,6 +412,7 @@ export class FrigateCard extends LitElement {
     } else {
       this._view = view;
     }
+    this._resetJSMPEGIfNecessary();
   }
 
   // Determine whether the card should be updated.
@@ -620,7 +627,7 @@ export class FrigateCard extends LitElement {
 
   // Render a progress spinner while content loads.
   protected _renderProgressIndicator(): TemplateResult {
-    return html` <div class="frigate-card-attention">
+    return html` <div class="attention">
       <ha-circular-progress active="true" size="large"></ha-circular-progress>
     </div>`;
   }
@@ -947,10 +954,80 @@ export class FrigateCard extends LitElement {
     return null;
   }
 
+  protected async _getJSMPEGURL(): Promise<string | null> {
+    if (!this._hass) {
+      return null;
+    }
+
+    const request = {
+      type: 'auth/sign_path',
+      path:
+        `/api/frigate/${this.config.frigate_client_id}` +
+        `/jsmpeg/${this.config.frigate_camera_name}`,
+    };
+    // Sign the path so it includes an authSig parameter.
+    let response;
+    try {
+      response = await this._makeWSRequest(signedPathSchema, request);
+    } catch (err) {
+      console.warn(err);
+      return null;
+    }
+    const url = this._hass.hassUrl(response.path);
+    return url.replace(/^http/i, 'ws');
+  }
+
+  protected _resetJSMPEGIfNecessary(): void {
+    if (!this._view.is('live') || this.config.live_provider != 'frigate-jsmpeg') {
+      if (this._jsmpegPlayer) {
+        this._jsmpegPlayer.destroy();
+        this._jsmpegPlayer = null;
+      }
+      this._jsmpegCanvasElement = null;
+    }
+  }
+
+  // Cleanup and/or start the JSMPEG player.
+  protected async _renderJSMPEGPlayer(): Promise<TemplateResult> {
+    if (!this._jsmpegCanvasElement) {
+      this._jsmpegCanvasElement = document.createElement('canvas');
+      this._jsmpegCanvasElement.className = 'media';
+    }
+
+    if (!this._jsmpegPlayer) {
+      const jsmpeg_url = await this._getJSMPEGURL();
+
+      if (!jsmpeg_url) {
+        return this._renderError('Could not retrieve or sign JSMPEG websocket path');
+      }
+
+      // Return the html canvas node only after the JSMPEG video has loaded and
+      // is playing, to reduce the amount of time the user is staring at a blank
+      // white canvas (instead they get the progress spinner until this promise
+      // resolves).
+      return new Promise<TemplateResult>((resolve) => {
+        this._jsmpegPlayer = new JSMpeg.VideoElement(
+          this,
+          jsmpeg_url,
+          {
+            canvas: this._jsmpegCanvasElement,
+            hooks: {
+              play: () => {
+                resolve(html`${this._jsmpegCanvasElement}`);
+              },
+            },
+          },
+          { protocols: [], videoBufferSize: 1024 * 1024 * 4 },
+        );
+      });
+    }
+    return html`${this._jsmpegCanvasElement}`;
+  }
+
   // Render the live viewer.
   // Note: The live viewer is the main element used to size the overall card. It
   // is always rendered (but sometimes hidden).
-  protected _renderLiveViewer(): TemplateResult {
+  protected async _renderLiveViewer(): Promise<TemplateResult> {
     if (!this._hass || !(this.config.camera_entity in this._hass.states)) {
       return this._renderAttentionIcon(
         'mdi:camera-off',
@@ -960,6 +1037,9 @@ export class FrigateCard extends LitElement {
     if (this._webrtcElement) {
       return html`${this._webrtcElement}`;
     }
+    if (this.config.live_provider == 'frigate-jsmpeg') {
+      return await this._renderJSMPEGPlayer();
+    }
     return html` <ha-camera-stream
       .hass=${this._hass}
       .stateObj=${this._hass.states[this.config.camera_entity]}
@@ -1016,7 +1096,9 @@ export class FrigateCard extends LitElement {
           ${this._view.is('clip') || this._view.is('snapshot')
             ? until(this._renderViewer(), this._renderProgressIndicator())
             : ``}
-          ${this._view.is('live') ? this._renderLiveViewer() : ``}
+          ${this._view.is('live')
+            ? until(this._renderLiveViewer(), this._renderProgressIndicator())
+            : ``}
         </div>
       </div>
       ${this.config.menu_mode != 'above' ? this._renderMenu() : ''}

src/localize/languages/en.json

@@ -68,6 +68,7 @@
   },
   "live_provider": {
     "frigate": "Frigate",
+    "frigate-jsmpeg": "Frigate JSMpeg",
     "webrtc": "WebRTC"
   },
   "webrtc": {

src/scss/card.scss

@@ -34,7 +34,7 @@
   opacity: 1.0;
 }
 
-.frigate-card-contents img.media,video.media {
+.frigate-card-contents img.media,video.media,canvas.media {
   width: 100%;
 }
 
@@ -49,7 +49,7 @@
   justify-content: center;
   align-items: center;
   box-sizing: border-box;
-  padding: 5%;
+  padding: 10%;
 }
 
 .frigate-card-image-list {
@@ -83,11 +83,16 @@ ha-card {
   width: 100%;
   height: 100%;
   position: relative;
+  color: var(--secondary-text-color, white);
   background-color: var(--secondary-background-color, black);
 
   transform-style: preserve-3d; /* Safari brings video elements forward without this */
 }
 
+ha-card a {
+  color: var(--primary-text-color, white);
+}
+
 /* Don't drop shadow or have radius for nested webrtc card */
 webrtc-camera ha-card {
   box-shadow: none;

src/types.ts

@@ -60,7 +60,7 @@ export const frigateCardConfigSchema = z.object({
         .transform((val) => Number(val)),
     )
     .optional().default(180),
-  live_provider: z.enum(['frigate', 'webrtc']).default('frigate'),
+  live_provider: z.enum(['frigate', 'frigate-jsmpeg', 'webrtc']).default('frigate'),
   webrtc: z.object({}).passthrough().optional(),
   label: z.string().optional(),
   zone: z.string().optional(),
@@ -96,6 +96,10 @@ export interface MenuButton {
   emphasize?: boolean;
 }
 
+export interface ExtendedHomeAssistant {
+  hassUrl(path?): string;
+}
+
 /**
  * Media Browser API types.
  */
@@ -144,3 +148,8 @@ export interface BrowseMediaNeighbors {
   next: BrowseMediaSource | null;
   nextIndex: number | null;
 }
+
+export const signedPathSchema = z.object({
+  path: z.string(),
+});
+export type SignedPath = z.infer<typeof signedPathSchema>;