From f9ea4355867090e54f0d88420dc74e8225555102 Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Sun, 12 Sep 2021 15:36:41 -0700 Subject: [PATCH 1/3] Support live JSMPEG view. --- README.md | 12 ++++- package.json | 1 + src/editor.ts | 37 +++++++------- src/frigate-hass-card.ts | 88 ++++++++++++++++++++++++++++++++-- src/localize/languages/en.json | 1 + src/scss/card.scss | 9 +++- src/types.ts | 13 ++++- 7 files changed, 133 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 20cf00c8..5151aad6 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ lovelace: | Option | Default | Description | | ------------- | --------------------------------------------- | - | | `frigate_camera_name` | The string after the "camera." in the `camera_entity` option (above). | This parameter allows the camera name heuristic to be overriden for cases where the entity name does not cleanly map to the Frigate camera name (e.g. when the Frigate camera name is capitalized, but the entity name is lower case). This camera name is used for communicating with the Frigate backend, e.g. for fetching events. | +| `live_provider` | `frigate` | Whether `frigate` (the default Frigate camera in Home Assistant which uses an RTMP stream), `frigate-jsmpeg` (JSMPEG stream proxied from the Frigate backend) or `webrtc` should provide the live camera view. See [note below on the required integration version](#jsmpeg-troubleshooting) for `frigate-jsmpeg` to function.| | `view_default` | `live` | The view to show by default. See [views](#views) below.| | `frigate_client_id` | `frigate` | The Frigate client id to use. If this Home Assistant server has multiple Frigate server backends configured, this selects which server should be used. It should be set to the MQTT client id configured for this server, see [Frigate Integration Multiple Instance Support](https://blakeblackshear.github.io/frigate/usage/home-assistant/#multiple-instance-support).| | `view_timeout` | | A numbers of seconds of inactivity after which the card will reset to the default configured view. Inactivity is defined as lack of interaction with the Frigate menu.| @@ -93,7 +94,6 @@ events/snapshots/UI. A perfect combination! | Option | Default | Description | | ------------- | - | -------------------------------------------- | -| `live_provider` | | Whether `frigate` or `webrtc` should provide the live camera view.| | `webrtc.entity` | | The RTSP entity to use with WebRTC.| | `webrtc.*`| | Any other options in a `webrtc:` YAML dictionary are silently passed through to WebRTC. See [WebRTC Configuration](https://github.com/AlexxIT/WebRTC#configuration) for full details this external card provides.| @@ -242,6 +242,16 @@ This card supports full editing via the Lovelace card editor. Additional arbitra ## Troubleshooting + + +### JSMPEG live camera only shows a 'spinner' + +**Note:** As of 2021-09-12, no released version of the [Frigate +integration](https://github.com/blakeblackshear/frigate-hass-integration) +supports JSMPEG proxying. That functionality is already merged, and will be in +the release *after* (not including) v2.0.0. The `frigate-jsmpeg` live provider +will not work with earlier integration versions. + ### Failed to fetch **Note:** This error should no longer be possible >= v0.1.5 . diff --git a/package.json b/package.json index 48bb157a..0313b1fa 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "author": "Dermot Duffy ", "license": "MIT", "dependencies": { + "@cycjimmy/jsmpeg-player": "^5.0.1", "@material/image-list": "^12.0.0", "custom-card-helpers": "^1.7.2", "dayjs": "^1.10.6", diff --git a/src/editor.ts b/src/editor.ts index 79a6a0bd..7cca3ac4 100644 --- a/src/editor.ts +++ b/src/editor.ts @@ -126,6 +126,7 @@ export class FrigateCardEditor extends LitElement implements LovelaceCardEditor const liveProvider = { '': '', frigate: localize('live_provider.frigate'), + 'frigate-jsmpeg': localize('live_provider.frigate-jsmpeg'), webrtc: localize('live_provider.webrtc'), }; @@ -182,6 +183,24 @@ export class FrigateCardEditor extends LitElement implements LovelaceCardEditor .configValue=${'frigate_camera_name'} @value-changed=${this._valueChanged} > + + + ${Object.keys(liveProvider).map((key) => { + return html` + ${liveProvider[key]} + `; + })} + + ${options.webrtc.show ? html`
- - - ${Object.keys(liveProvider).map((key) => { - return html` - ${liveProvider[key]} - `; - })} - - { return {}; } - set hass(hass: HomeAssistant) { + set hass(hass: HomeAssistant & ExtendedHomeAssistant) { if (this._webrtcElement) { this._webrtcElement.hass = hass; } @@ -266,12 +270,14 @@ export class FrigateCard extends LitElement { } @property({ attribute: false }) - protected _hass: HomeAssistant | null = null; + protected _hass: (HomeAssistant & ExtendedHomeAssistant) | null = null; @state() public config!: FrigateCardConfig; protected _interactionTimerID: number | null = null; + protected _jsmpegCanvasElement: any | null = null; + protected _jsmpegPlayer: any | null = null; protected _webrtcElement: any | null = null; @property({ attribute: false }) @@ -406,6 +412,7 @@ export class FrigateCard extends LitElement { } else { this._view = view; } + this._resetJSMPEGIfNecessary(); } // Determine whether the card should be updated. @@ -620,7 +627,7 @@ export class FrigateCard extends LitElement { // Render a progress spinner while content loads. protected _renderProgressIndicator(): TemplateResult { - return html`
+ return html`
`; } @@ -947,10 +954,76 @@ export class FrigateCard extends LitElement { return null; } + protected async _getJSMPEGURL(): Promise { + if (!this._hass) { + return null; + } + + const request = { + type: 'auth/sign_path', + path: + `/api/frigate/${this.config.frigate_client_id}` + + `/jsmpeg/${this.config.frigate_camera_name}`, + }; + // Sign the path so it includes an authSig parameter. + let response; + try { + response = await this._makeWSRequest(signedPathSchema, request); + } catch (err) { + console.warn(err); + return null; + } + const url = this._hass.hassUrl(response.path); + return url.replace(/^http/i, 'ws'); + } + + protected _resetJSMPEGIfNecessary(): void { + if (!this._view.is('live') || this.config.live_provider != 'frigate-jsmpeg') { + if (this._jsmpegPlayer) { + this._jsmpegPlayer.destroy(); + this._jsmpegPlayer = null; + } + this._jsmpegCanvasElement = null; + } + } + + // Cleanup and/or start the JSMPEG player. + protected async _renderJSMPEG(): Promise { + if (!this._jsmpegCanvasElement) { + this._jsmpegCanvasElement = document.createElement('canvas'); + this._jsmpegCanvasElement.className = 'media'; + } + + if (!this._jsmpegPlayer) { + const jsmpeg_url = await this._getJSMPEGURL(); + + if (!jsmpeg_url) { + return this._renderError('Could not retrieve or sign JSMPEG websocket path'); + } + + return new Promise((resolve) => { + this._jsmpegPlayer = new JSMpeg.VideoElement( + this, + jsmpeg_url, + { + canvas: this._jsmpegCanvasElement, + hooks: { + play: () => { + resolve(html`${this._jsmpegCanvasElement}`); + }, + }, + }, + { protocols: [], videoBufferSize: 1024 * 1024 * 4 }, + ); + }); + } + return html`${this._jsmpegCanvasElement}`; + } + // Render the live viewer. // Note: The live viewer is the main element used to size the overall card. It // is always rendered (but sometimes hidden). - protected _renderLiveViewer(): TemplateResult { + protected async _renderLiveViewer(): Promise { if (!this._hass || !(this.config.camera_entity in this._hass.states)) { return this._renderAttentionIcon( 'mdi:camera-off', @@ -960,6 +1033,9 @@ export class FrigateCard extends LitElement { if (this._webrtcElement) { return html`${this._webrtcElement}`; } + if (this.config.live_provider == 'frigate-jsmpeg') { + return await this._renderJSMPEG(); + } return html`
${this.config.menu_mode != 'above' ? this._renderMenu() : ''} diff --git a/src/localize/languages/en.json b/src/localize/languages/en.json index 2ed04b20..10035590 100644 --- a/src/localize/languages/en.json +++ b/src/localize/languages/en.json @@ -68,6 +68,7 @@ }, "live_provider": { "frigate": "Frigate", + "frigate-jsmpeg": "Frigate JSMpeg", "webrtc": "WebRTC" }, "webrtc": { diff --git a/src/scss/card.scss b/src/scss/card.scss index c9a42ec0..2055ea1b 100644 --- a/src/scss/card.scss +++ b/src/scss/card.scss @@ -34,7 +34,7 @@ opacity: 1.0; } -.frigate-card-contents img.media,video.media { +.frigate-card-contents img.media,video.media,canvas.media { width: 100%; } @@ -49,7 +49,7 @@ justify-content: center; align-items: center; box-sizing: border-box; - padding: 5%; + padding: 10%; } .frigate-card-image-list { @@ -83,11 +83,16 @@ ha-card { width: 100%; height: 100%; position: relative; + color: var(--secondary-text-color, white); background-color: var(--secondary-background-color, black); transform-style: preserve-3d; /* Safari brings video elements forward without this */ } +ha-card a { + color: var(--primary-text-color, white); +} + /* Don't drop shadow or have radius for nested webrtc card */ webrtc-camera ha-card { box-shadow: none; diff --git a/src/types.ts b/src/types.ts index 56bf9e1b..9d139e92 100644 --- a/src/types.ts +++ b/src/types.ts @@ -60,7 +60,7 @@ export const frigateCardConfigSchema = z.object({ .transform((val) => Number(val)), ) .optional().default(180), - live_provider: z.enum(['frigate', 'webrtc']).default('frigate'), + live_provider: z.enum(['frigate', 'frigate-jsmpeg', 'webrtc']).default('frigate'), webrtc: z.object({}).passthrough().optional(), label: z.string().optional(), zone: z.string().optional(), @@ -96,6 +96,10 @@ export interface MenuButton { emphasize?: boolean; } +export interface ExtendedHomeAssistant { + hassUrl(path?): string; +} + /** * Media Browser API types. */ @@ -143,4 +147,9 @@ export interface BrowseMediaNeighbors { next: BrowseMediaSource | null; nextIndex: number | null; -} \ No newline at end of file +} + +export const signedPathSchema = z.object({ + path: z.string(), +}); +export type SignedPath = z.infer; From 9a8d0803c89cf73a6f43dc2440dcde6f2e9d4f5f Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Sun, 12 Sep 2021 15:41:20 -0700 Subject: [PATCH 2/3] Add comment explaining odd choice of Promise usage. --- src/frigate-hass-card.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/frigate-hass-card.ts b/src/frigate-hass-card.ts index ec37f360..3f93c49c 100644 --- a/src/frigate-hass-card.ts +++ b/src/frigate-hass-card.ts @@ -1001,6 +1001,10 @@ export class FrigateCard extends LitElement { return this._renderError('Could not retrieve or sign JSMPEG websocket path'); } + // Return the html canvas node only after the JSMPEG video has loaded and + // is playing, to reduce the amount of time the user is staring at a blank + // white canvas (instead they get the progress spinner until this promise + // resolves). return new Promise((resolve) => { this._jsmpegPlayer = new JSMpeg.VideoElement( this, From 04131ec363b33e88339685560bd8ce8be7225985 Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Sun, 12 Sep 2021 15:46:41 -0700 Subject: [PATCH 3/3] Make function name slightly more intuitive. --- src/frigate-hass-card.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frigate-hass-card.ts b/src/frigate-hass-card.ts index 3f93c49c..8a30d99a 100644 --- a/src/frigate-hass-card.ts +++ b/src/frigate-hass-card.ts @@ -988,7 +988,7 @@ export class FrigateCard extends LitElement { } // Cleanup and/or start the JSMPEG player. - protected async _renderJSMPEG(): Promise { + protected async _renderJSMPEGPlayer(): Promise { if (!this._jsmpegCanvasElement) { this._jsmpegCanvasElement = document.createElement('canvas'); this._jsmpegCanvasElement.className = 'media'; @@ -1038,7 +1038,7 @@ export class FrigateCard extends LitElement { return html`${this._webrtcElement}`; } if (this.config.live_provider == 'frigate-jsmpeg') { - return await this._renderJSMPEG(); + return await this._renderJSMPEGPlayer(); } return html`