From 6b2036676c87289043faf70fa794b828283227f0 Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Mon, 9 Jan 2023 21:15:15 -0800 Subject: [PATCH] Move seeking support into the engine. --- src/camera/engine.ts | 7 ++ src/camera/frigate/engine-frigate.ts | 58 +++++++++++++ src/camera/frigate/requests.ts | 1 + src/camera/manager.ts | 22 +++++ src/components/thumbnail-carousel.ts | 3 +- src/components/thumbnail.ts | 19 ++--- src/components/timeline-core.ts | 24 +----- src/components/viewer.ts | 48 ++++++----- src/utils/media-to-view.ts | 123 ++------------------------- src/utils/timeline-source.ts | 6 +- src/view/media.ts | 5 ++ src/view/view.ts | 2 - 12 files changed, 142 insertions(+), 176 deletions(-) diff --git a/src/camera/engine.ts b/src/camera/engine.ts index 21513008..3fef2d24 100644 --- a/src/camera/engine.ts +++ b/src/camera/engine.ts @@ -77,4 +77,11 @@ export interface CameraManagerEngine { queries: MediaQueries, results: MediaQueriesResults, ): boolean; + + getMediaSeekTime( + hass: HomeAssistant, + cameras: Map, + media: ViewMedia, + target: Date, + ): Promise; } diff --git a/src/camera/frigate/engine-frigate.ts b/src/camera/frigate/engine-frigate.ts index 646f4ecd..1ccb0374 100644 --- a/src/camera/frigate/engine-frigate.ts +++ b/src/camera/frigate/engine-frigate.ts @@ -392,6 +392,32 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { ); } + public async getMediaSeekTime( + hass: HomeAssistant, + cameras: Map, + media: ViewMedia, + target: Date, + ): Promise { + const start = media.getStartTime(); + const end = media.getEndTime(); + if (!start || !end || target < start || target > end) { + return null; + } + + const query: RecordingSegmentsQuery = { + cameraID: media.getCameraID(), + start: start, + end: end, + type: QueryType.RecordingSegments, + }; + + const segments = await this.getRecordingSegments(hass, cameras, query); + const out = segments + ? this._getSeekTimeInSegments(start, target, segments.segments) + : null; + return out; + } + protected _getQueryableCameraConfig( cameras: Map, cameraID: string, @@ -466,4 +492,36 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { `Released ${segmentsStart - countSegments()} segment(s)`, ); } + + /** + * Get the number of seconds to seek into a video stream consisting of the + * provided segments to reach the target time provided. + * @param startTime The earliest allowable time to seek from. + * @param targetTime Target time. + * @param segments An array of segments dataset items. Must be sorted from oldest to youngest. + * @returns + */ + protected _getSeekTimeInSegments( + startTime: Date, + targetTime: Date, + segments: RecordingSegment[], + ): number | null { + if (!segments.length) { + return null; + } + let seekMilliseconds = 0; + + // Inspired by: https://github.com/blakeblackshear/frigate/blob/release-0.11.0/web/src/routes/Recording.jsx#L27 + for (const segment of segments) { + const segmentStart = fromUnixTime(segment.start_time); + if (segmentStart > targetTime) { + break; + } + const segmentEnd = fromUnixTime(segment.end_time); + const start = segmentStart < startTime ? startTime : segmentStart; + const end = segmentEnd > targetTime ? targetTime : segmentEnd; + seekMilliseconds += end.getTime() - start.getTime(); + } + return seekMilliseconds / 1000; + } } diff --git a/src/camera/frigate/requests.ts b/src/camera/frigate/requests.ts index 0675e364..4ff5782c 100644 --- a/src/camera/frigate/requests.ts +++ b/src/camera/frigate/requests.ts @@ -26,6 +26,7 @@ export const getRecordingsSummary = async ( type: 'frigate/recordings/summary', instance_id: client_id, camera: camera_name, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, }, true, ); diff --git a/src/camera/manager.ts b/src/camera/manager.ts index c0bcb851..2dc1cb2d 100644 --- a/src/camera/manager.ts +++ b/src/camera/manager.ts @@ -278,6 +278,28 @@ export class CameraManager { return true; } + public async getMediaSeekTime( + hass: HomeAssistant, + media: ViewMedia, + target: Date, + ): Promise { + const startTime = media.getStartTime(); + const endTime = media.getEndTime(); + const cameraConfig = this._cameras.get(media.getCameraID()); + if ( + !cameraConfig || + !startTime || + !endTime || + target < startTime || + target > endTime + ) { + return null; + } + + const engine = this._engineFactory.getEngineForCamera(cameraConfig); + return (await engine?.getMediaSeekTime(hass, this._cameras, media, target)) ?? null; + } + protected async _handleQuery( hass: HomeAssistant, query: QT | QT[], diff --git a/src/components/thumbnail-carousel.ts b/src/components/thumbnail-carousel.ts index d061165e..0f7d75ac 100644 --- a/src/components/thumbnail-carousel.ts +++ b/src/components/thumbnail-carousel.ts @@ -160,6 +160,7 @@ export class FrigateCardThumbnailCarousel extends LitElement { 'slide-selected': this.selected === index, }; + const seekTarget = this.view?.context?.mediaViewer?.seek; return html` ` : ``} - ${this.mediaSeek + ${this.seek ? html`
${localize('event.seek')} - ${format(fromUnixTime(this.mediaSeek.seekTime), 'HH:mm:ss')} + ${format(this.seek, 'HH:mm:ss')}
` : html``} @@ -183,7 +182,7 @@ export class FrigateCardThumbnailDetailsRecording extends LitElement { public media?: RecordingViewMedia; @property({ attribute: false }) - public mediaSeek?: MediaSeek; + public seek?: Date; @property({ attribute: false }) public cameraTitle?: string; @@ -195,10 +194,10 @@ export class FrigateCardThumbnailDetailsRecording extends LitElement { const eventCount = this.media.getEventCount(); return html`
${this.cameraTitle ?? ''}
- ${this.mediaSeek + ${this.seek ? html`
${localize('recording.seek')} - ${format(fromUnixTime(this.mediaSeek.seekTime), 'HH:mm:ss')} + ${format(this.seek, 'HH:mm:ss')}
` : html``}
@@ -242,7 +241,7 @@ export class FrigateCardThumbnail extends LitElement { public show_timeline_control = false; @property({ attribute: false }) - public mediaSeek?: MediaSeek; + public seek?: Date; @property({ attribute: false }) public view?: Readonly; @@ -314,13 +313,13 @@ export class FrigateCardThumbnail extends LitElement { ${this.details && ViewMediaClassifier.isEvent(this.media) ? html`` : this.details && ViewMediaClassifier.isRecording(this.media) ? html`` : html``} ${shouldShowTimelineControl diff --git a/src/components/timeline-core.ts b/src/components/timeline-core.ts index 3e9c3404..71acd690 100644 --- a/src/components/timeline-core.ts +++ b/src/components/timeline-core.ts @@ -48,7 +48,6 @@ import { createViewForEvents, createViewForRecordings, findClosestMediaIndex, - generateMediaViewerContext, } from '../utils/media-to-view'; import { CameraManager } from '../camera/manager'; import { EventMediaQueries, MediaQueries } from '../view/media-queries'; @@ -407,16 +406,6 @@ export class FrigateCardTimelineCore extends LitElement { } const canSeek = !!this.view?.isViewerView(); - - const context = canSeek - ? await generateMediaViewerContext( - this.hass, - this.cameraManager, - media, - targetTime, - ) - : null; - const newResults = this._locked ? null : results @@ -444,7 +433,7 @@ export class FrigateCardTimelineCore extends LitElement { }) // Whether or not to set the timeline window. .mergeInContext({ ...this._generateTimelineContext({ noSetWindow: true }), - ...context, + ...(canSeek && { mediaViewer: { seek: targetTime }}) }) .dispatchChangeEvent(this); } @@ -533,12 +522,7 @@ export class FrigateCardTimelineCore extends LitElement { // view change. if (eventView && results && results.length) { eventView.mergeInContext( - await generateMediaViewerContext( - this.hass, - this.cameraManager, - results, - properties.time, - ), + {mediaViewer: {seek: properties.time}} ); view = eventView; } @@ -608,7 +592,7 @@ export class FrigateCardTimelineCore extends LitElement { } const prefetchedWindow = this._getPrefetchWindow(properties); - await this._timelineSource?.refresh(this.hass, this.cameras, prefetchedWindow); + await this._timelineSource?.refresh(this.hass, prefetchedWindow); // Don't show event thumbnails if the user is looking at recordings, // as the recording "hours" are the media, not the event @@ -918,7 +902,7 @@ export class FrigateCardTimelineCore extends LitElement { // (via fetchIfNecessary) may update the timeline contents which causes // the visjs timeline to stop dragging/panning operations which is very // disruptive to the user. - await this._timelineSource?.refresh(this.hass, this.cameras, prefetchedWindow); + await this._timelineSource?.refresh(this.hass, prefetchedWindow); } const mediaID = media?.getID(); diff --git a/src/components/viewer.ts b/src/components/viewer.ts index 35df33db..33427328 100644 --- a/src/components/viewer.ts +++ b/src/components/viewer.ts @@ -58,17 +58,8 @@ import { guard } from 'lit/directives/guard.js'; import { localize } from '../localize/localize.js'; import { MediaQueriesResults } from '../view/media-queries-results.js'; -export interface MediaSeek { - // Specifies the point at which this recording should be played, the - // seek_time is the date of the desired play point (for display purposes - // usually), and seek_seconds is the number of seconds to seek into the video - // stream to reach that point. - seekTime: number; - seekSeconds: number; -} - export interface MediaViewerViewContext { - seek: Map; + seek?: Date; } declare module 'view' { @@ -260,6 +251,8 @@ export class FrigateCardViewerCarousel extends LitElement { * @param changedProperties The properties that were changed in this render. */ updated(changedProperties: PropertyValues): void { + super.updated(changedProperties); + if (changedProperties.has('view')) { const oldView = changedProperties.get('view') as View | undefined; // Seek into the video if the seek time has changed (this is also called @@ -269,7 +262,6 @@ export class FrigateCardViewerCarousel extends LitElement { this._seekHandler(); } } - super.updated(changedProperties); } /** @@ -418,17 +410,26 @@ export class FrigateCardViewerCarousel extends LitElement { * Handle the user selecting a new slide in the carousel. */ protected _setViewHandler(ev: CustomEvent): void { + // The slide may already be selected on load, so don't dispatch a new view + // unless necessary. if (ev.detail.index !== this.view?.queryResults?.getSelectedIndex()) { this._setViewSelectedIndex(ev.detail.index); } } protected _setViewSelectedIndex(index: number): void { - // The slide may already be selected on load, so don't dispatch a new view - // unless necessary. + const newResults = this.view?.queryResults?.clone().selectResult(index); + if (!newResults) { + return; + } + const cameraID = newResults.getSelectedResult()?.getCameraID(); + this.view ?.evolve({ - queryResults: this.view.queryResults?.clone().selectResult(index), + queryResults: newResults, + + // Always change the camera to the owner of the selected media. + ...(cameraID && { camera: cameraID }), }) // Ensure the timeline is able to update its position. .mergeInContext({ timeline: { noSetWindow: false } }) @@ -626,15 +627,18 @@ export class FrigateCardViewerCarousel extends LitElement { /** * Fire a media show event when a slide is selected. */ - protected _seekHandler(): void { - const selectedIndex = this.view?.queryResults?.getSelectedIndex() ?? null; - const seek = - selectedIndex !== null - ? this.view?.context?.mediaViewer?.seek.get(selectedIndex) - : null; + protected async _seekHandler(): Promise { + const seek = this.view?.context?.mediaViewer?.seek; + const media = this.view?.queryResults?.getSelectedResult(); + if (!this.hass || !media || !seek) { + return; + } + + const seekTime = + (await this.cameraManager?.getMediaSeekTime(this.hass, media, seek)) ?? null; const player = this._getPlayer(); - if (player && seek) { - player.seek(seek.seekSeconds); + if (player && seekTime !== null) { + player.seek(seekTime); } } diff --git a/src/utils/media-to-view.ts b/src/utils/media-to-view.ts index fcba2c4b..137b54e4 100644 --- a/src/utils/media-to-view.ts +++ b/src/utils/media-to-view.ts @@ -1,25 +1,16 @@ import add from 'date-fns/add'; -import fromUnixTime from 'date-fns/fromUnixTime'; -import startOfHour from 'date-fns/startOfHour'; import sub from 'date-fns/sub'; import { ViewContext } from 'view'; -import { - CameraConfig, - ClipsOrSnapshotsOrAll, - FrigateCardView, - RecordingSegment, -} from '../types'; +import { CameraConfig, ClipsOrSnapshotsOrAll, FrigateCardView } from '../types'; import { View } from '../view/view'; import { EventMediaQueries, RecordingMediaQueries } from '../view/media-queries'; import { CameraManager } from '../camera/manager'; import { getAllDependentCameras } from './camera.js'; import { ViewMedia } from '../view/media'; -import { ViewMediaClassifier } from '../view/media-classifier'; import { HomeAssistant } from 'custom-card-helpers'; import { dispatchFrigateCardErrorEvent } from '../components/message'; import { MediaQueriesResults } from '../view/media-queries-results'; import { errorToConsole } from './basic'; -import { RecordingSegmentsQueryResults } from '../camera/types'; export const changeViewToRecentEventsForCameraAndDependents = async ( element: HTMLElement, @@ -168,12 +159,11 @@ export const createViewForRecordings = async ( queryResults.selectBestResult((media) => findClosestMediaIndex(media, options.targetTime as Date, cameraIDs), ); - viewerContext = await generateMediaViewerContext( - hass, - cameraManager, - mediaArray, - options.targetTime, - ); + viewerContext = { + mediaViewer: { + seek: options.targetTime, + }, + }; } return ( @@ -187,75 +177,6 @@ export const createViewForRecordings = async ( ); }; -/** - * Generate the media view context for a set of media children (used to set - * seek times into each media item). - * @param hass The Home Assistant object. - * @param cameraManager The datamanager to use for data access. - * @param media The media. - * @param targetTime The target time. - * @returns The ViewContext. - */ -export const generateMediaViewerContext = async ( - hass: HomeAssistant, - cameraManager: CameraManager, - media: ViewMedia[], - targetTime: Date, -): Promise => { - const seek = new Map(); - const hourStart = startOfHour(targetTime); - - for (const [index, child] of media.entries()) { - const start = child.getStartTime(); - const end = child.getEndTime(); - if (!start || !end) { - continue; - } - - let seekSeconds: number | null = null; - - if (targetTime >= start && targetTime <= end) { - const query = cameraManager.generateDefaultRecordingSegmentsQueries( - child.getCameraID(), - { - start: start, - end: end, - }, - )[0]; - let segments: RecordingSegmentsQueryResults | null; - - try { - segments = (await cameraManager.getRecordingSegments(hass, query)).get( - query, - ) ?? null; - } catch (e) { - errorToConsole(e as Error); - // View context is never critical. Ignore errors which will at least - // allow the video to load even if it doesn't seek to the correct - // location. - return {}; - } - - if (segments) { - seekSeconds = getSeekTimeInSegments( - // Recordings start from the top of the hour. - ViewMediaClassifier.isRecording(child) ? hourStart : start, - targetTime, - segments.segments, - ); - } - } - - if (seekSeconds !== null) { - seek.set(index, { - seekSeconds: seekSeconds, - seekTime: targetTime.getTime() / 1000, - }); - } - } - return seek.size > 0 ? { mediaViewer: { seek: seek } } : {}; -}; - /** * Find the closest matching media object. * @param mediaArray The media. Must be sorted most recent first. @@ -302,35 +223,3 @@ export const findClosestMediaIndex = ( } return bestMatch ? bestMatch.index : null; }; - -/** - * Get the number of seconds to seek into a video stream consisting of the - * provided segments to reach the target time provided. - * @param startTime The earliest allowable time to seek from. - * @param targetTime Target time. - * @param segments An array of segments dataset items. Must be sorted from oldest to youngest. - * @returns - */ -const getSeekTimeInSegments = ( - startTime: Date, - targetTime: Date, - segments: RecordingSegment[], -): number | null => { - if (!segments.length) { - return null; - } - let seekMilliseconds = 0; - - // Inspired by: https://github.com/blakeblackshear/frigate/blob/release-0.11.0/web/src/routes/Recording.jsx#L27 - for (const segment of segments) { - const segmentStart = fromUnixTime(segment.start_time); - if (segmentStart > targetTime) { - break; - } - const segmentEnd = fromUnixTime(segment.end_time); - const start = segmentStart < startTime ? startTime : segmentStart; - const end = segmentEnd > targetTime ? targetTime : segmentEnd; - seekMilliseconds += end.getTime() - start.getTime(); - } - return seekMilliseconds / 1000; -}; diff --git a/src/utils/timeline-source.ts b/src/utils/timeline-source.ts index 7cdfe4f7..fea859b3 100644 --- a/src/utils/timeline-source.ts +++ b/src/utils/timeline-source.ts @@ -87,12 +87,11 @@ export class TimelineDataSource { public async refresh( hass: HomeAssistant, - cameras: Map, window: TimelineWindow, ): Promise { try { await Promise.all([ - this._refreshEvents(hass, cameras, window), + this._refreshEvents(hass, window), this._refreshRecordings(hass, window), ]); } catch (e) { @@ -122,7 +121,6 @@ export class TimelineDataSource { protected async _refreshEvents( hass: HomeAssistant, - cameras: Map, window: TimelineWindow, ): Promise { if ( @@ -145,7 +143,7 @@ export class TimelineDataSource { for (const media of results?.getResults() ?? []) { const endTime = media.getEndTime(); const startTime = media.getStartTime(); - const id = media.getID(cameras.get(media.getCameraID())); + const id = media.getID(); if (id && startTime) { this._dataset.update({ id: id, diff --git a/src/view/media.ts b/src/view/media.ts index ac04a520..a1f198e7 100644 --- a/src/view/media.ts +++ b/src/view/media.ts @@ -38,6 +38,11 @@ export class ViewMedia { public isFavorite(): boolean | null { return null; } + public includesTime(seek: Date): boolean { + const startTime = this.getStartTime(); + const endTime = this.getEndTime(); + return !!startTime && !!endTime && seek >= startTime && seek <= endTime; + } // Sets the favorite attribute (if any). This purely sets the media item as a // favorite in JS. diff --git a/src/view/view.ts b/src/view/view.ts index e8cd9f97..78fe5bd6 100644 --- a/src/view/view.ts +++ b/src/view/view.ts @@ -1,14 +1,12 @@ // Minor / later: // - TODO: ts-prune https://camchenry.com/blog/deleting-dead-code-in-typescript // - TODO: getRecordingTitle should use getCameraTitle but need hass. -// - TODO: Pass timezone to recordings & event summary endpoint. // Hard: // - TODO: Implement dragging the timeline seeking forward in both Frigate recordings & events. // - TODO: Implement gallery. // - TODO: Remove FrigateBrowseMediaSource if not necessary (post-gallery). // - TODO: Remove browse-media.ts TODOs. -// - TODO: In generateMediaViewerContext there is an assumption that recordings start/end on the hour, which is true for Frigate but that assumption should be in the engine. // - TODO: What should the timeline do when an event is clicked on that is not in the queryResults (or if queryResults is empty)? // - TODO: Should the timeline data source clear events (as it currently does) when the query changes?