From 36f14e153c53b91f9132c17933dcffa9bd4fda3a Mon Sep 17 00:00:00 2001 From: Dermot Duffy Date: Sun, 8 Jan 2023 19:32:13 -0800 Subject: [PATCH] Add segment garbage collection. --- src/camera/cache.ts | 29 +++++++++ src/camera/frigate/engine-frigate.ts | 88 ++++++++++++++++++++++++++-- src/camera/frigate/types.ts | 7 +-- src/view/view.ts | 10 ++-- 4 files changed, 118 insertions(+), 16 deletions(-) diff --git a/src/camera/cache.ts b/src/camera/cache.ts index 9679d31b..a0d997e7 100644 --- a/src/camera/cache.ts +++ b/src/camera/cache.ts @@ -103,6 +103,20 @@ export class MemoryRangedCache { } return output; } + + public size(): number { + return this._data.length; + } + + /** + * Remove old data that matches a given predicate. No change to the covered + * ranges is made, i.e. this is asserting authoritiatively that this data does + * not exist in the current ranges. + * @param predicate A predicate to run on each data element. + */ + public expireMatches(predicate: (data: Data) => boolean): void { + this._data = this._data.filter(predicate); + } } export class RecordingSegmentsCache { @@ -128,4 +142,19 @@ export class RecordingSegmentsCache { public get(cameraID: string, range: DateRange): RecordingSegment[] | null { return this._segments.get(cameraID)?.get(range) ?? null; } + + public getCache(cameraID: string): MemoryRangedCache | null { + return this._segments.get(cameraID) ?? null; + } + + public getCameraIDs(): string[] { + return [...this._segments.keys()]; + } + + public expireMatches( + cameraID: string, + func: (segment: RecordingSegment) => boolean, + ): void { + this._segments.get(cameraID)?.expireMatches(func); + } } diff --git a/src/camera/frigate/engine-frigate.ts b/src/camera/frigate/engine-frigate.ts index cb78ed42..04552145 100644 --- a/src/camera/frigate/engine-frigate.ts +++ b/src/camera/frigate/engine-frigate.ts @@ -1,10 +1,9 @@ import { HomeAssistant } from 'custom-card-helpers'; import add from 'date-fns/add'; import endOfHour from 'date-fns/endOfHour'; -import getUnixTime from 'date-fns/getUnixTime'; import startOfHour from 'date-fns/startOfHour'; import { CAMERA_BIRDSEYE } from '../../const'; -import { CameraConfig } from '../../types'; +import { CameraConfig, RecordingSegment } from '../../types'; import { MediaQueriesResults } from '../../view/media-queries-results'; import { MediaQueriesClassifier } from '../../view/media-queries-classifier'; import { ViewMedia, ViewMediaFactory } from '../../view/media'; @@ -42,6 +41,10 @@ import { } from './requests'; import { MediaQueries } from '../../view/media-queries'; import orderBy from 'lodash-es/orderBy'; +import throttle from 'lodash-es/throttle'; +import { runWhenIdleIfSupported } from '../../utils/basic'; +import { fromUnixTime } from 'date-fns'; +import { sum } from 'lodash-es'; const EVENT_REQUEST_CACHE_MAX_AGE_SECONDS = 60; const RECORDING_SUMMARY_REQUEST_CACHE_MAX_AGE_SECONDS = 60; @@ -74,6 +77,13 @@ class FrigateQueryResultsClassifier { export class FrigateCameraManagerEngine implements CameraManagerEngine { protected _recordingSegmentsCache: RecordingSegmentsCache; + // Garbage collect segments at most once an hour. + protected _throttledSegmentGarbageCollector = throttle( + this._garbageCollectSegments.bind(this), + 60 * 60 * 1000, + { trailing: true }, + ); + constructor(recordingSegmentsCache: RecordingSegmentsCache) { this._recordingSegmentsCache = recordingSegmentsCache; } @@ -220,9 +230,9 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { (!query.end || endHour <= query.end) ) { recordings.push({ - camera: cameraConfig.frigate.camera_name, - start_time: getUnixTime(startHour), - end_time: getUnixTime(endHour), + cameraID: query.cameraID, + startTime: startHour, + endTime: endHour, events: hourData.events, }); } @@ -234,7 +244,7 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { // this simulates it. recordings = orderBy( recordings, - (recording: FrigateRecording) => recording.start_time, + (recording: FrigateRecording) => recording.startTime, 'desc', ).slice(0, query.limit); } @@ -287,6 +297,8 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { const segments = await getRecordingSegments(hass, request); this._recordingSegmentsCache.add(query.cameraID, range, segments); + runWhenIdleIfSupported(() => this._throttledSegmentGarbageCollector(hass, cameras)); + return { type: QueryResultsType.RecordingSegments, engine: Engine.Frigate, @@ -381,4 +393,68 @@ export class FrigateCameraManagerEngine implements CameraManagerEngine { } return cameraConfig; } + + /** + * Garbage collect recording segments that no longer feature in the recordings + * returned by the Frigate backend. + */ + protected async _garbageCollectSegments( + hass: HomeAssistant, + cameras: Map, + ): Promise { + const cameraIDs = this._recordingSegmentsCache.getCameraIDs(); + const recordingQueries: RecordingQuery[] = cameraIDs.map((cameraID) => ({ + cameraID: cameraID, + type: QueryType.Recording, + })); + + const countSegments = () => + sum( + cameraIDs.map( + (cameraID) => this._recordingSegmentsCache.getCache(cameraID)?.size() ?? 0, + ), + ); + const segmentsStart = countSegments(); + + const results: Map = new Map(); + + await Promise.all( + recordingQueries.map((query) => + (async () => { + const recordings = await this.getRecordings(hass, cameras, query); + if (recordings && recordings.engine === Engine.Frigate) { + results.set(query, recordings as FrigateRecordingQueryResults); + } + })(), + ), + ); + + // Performance: _recordingSegments is potentially very large (e.g. 10K - 1M + // items) and each item must be examined, so care required here to stick to + // nothing worse than O(n) performance. + const getHourID = (cameraID: string, startTime: Date): string => { + return `${cameraID}/${startTime.getDate()}/${startTime.getHours()}`; + }; + + for (const [query, result] of results) { + const goodHours: Set = new Set(); + for (const recording of result.recordings) { + goodHours.add(getHourID(recording.cameraID, recording.startTime)); + } + + this._recordingSegmentsCache.expireMatches( + query.cameraID, + (segment: RecordingSegment) => { + const hourID = getHourID(query.cameraID, fromUnixTime(segment.start_time)); + // ~O(1) lookup time for a JS set. + return goodHours.has(hourID); + }, + ); + } + + console.debug( + 'Frigate Card recording segment garbage collection: ' + + `Released ${segmentsStart - countSegments()} segment(s)`, + ); + } } diff --git a/src/camera/frigate/types.ts b/src/camera/frigate/types.ts index 26608e77..81135e91 100644 --- a/src/camera/frigate/types.ts +++ b/src/camera/frigate/types.ts @@ -50,9 +50,8 @@ export const retainResultSchema = z.object({ export type RetainResult = z.infer; export interface FrigateRecording { - // Frigate camera name (may not be unique) - camera: string; - start_time: number; - end_time: number; + cameraID: string; + startTime: Date; + endTime: Date; events: number; } \ No newline at end of file diff --git a/src/view/view.ts b/src/view/view.ts index 10e41f5d..f358aad2 100644 --- a/src/view/view.ts +++ b/src/view/view.ts @@ -1,7 +1,7 @@ // Medium: -// - TODO: Add garbage collecting of segments not present in the recording summaries anymore. // - TODO: Move frigate specific view-media under the camera manager. // - TODO: ts-prune https://camchenry.com/blog/deleting-dead-code-in-typescript +// - TODO: Use sortedUniqBy in the engine sort/uniq combo. // Hard: // - TODO: Implement dragging the timeline seeking forward in both Frigate recordings & events. @@ -85,8 +85,9 @@ export class View { !curr || prev.view !== curr.view || prev.camera !== curr.camera || - // When in the live view, the target contains the events that happened in - // the past -- not reflective of the actual live media viewer. + // When in the live view, the queryResults contain the events that + // happened in the past -- not reflective of the actual live media viewer + // the user is seeing. (curr.view !== 'live' && (prev.queryResults !== curr.queryResults || prev.queryResults?.getSelectedResult() !== @@ -103,9 +104,6 @@ export class View { camera: this.camera, query: this.query?.clone() ?? null, queryResults: this.queryResults?.clone() ?? null, - // target: this.target, - // targetIndex: this.targetIndex, - // targetFingerprint: this.targetFingerprint, context: this.context, }); }