feat: Add support for Frigate reviews / detections [initial PR] (#2315)

- Add support for Frigate reviews / detections.
 - Add support for GenAI metadata.
- Significant internal refactor to more flexible "UnifiedQuery" to allow
mixing cameras with simple metadata and review metadata (e.g. a timeline
view of a Frigate camera with reviews, and a Reolink camera with simple
metadata).
 - Add support for folder media as camera media.

There are a few more PRs to commit prior to this going live, but
commiting this for now due to the scale of the change.

BREAKING CHANGE: `media_type` and `events_type` are retired under
`live`, `viewer` and `timeline` configuration sections, instead media
type is associated (once) with the camera under `media`.
This commit is contained in:
Dermot Duffy
2026-01-19 13:32:50 -08:00
committed by GitHub
parent 6eb0a87ca8
commit fc32727860
215 changed files with 14491 additions and 6160 deletions
+6
View File
@@ -54,6 +54,12 @@ export class ViewItemManager {
}
}
public async reviewMedia(item: ViewItem, reviewed: boolean): Promise<void> {
if (ViewItemClassifier.isReview(item)) {
return await this._api.getCameraManager().reviewMedia(item, reviewed);
}
}
private _getMediaSource(item: ViewItem): ViewMediaSource | null {
if (ViewItemClassifier.isMedia(item) && item.getCameraID()) {
return ViewMediaSource.Camera;
@@ -1,13 +1,16 @@
import { Query } from '../../../view/query';
import { QueryResults } from '../../../view/query-results';
import { UnifiedQuery } from '../../../view/unified-query';
import { View } from '../../../view/view';
import { ViewModifier } from '../types';
export class SetQueryViewModifier implements ViewModifier {
protected _query?: Query | null;
protected _query?: UnifiedQuery | null;
protected _queryResults?: QueryResults | null;
constructor(options?: { query?: Query | null; queryResults?: QueryResults | null }) {
constructor(options?: {
query?: UnifiedQuery | null;
queryResults?: QueryResults | null;
}) {
this._query = options?.query;
this._queryResults = options?.queryResults;
}
-185
View File
@@ -1,185 +0,0 @@
import { CapabilitySearchKeys, MediaQuery } from '../../camera-manager/types';
import { MEDIA_CHUNK_SIZE_DEFAULT } from '../../const';
import { ClipsOrSnapshotsOrAll } from '../../types';
import { findBestMediaTimeIndex } from '../../utils/find-best-media-time-index';
import { ViewItem } from '../../view/item';
import {
EventMediaQuery,
FolderViewQuery,
MediaQueries,
Query,
RecordingMediaQuery,
} from '../../view/query';
import { QueryClassifier } from '../../view/query-classifier';
import { QueryResults } from '../../view/query-results';
import { CardViewAPI } from '../types';
import { QueryExecutorOptions, QueryExecutorResult } from './types';
export class QueryExecutor {
protected _api: CardViewAPI;
constructor(api: CardViewAPI) {
this._api = api;
}
public async executeDefaultEventQuery(options?: {
cameraID?: string;
eventsMediaType?: ClipsOrSnapshotsOrAll;
executorOptions?: QueryExecutorOptions;
}): Promise<QueryExecutorResult | null> {
const capabilitySearch: CapabilitySearchKeys =
!options?.eventsMediaType || options?.eventsMediaType === 'all'
? {
anyCapabilities: ['clips', 'snapshots'],
}
: options.eventsMediaType;
const cameraManager = this._api.getCameraManager();
const cameraIDs = options?.cameraID
? cameraManager
.getStore()
.getAllDependentCameras(options.cameraID, capabilitySearch)
: cameraManager.getStore().getCameraIDsWithCapability(capabilitySearch);
if (!cameraIDs.size) {
return null;
}
const rawQueries = cameraManager.generateDefaultEventQueries(cameraIDs, {
limit: this._getChunkLimit(),
...(options?.eventsMediaType === 'clips' && { hasClip: true }),
...(options?.eventsMediaType === 'snapshots' && { hasSnapshot: true }),
});
if (!rawQueries) {
return null;
}
const queries = new EventMediaQuery(rawQueries);
return await this.executeMediaQuery(queries, options?.executorOptions);
}
public async executeDefaultRecordingQuery(options?: {
cameraID?: string;
executorOptions?: QueryExecutorOptions;
}): Promise<QueryExecutorResult | null> {
const cameraManager = this._api.getCameraManager();
const cameraIDs = options?.cameraID
? cameraManager.getStore().getAllDependentCameras(options.cameraID, 'recordings')
: cameraManager.getStore().getCameraIDsWithCapability('recordings');
if (!cameraIDs.size) {
return null;
}
const rawQueries = cameraManager.generateDefaultRecordingQueries(cameraIDs, {
limit: this._getChunkLimit(),
});
if (!rawQueries) {
return null;
}
const queries = new RecordingMediaQuery(rawQueries);
return await this.executeMediaQuery(queries, options?.executorOptions);
}
public async executeQuery(
query: Query,
executorOptions?: QueryExecutorOptions,
): Promise<QueryExecutorResult | null> {
/* istanbul ignore else: this path cannot be reached -- @preserve */
if (QueryClassifier.isMediaQuery(query)) {
return await this.executeMediaQuery(query, executorOptions);
} else if (QueryClassifier.isFolderQuery(query)) {
return await this._executeFolderQuery(query, executorOptions);
}
/* istanbul ignore next: this path cannot be reached -- @preserve */
return null;
}
public async executeMediaQuery(
query: MediaQueries,
executorOptions?: QueryExecutorOptions,
): Promise<QueryExecutorResult | null> {
const queries = query.getQuery();
if (!queries) {
return null;
}
const mediaArray = await this._api
.getCameraManager()
.executeMediaQueries<MediaQuery>(queries, {
useCache: executorOptions?.useCache,
});
const queryResults = mediaArray
? this._generateQueriesResults(mediaArray, executorOptions)
: null;
return queryResults ? { query, queryResults } : null;
}
private _generateQueriesResults(
itemArray: ViewItem[],
executorOptions?: QueryExecutorOptions,
): QueryResults | null {
const queryResults = new QueryResults({ results: itemArray });
if (executorOptions?.rejectResults?.(queryResults)) {
return null;
}
if (executorOptions?.selectResult?.id) {
queryResults.selectBestResult((media) =>
media.findIndex((m) => m.getID() === executorOptions.selectResult?.id),
);
} else if (executorOptions?.selectResult?.func) {
queryResults.selectResultIfFound(executorOptions.selectResult.func);
} else if (executorOptions?.selectResult?.time) {
queryResults.selectBestResult((itemArray) =>
findBestMediaTimeIndex(
itemArray,
executorOptions.selectResult?.time?.time as Date,
executorOptions.selectResult?.time?.favorCameraID,
),
);
}
return queryResults;
}
public async executeFolderQuery(
executorOptions?: QueryExecutorOptions,
): Promise<QueryExecutorResult | null> {
const folder = this._api.getFoldersManager().getFolder(executorOptions?.folder);
if (!folder) {
return null;
}
const query = this._api.getFoldersManager().generateDefaultFolderQuery(folder);
if (!query) {
return null;
}
return this._executeFolderQuery(new FolderViewQuery(query), executorOptions);
}
private async _executeFolderQuery(
query: FolderViewQuery,
executorOptions?: QueryExecutorOptions,
): Promise<QueryExecutorResult | null> {
const rawQuery = query.getQuery();
if (!rawQuery) {
return null;
}
const itemArray = await this._api
.getFoldersManager()
.expandFolder(rawQuery, this._api.getConditionStateManager().getState(), {
useCache: executorOptions?.useCache,
});
const queryResults = itemArray
? this._generateQueriesResults(itemArray, executorOptions)
: null;
return queryResults ? { query, queryResults } : null;
}
protected _getChunkLimit(): number {
const cardWideConfig = this._api.getConfigManager().getCardWideConfig();
return (
cardWideConfig?.performance?.features.media_chunk_size ?? MEDIA_CHUNK_SIZE_DEFAULT
);
}
}
+1 -6
View File
@@ -2,7 +2,7 @@ import { ViewContext } from 'view';
import { AdvancedCameraCardError } from '../../types.js';
import { ViewItem } from '../../view/item.js';
import { QueryResults } from '../../view/query-results.js';
import { Query } from '../../view/query.js';
import { View, ViewParameters } from '../../view/view.js';
export interface ViewModifier {
@@ -26,11 +26,6 @@ export interface QueryExecutorOptions {
useCache?: boolean;
}
export interface QueryExecutorResult {
query: Query;
queryResults: QueryResults;
}
export interface ViewFactoryOptions {
// An existing view to evolve from.
baseView?: View | null;
+10 -11
View File
@@ -1,7 +1,6 @@
import { ViewContext } from 'view';
import { log } from '../../utils/debug';
import { getStreamCameraID } from '../../utils/substream';
import { QueryClassifier } from '../../view/query-classifier';
import { View } from '../../view/view';
import { InitializationAspect } from '../initialization-manager';
import { CardViewAPI } from '../types';
@@ -234,19 +233,19 @@ export class ViewManager implements ViewManagerInterface {
// If the user is currently using the viewer, and then switches to the
// gallery we make an attempt to keep the query/queryResults the same so
// the gallery can be used to click back and forth to the viewer, and the
// selected media can be centered in the gallery. See the matching code in
// `updated()` in `gallery.ts`. We specifically must ensure that the new
// target media of the gallery (e.g. clips, snapshots or recordings) is
// equal to the queries that are currently used in the viewer.
// selected media can be centered in the gallery.
//
// See: https://github.com/dermotduffy/advanced-camera-card/issues/885
const switchingFromViewerToGallery =
this._view?.isViewerView() && newView?.isMediaGalleryView();
const newMediaType = newView?.getDefaultMediaType();
const alreadyHasMatchingQuery =
QueryClassifier.getMediaType(this._view?.query) === newMediaType;
return !!switchingFromViewerToGallery && alreadyHasMatchingQuery;
if (!this._view?.isViewerView() || !newView?.isGalleryView()) {
return false;
}
// Check if the viewer came from this gallery type. If they did, we preserve
// the query/results to ensure consistency in navigation between media &
// gallery.
const originView = this._view?.context?.gallery?.originView;
return originView === newView.view;
}
public setViewWithMergedContext(context: ViewContext | null): void {
+149 -55
View File
@@ -1,11 +1,15 @@
import { sub } from 'date-fns';
import { ClipsOrSnapshotsOrAll } from '../../types';
import { MEDIA_CHUNK_SIZE_DEFAULT } from '../../const';
import { findBestMediaTimeIndex } from '../../utils/find-best-media-time-index';
import { QueryResults } from '../../view/query-results';
import { UnifiedQuery } from '../../view/unified-query';
import { MediaTypeSpec, UnifiedQueryBuilder } from '../../view/unified-query-builder';
import { UnifiedQueryRunner } from '../../view/unified-query-runner';
import { View } from '../../view/view';
import { CardViewAPI } from '../types';
import { MergeContextViewModifier } from './modifiers/merge-context';
import { RemoveContextPropertyViewModifier } from './modifiers/remove-context-property';
import { SetQueryViewModifier } from './modifiers/set-query';
import { QueryExecutor } from './query-executor';
import { QueryExecutorOptions, ViewModifier } from './types';
/**
@@ -14,27 +18,42 @@ import { QueryExecutorOptions, ViewModifier } from './types';
* and if a query is made as part of this view the result can be applied later.
*/
export class ViewQueryExecutor {
protected _api: CardViewAPI;
protected _executor: QueryExecutor;
private _api: CardViewAPI;
constructor(api: CardViewAPI, executor?: QueryExecutor) {
constructor(api: CardViewAPI) {
this._api = api;
this._executor = executor ?? new QueryExecutor(api);
}
public async getExistingQueryModifiers(
view: View,
queryExecutorOptions?: QueryExecutorOptions,
): Promise<ViewModifier[] | null> {
return view.query
if (!view.query) {
return null;
}
const runner = new UnifiedQueryRunner(
this._api.getCameraManager(),
this._api.getFoldersManager(),
this._api.getConditionStateManager(),
);
const items = await runner.execute(view.query, {
useCache: queryExecutorOptions?.useCache,
});
const queryResults = this._applyResultSelection(
new QueryResults({ results: items }),
queryExecutorOptions,
);
return queryResults
? [
new SetQueryViewModifier({
queryResults: (
await this._executor.executeQuery(view.query, queryExecutorOptions)
)?.queryResults,
queryResults,
}),
]
: [];
: null;
}
public async getNewQueryModifiers(
@@ -47,7 +66,7 @@ export class ViewQueryExecutor {
});
}
protected async _executeNewQuery(
private async _executeNewQuery(
view: View,
queryExecutorOptions?: QueryExecutorOptions,
): Promise<ViewModifier[] | null> {
@@ -56,71 +75,111 @@ export class ViewQueryExecutor {
return null;
}
const mediaType = view?.getDefaultMediaType();
const viewModifiers: ViewModifier[] = [];
const builder = new UnifiedQueryBuilder(
this._api.getCameraManager(),
this._api.getFoldersManager(),
);
const runner = new UnifiedQueryRunner(
this._api.getCameraManager(),
this._api.getFoldersManager(),
this._api.getConditionStateManager(),
);
const executeMediaQuery = async (
mediaType: ClipsOrSnapshotsOrAll | 'recordings' | null,
): Promise<ViewModifier[]> => {
/* istanbul ignore if: this path cannot be reached -- @preserve */
if (!mediaType) {
const executeQuery = async (query: UnifiedQuery | null): Promise<ViewModifier[]> => {
if (!query) {
return [];
}
const results =
mediaType === 'recordings'
? await this._executor.executeDefaultRecordingQuery({
...(!view.isGrid() && { cameraID: view.camera }),
executorOptions: queryExecutorOptions,
})
: mediaType === 'clips' || mediaType === 'snapshots' || mediaType === 'all'
? await this._executor.executeDefaultEventQuery({
...(!view.isGrid() && { cameraID: view.camera }),
eventsMediaType: mediaType,
executorOptions: queryExecutorOptions,
})
: /* istanbul ignore next -- @preserve */
null;
const items = await runner.execute(query, {
useCache: queryExecutorOptions?.useCache,
});
return results ? [new SetQueryViewModifier(results)] : [];
return [
new SetQueryViewModifier({
query,
queryResults: new QueryResults({ results: items }),
}),
];
};
const executeFolderQuery = async (): Promise<ViewModifier[]> => {
const results = await this._executor.executeFolderQuery(queryExecutorOptions);
return results ? [new SetQueryViewModifier(results)] : [];
};
const cameraForQuery = view.isGrid() ? undefined : view.camera;
switch (view.view) {
case 'live':
if (config.live.controls.thumbnails.mode !== 'none') {
viewModifiers.push(
...(await executeMediaQuery(
config.live.controls.thumbnails.media_type === 'recordings'
? 'recordings'
: config.live.controls.thumbnails.events_media_type,
)),
);
const defaultQuery = builder.buildDefaultCameraQuery(cameraForQuery);
if (defaultQuery) {
viewModifiers.push(...(await executeQuery(defaultQuery)));
}
}
break;
case 'media':
// If the user is looking at media in the `media` view and then
// changes camera (via the menu) it should default to showing clips
// for the new camera.
viewModifiers.push(...(await executeMediaQuery('clips')));
case 'timeline':
// Timeline view always queries all cameras with media capabilities.
viewModifiers.push(...(await executeQuery(builder.buildDefaultCameraQuery())));
break;
case 'media':
// If the user is looking at media in the `media` view and then
// changes camera (via the menu) it should default to showing clips
// for the new camera.
case 'clip':
case 'clips':
viewModifiers.push(
...(await executeQuery(
builder.buildCameraMediaQuery(MediaTypeSpec.clips(), {
cameraID: cameraForQuery,
limit: this._getLimit(),
}),
)),
);
break;
case 'snapshot':
case 'snapshots':
viewModifiers.push(
...(await executeQuery(
builder.buildCameraMediaQuery(MediaTypeSpec.snapshots(), {
cameraID: cameraForQuery,
limit: this._getLimit(),
}),
)),
);
break;
case 'recording':
case 'recordings':
viewModifiers.push(...(await executeMediaQuery(mediaType)));
viewModifiers.push(
...(await executeQuery(
builder.buildCameraMediaQuery(MediaTypeSpec.recordings(), {
cameraID: cameraForQuery,
limit: this._getLimit(),
}),
)),
);
break;
case 'review':
case 'reviews':
viewModifiers.push(
...(await executeQuery(
builder.buildCameraMediaQuery(MediaTypeSpec.reviews(), {
cameraID: cameraForQuery,
limit: this._getLimit(),
}),
)),
);
break;
case 'folder':
case 'folders':
viewModifiers.push(...(await executeFolderQuery()));
viewModifiers.push(
...(await executeQuery(
builder.buildDefaultFolderQuery(queryExecutorOptions?.folder, {
limit: this._getLimit(),
}),
)),
);
break;
}
@@ -131,7 +190,7 @@ export class ViewQueryExecutor {
return viewModifiers;
}
protected _getTimelineWindowViewModifier(view: View): ViewModifier[] {
private _getTimelineWindowViewModifier(view: View): ViewModifier[] {
if (view.is('live')) {
// For live views, always force the timeline to now, regardless of
// presence or not of events.
@@ -166,7 +225,7 @@ export class ViewQueryExecutor {
}
}
protected _getSeekTimeModifier(time?: Date): ViewModifier[] {
private _getSeekTimeModifier(time?: Date): ViewModifier[] {
if (time) {
return [
new MergeContextViewModifier({
@@ -179,4 +238,39 @@ export class ViewQueryExecutor {
return [new RemoveContextPropertyViewModifier('mediaViewer', 'seek')];
}
}
private _applyResultSelection(
queryResults: QueryResults,
options?: QueryExecutorOptions,
): QueryResults | null {
if (options?.rejectResults?.(queryResults)) {
return null;
}
const timeSelection = options?.selectResult?.time;
if (options?.selectResult?.id) {
queryResults.selectBestResult((media) =>
media.findIndex((m) => m.getID() === options.selectResult?.id),
);
} else if (options?.selectResult?.func) {
queryResults.selectResultIfFound(options.selectResult.func);
} else if (timeSelection) {
queryResults.selectBestResult((itemArray) =>
findBestMediaTimeIndex(
itemArray,
timeSelection.time,
timeSelection.favorCameraID,
),
);
}
return queryResults;
}
private _getLimit(): number {
return (
this._api.getConfigManager().getConfig()?.performance?.features
?.media_chunk_size ?? MEDIA_CHUNK_SIZE_DEFAULT
);
}
}