From 1d44df6e96807d6042411fafff2ebf4cfc82f793 Mon Sep 17 00:00:00 2001 From: Holden Date: Sun, 14 Jun 2026 18:41:37 +0000 Subject: [PATCH] docs: annotate known limitations and Frigate API improvement hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds inline LIMITATION / TODO(frigate-api) comments at each specific code site rather than a separate doc that would drift from the code. frigate_api.py — recognize_face: Mean-embedding limitation: score reflects the arithmetic mean of all training embeddings. A bimodal set (frontals + profiles) has a mean between clusters, making both ends look more novel than they are. Fixable if Frigate exposes per-file embeddings for nearest-neighbour comparison. frigate_api.py — get_all_frigate_person_files: "train" key exclusion is a hardcoded string. If Frigate adds other special top-level keys in /api/faces they'll be silently treated as person names. Needs a typed schema when Frigate documents the contract. executor.py — recognize_face call site: Async rebuild: each deletion triggers a background model rebuild in Frigate. Subsequent recognize calls in the same run return None (rebuild in progress), degrading quality replacement for later candidates. Fixable with a rebuild-complete signal from Frigate. executor.py — effective_count / manual file handling: Manually-added files are invisible to diversity decisions. Winnow observes their effect only indirectly via the Frigate score, not by measuring their embedding distribution. Per-file embeddings from Frigate would allow direct diversity measurement against the full set. executor.py — Frigate version assumption: All face training endpoints are v0.16+. No version check at startup; failures on older versions are opaque 404s. cache.py — MODEL_VERSIONS: Version string is a hardcoded constant. Manual model file replacement (custom weights, InsightFace update) won't invalidate cached embeddings. Needs file-checksum-derived versioning or a CLEAR_EMBEDDING_CACHE flag. diversity.py — thumbnail-resolution embeddings: Diversity selection runs InsightFace on preview thumbnails; the actual training crop comes from full-resolution originals. Negligible in practice but degrades if Immich preview quality is low. --- winnow/cache.py | 8 +++++++- winnow/diversity.py | 7 +++++++ winnow/executor.py | 21 +++++++++++++++++++++ winnow/frigate_api.py | 14 ++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/winnow/cache.py b/winnow/cache.py index 828139f..d554401 100644 --- a/winnow/cache.py +++ b/winnow/cache.py @@ -12,7 +12,13 @@ import numpy as np logger = logging.getLogger(__name__) -# Model versions — bump these when the upstream model changes +# Model versions — bump these when the upstream model changes. +# LIMITATION — no automatic invalidation: if the user replaces the buffalo_l +# model files on disk (e.g. custom weights, InsightFace update) without +# changing INSIGHTFACE_HOME, the version string here stays "buffalo_l_v1" and +# stale embeddings from the old model are served from cache indefinitely. +# TODO: derive the version from a checksum of the model files, or expose a +# --clear-cache / CLEAR_EMBEDDING_CACHE flag so users can force invalidation. MODEL_VERSIONS = { "insightface": "buffalo_l_v1", "immich": "immich_buffalo_l_v1", diff --git a/winnow/diversity.py b/winnow/diversity.py index dfe6971..c7117c6 100644 --- a/winnow/diversity.py +++ b/winnow/diversity.py @@ -203,6 +203,13 @@ def _select_by_embedding( # Process in bounded batches so at most _BATCH decoded images live in RAM # at once. With 472 candidates each thumbnail is ~3-8 MB decoded; loading # all at once easily exhausts a 4 GB container limit on CPU. + # LIMITATION — thumbnail-resolution embeddings drive full-res crop selection: + # diversity selection runs InsightFace on Immich preview thumbnails (~720p) + # to avoid downloading full-res for every candidate, but the training crop + # comes from the full-resolution original. Embeddings from thumbnails are + # representative in practice, but heavy JPEG compression on a preview could + # produce a subtly different embedding than the full-res version. For most + # libraries this is negligible; it matters if Immich preview quality is low. from concurrent.futures import ThreadPoolExecutor, as_completed _BATCH = 32 diff --git a/winnow/executor.py b/winnow/executor.py index fbf3b51..bd56342 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -311,6 +311,11 @@ def upload_to_frigate(jobs: list[dict]) -> None: rprint("[yellow]⚠️ FRIGATE_URL not set, skipping upload.[/yellow]") return + # TODO: verify Frigate version at startup (GET /api/version) and warn if + # below v0.16. The face training API (/api/faces/{name}/register, + # /api/faces/{name}/delete, /api/faces/recognize) was introduced in v0.16; + # older versions return 404s that surface as opaque upload failures. + rprint("\n[bold cyan]📤 Uploading to Frigate[/bold cyan]") rprint(f" Target: [dim]{frigate_url}[/dim]") @@ -380,6 +385,14 @@ def upload_to_frigate(jobs: list[dict]) -> None: # manually-added Frigate files don't consume winnow's managed quota. # Replacement targets also come exclusively from the tracker, so manually # added files are never selected for deletion — only winnow-uploaded ones. + # LIMITATION — manual files are invisible to diversity decisions: winnow + # can observe their effect on the Frigate score (indirectly, via recognize) + # but cannot measure their embedding distribution directly. If a user has + # 20 manually-added frontals and winnow has room for 20 more, winnow may + # add more frontals because it can't see that frontals are already covered. + # TODO(frigate-api): if Frigate exposes per-file embeddings, compute + # diversity against the full training set (tracked + manual) rather than + # relying solely on the Frigate score as a proxy signal. _snapshot = ( all_frigate_files.get(name, []) if all_frigate_files is not None else get_frigate_person_files(name) @@ -447,6 +460,14 @@ def upload_to_frigate(jobs: list[dict]) -> None: # Frigate rebuilds its model asynchronously after any delete (clear + background # thread), so the first recognize call after a deletion returns None — our code # handles this conservatively by skipping that candidate until the next run. + # LIMITATION — async rebuild during multi-replacement runs: each deletion in a + # single run triggers a background model rebuild in Frigate. Subsequent recognize + # calls in the same run may get None (rebuild in progress), causing later + # candidates to fall back to blur-score replacement or be skipped entirely. + # The more replacements that happen in one run, the worse the scoring gets. + # TODO(frigate-api): if Frigate exposes a model generation counter or a + # rebuild-complete signal, poll it between recognize calls during replacement + # sequences rather than accepting stale/None scores. pre_fscore: float | None = None if Config.ENABLE_FRIGATE_SCORES and pre_run_count > 0: if not at_cap or person_has_fscores: diff --git a/winnow/frigate_api.py b/winnow/frigate_api.py index 3d3b6fd..38ce3f5 100644 --- a/winnow/frigate_api.py +++ b/winnow/frigate_api.py @@ -33,6 +33,10 @@ def get_all_frigate_person_files() -> dict[str, list[str]] | None: return None # Response: {person_name: [file, ...], "train": [...], ...} # "train" is a flat pending list, not a person — skip it. + # TODO(frigate-api): "train" is the only known special key as of Frigate v0.16. + # If Frigate adds other top-level non-person keys, they'll be silently treated + # as person names here. Switch to an allowlist or a typed schema when Frigate + # documents its response contract. return { name: files for name, files in data.items() @@ -75,6 +79,16 @@ def recognize_face(file_path: str) -> tuple[str | None, float] | None: Returns None if FRIGATE_URL is unset, the API is unreachable, no face is detected, or face recognition is not enabled in Frigate. + + LIMITATION — mean embedding comparison: the score reflects similarity to + the arithmetic mean of all training embeddings, not to individual ones. + A bimodal training set (e.g. frontals + profiles) has a mean that sits + between both clusters, making candidates from either cluster look more + novel than they are. Winnow could add redundant frontals while the score + suggests novelty, because the mean is pulled toward profiles. + TODO(frigate-api): if Frigate exposes per-file embeddings via the API, + replace mean-comparison with nearest-neighbour distance across individual + training embeddings for accurate coverage detection. """ frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/") if not frigate_url: