From 110a45f4674edaa292631f4dd11d6ebf1df3bc4a Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 17:22:08 +0000 Subject: [PATCH] perf: batch GET /api/faces; skip download on low confidence; batch gate tracker writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetch all Frigate training files once before the upload loop instead of once per person — for N people this reduces GET /api/faces calls from N to 1. Falls back to per-person calls if the pre-fetch fails. Check InsightFace detection confidence immediately after face enrichment, before fetching the full-resolution image. Assets that fail MIN_CONFIDENCE are skipped without downloading, saving potentially large image downloads. Collapse the gate removal tracker writes from 3×N file ops into 2 total via remove_and_reclassify_batch: one write to the uploaded tracker (remove file mappings + remove from flat set) and one write to the rejected tracker. Co-Authored-By: Claude Sonnet 4.6 --- winnow/executor.py | 27 ++++++++++++++++++++++----- winnow/frigate_api.py | 16 ++++++++++++++++ winnow/upload_tracker.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/winnow/executor.py b/winnow/executor.py index d9c90c3..f29c664 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -13,7 +13,7 @@ from rich import print as rprint from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn from .config import Config, get_headers -from .frigate_api import delete_frigate_person_files, get_frigate_person_files, recognize_face +from .frigate_api import delete_frigate_person_files, get_all_frigate_person_files, get_frigate_person_files, recognize_face from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image from .log_config import console @@ -29,6 +29,7 @@ from .upload_tracker import ( mark_uploaded, reclassify_as_rejected, record_frigate_file, + remove_and_reclassify_batch, remove_frigate_file, ) @@ -186,6 +187,17 @@ def execute_jobs(jobs: list[dict]) -> None: # from the Immich faces API (not included in search/metadata results) if mode == "face": asset = _enrich_asset_with_face_data(asset, person) + # Skip download if detection confidence already disqualifies + # the asset — avoids fetching a large image we'll discard. + conf = asset.get("face_confidence") + if conf is not None and conf < Config.MIN_CONFIDENCE: + progress.console.print( + f"[yellow]Skipped {asset['id']}" + f" (confidence {conf:.2f} < {Config.MIN_CONFIDENCE})[/yellow]" + ) + progress.advance(job_task) + progress.advance(overall_task) + continue # Use full-resolution for final output when configured if use_full_res: @@ -309,6 +321,10 @@ def upload_to_frigate(jobs: list[dict]) -> None: uploaded, failed, gate_total = 0, 0, 0 max_retries = 2 + # Fetch all Frigate training files once — avoids one GET /api/faces per person. + # Falls back to per-person calls inside the loop if this fetch fails. + all_frigate_files = get_all_frigate_person_files() + with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -346,7 +362,10 @@ def upload_to_frigate(jobs: list[dict]) -> None: # Snapshot live Frigate files for post-upload reconciliation diff only. # effective_count is sourced from the tracker (mapped files) so that # manually-added Frigate files don't consume winnow's managed quota. - _snapshot = get_frigate_person_files(name) + _snapshot = ( + all_frigate_files.get(name, []) if all_frigate_files is not None + else get_frigate_person_files(name) + ) if _snapshot is None: # Frigate GET is down; fall back to the tracker's mapped filenames # as the pre-upload baseline. reconciliation will still work unless @@ -594,9 +613,7 @@ def upload_to_frigate(jobs: list[dict]) -> None: ) if to_delete: if delete_frigate_person_files(name, [fn for fn, _ in to_delete]): - for frigate_fn, aid in to_delete: - remove_frigate_file(name, frigate_fn) - reclassify_as_rejected(aid, name) + remove_and_reclassify_batch(name, to_delete) gate_removed = len(to_delete) effective_count -= gate_removed gate_total += gate_removed diff --git a/winnow/frigate_api.py b/winnow/frigate_api.py index b3eef62..5e7f30f 100644 --- a/winnow/frigate_api.py +++ b/winnow/frigate_api.py @@ -40,6 +40,22 @@ def get_frigate_face_counts() -> dict[str, int] | None: } +def get_all_frigate_person_files() -> dict[str, list[str]] | None: + """Return {person_name: [filename, ...]} for every person in Frigate. + + Single call used to build per-person snapshots before the upload loop, + avoiding one GET /api/faces per person. Returns None if unavailable. + """ + data = _get_faces_data() + if data is None: + return None + return { + name: files + for name, files in data.items() + if name != "train" and isinstance(files, list) + } + + def get_frigate_person_files(person_name: str) -> list[str] | None: """Return the list of training filenames for a person in Frigate. diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index 3a297fa..bc720eb 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -147,6 +147,45 @@ def mark_rejected(asset_id: str, person_name: str | None = None) -> None: logger.debug(f"Marked {asset_id} as rejected ({person_name})") +def remove_and_reclassify_batch( + person_name: str, frigate_files_and_assets: list[tuple[str, str]] +) -> None: + """Remove Frigate file mappings and reclassify asset IDs as rejected in one pass. + + Replaces individual remove_frigate_file + reclassify_as_rejected calls in the + quality gate batch — 2 file writes total instead of 3×N. + """ + frigate_fns = [fn for fn, _ in frigate_files_and_assets] + asset_ids = [aid for _, aid in frigate_files_and_assets] + + # Uploaded tracker: remove file mappings + remove from flat set + uploaded_data = _load(UPLOAD_TRACKER_FILE) + flat_up = set(uploaded_data.get("uploaded_asset_ids", [])) + for aid in asset_ids: + flat_up.discard(aid) + uploaded_data["uploaded_asset_ids"] = sorted(flat_up) + by_person = uploaded_data.setdefault("by_person", {}) + entry = _migrate_entry(by_person.get(person_name, {})) + for fn in frigate_fns: + entry["frigate_files"].pop(fn, None) + by_person[person_name] = entry + _save(UPLOAD_TRACKER_FILE, uploaded_data) + + # Rejected tracker: add to flat set + by_person + rejected_data = _load(REJECT_TRACKER_FILE) + flat_rej = set(rejected_data.get("rejected_asset_ids", [])) + flat_rej.update(asset_ids) + rejected_data["rejected_asset_ids"] = sorted(flat_rej) + rej_by_person = rejected_data.setdefault("by_person", {}) + rej_entry = _migrate_entry(rej_by_person.get(person_name, {})) + ids = set(rej_entry["asset_ids"]) + ids.update(asset_ids) + rej_entry["asset_ids"] = sorted(ids) + rej_by_person[person_name] = rej_entry + _save(REJECT_TRACKER_FILE, rejected_data) + logger.debug(f"Batch-reclassified {len(asset_ids)} asset(s) as rejected ({person_name})") + + def reclassify_as_rejected(asset_id: str, person_name: str | None = None) -> None: """Move a gate-failed asset from the uploaded flat set to rejected.