From 03be6ce2cb5ddb86807a3a836c0d062048a70f8e Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 16:21:58 +0000 Subject: [PATCH] feat: store Frigate recognition scores and use them for quality replacement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After each successful upload, call POST /api/faces/recognize to get Frigate's own confidence score (0-1) for the uploaded crop. Store it in the tracker as frigate_scores alongside the existing blur score. When quality replacement activates and frigate_scores are present, use them for the replacement comparison instead of blur scores — an image Frigate recognizes poorly is a worse training image than one it recognizes well, regardless of sharpness. Falls back to blur scores on first run before any frigate_scores are populated. Also surfaces frigate_score in TRACE_CROP_SIZE output. Co-Authored-By: Claude Sonnet 4.6 --- winnow/cli.py | 2 ++ winnow/executor.py | 40 ++++++++++++++++------- winnow/frigate_api.py | 27 ++++++++++++++++ winnow/upload_tracker.py | 69 +++++++++++++++++++++++++++++++++------- 4 files changed, 115 insertions(+), 23 deletions(-) diff --git a/winnow/cli.py b/winnow/cli.py index 24e5b5c..007875e 100644 --- a/winnow/cli.py +++ b/winnow/cli.py @@ -41,6 +41,8 @@ def _handle_trace_crop(size_str: str) -> None: rprint(f" Immich URL: {immich_url}/photos/{m['asset_id']}") blur = m.get("blur_score") rprint(f" Blur score: {blur:.1f}" if blur is not None else " Blur score: unknown") + fscore = m.get("frigate_score") + rprint(f" Frigate score: {fscore:.2f}" if fscore is not None else " Frigate score: unknown") if m.get("frigate_filename"): rprint(f" Frigate file: {m['frigate_filename']}") else: diff --git a/winnow/executor.py b/winnow/executor.py index 2c2bdfe..05a3930 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -13,7 +13,7 @@ from rich import print as rprint from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn from .config import Config, get_headers -from .frigate_api import delete_frigate_person_files, get_frigate_person_files +from .frigate_api import delete_frigate_person_files, get_frigate_person_files, recognize_face from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image from .log_config import console @@ -22,6 +22,7 @@ from .upload_tracker import ( get_lowest_quality_mapped_file, get_tracked_frigate_file_count, get_tracked_frigate_filenames, + has_frigate_scores, mark_rejected, mark_uploaded, record_frigate_file, @@ -383,30 +384,45 @@ def upload_to_frigate(jobs: list[dict]) -> None: progress.console.print(f" [dim]⏭ {fname}: at cap, quality replacement disabled[/dim]") progress.advance(upload_task) continue - new_score = score_map.get(fname) - if new_score is None: - progress.console.print(f" [dim]⏭ {fname}: no confidence score, skipping replacement[/dim]") - progress.advance(upload_task) - continue + using_fscore = has_frigate_scores(name) + if using_fscore: + candidate_score = recognize_face(fpath) + if candidate_score is None: + progress.console.print( + f" [dim]⏭ {fname}: Frigate recognize unavailable, skipping replacement[/dim]" + ) + progress.advance(upload_task) + continue + score_label = "frigate" + else: + candidate_score = score_map.get(fname) + if candidate_score is None: + progress.console.print( + f" [dim]⏭ {fname}: no quality score, skipping replacement[/dim]" + ) + progress.advance(upload_task) + continue + score_label = "blur" worst = get_lowest_quality_mapped_file(name, exclude=failed_deletes) - if worst is None or new_score <= worst[2]: + if worst is None or candidate_score <= worst[2]: worst_score_str = f"{worst[2]:.3f}" if worst is not None else "N/A" progress.console.print( - f" [dim]⏭ {fname}: score {new_score:.3f} ≤ worst mapped" + f" [dim]⏭ {fname}: {score_label} {candidate_score:.3f} ≤ worst" f" {worst_score_str}, skipping[/dim]" ) progress.advance(upload_task) continue - # Delete the worst mapped file to make room for the better one worst_frigate_file, _worst_asset_id, worst_score = worst progress.console.print( - f" 🔄 {fname}: score {new_score:.3f} > {worst_score:.3f}," + f" 🔄 {fname}: {score_label} {candidate_score:.3f} > {worst_score:.3f}," f" replacing {worst_frigate_file}" ) if delete_frigate_person_files(name, [worst_frigate_file]): remove_frigate_file(name, worst_frigate_file) effective_count -= 1 - min_quality_score_for_slot = worst_score + # Slot floor guard uses blur scores only — frigate_score mode + # will re-evaluate the next candidate via recognize_face anyway. + min_quality_score_for_slot = score_map.get(fname) if not using_fscore else None else: logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") failed_deletes.add(worst_frigate_file) @@ -429,11 +445,13 @@ def upload_to_frigate(jobs: list[dict]) -> None: asset_id = asset_map.get(fname) if asset_id: + post_fscore = recognize_face(fpath) mark_uploaded( asset_id, person_name=name, score=score_map.get(fname), crop_dims=dims_map.get(fname), + frigate_score=post_fscore, ) actually_uploaded.append((fname, asset_id)) diff --git a/winnow/frigate_api.py b/winnow/frigate_api.py index b6d4950..b3eef62 100644 --- a/winnow/frigate_api.py +++ b/winnow/frigate_api.py @@ -53,6 +53,33 @@ def get_frigate_person_files(person_name: str) -> list[str] | None: return files if isinstance(files, list) else [] +def recognize_face(file_path: str) -> float | None: + """Submit an image to Frigate's recognize endpoint and return the confidence score. + + Returns None if FRIGATE_URL is unset, the API is unreachable, no face is + detected, or face recognition is not enabled in Frigate. + """ + frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/") + if not frigate_url: + return None + try: + with open(file_path, "rb") as f: + resp = requests.post( + f"{frigate_url}/api/faces/recognize", + files={"file": (os.path.basename(file_path), f, "image/jpeg")}, + timeout=15, + ) + if not resp.ok: + return None + data = resp.json() + if data.get("success") and "score" in data: + return round(float(data["score"]), 4) + return None + except Exception as e: + logger.debug(f"Frigate recognize failed for {file_path}: {e}") + return None + + def delete_frigate_person_files(person_name: str, filenames: list[str]) -> bool: """Delete specific training files for a person from Frigate. diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index 79d128b..b3d4528 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -11,13 +11,19 @@ Both are excluded from future candidate pools. To reset: by_person schema (frigate_uploaded_ids.json): { - "asset_ids": ["immich-id-1", ...], # all assets we attempted to upload - "scores": {"immich-id-1": 450.3}, # Laplacian blur variance at upload time - "frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID - "crop_dims": {"immich-id-1": [640, 480]}, # crop pixel dimensions at upload time - "frigate_count": 42 # last known Frigate training image count + "asset_ids": ["immich-id-1", ...], # all assets we attempted to upload + "scores": {"immich-id-1": 450.3}, # Laplacian blur variance at upload time + "frigate_scores": {"immich-id-1": 0.87}, # Frigate recognition confidence (0-1) post-upload + "frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID + "crop_dims": {"immich-id-1": [640, 480]}, # crop pixel dimensions at upload time + "frigate_count": 42 # last known Frigate training image count } +frigate_scores uses the same 0-1 sigmoid-mapped cosine similarity that Frigate +displays in its UI. When available, quality replacement uses frigate_scores in +preference to blur scores — an image Frigate cannot recognize is a poor training +image regardless of sharpness. + frigate_files only contains files winnow uploaded — files added manually through Frigate's UI are never mapped here and are never touched by quality replacement. """ @@ -77,9 +83,10 @@ def _get_ids(entry: list | dict) -> list[str]: def _migrate_entry(entry: list | dict) -> dict: """Ensure by_person entry is in the current dict format.""" if isinstance(entry, list): - return {"asset_ids": sorted(entry), "scores": {}, "frigate_files": {}, "crop_dims": {}} + return {"asset_ids": sorted(entry), "scores": {}, "frigate_scores": {}, "frigate_files": {}, "crop_dims": {}} entry.setdefault("asset_ids", []) entry.setdefault("scores", {}) + entry.setdefault("frigate_scores", {}) entry.setdefault("frigate_files", {}) entry.setdefault("crop_dims", {}) return entry @@ -91,6 +98,7 @@ def _mark( person_name: str | None, score: float | None = None, crop_dims: tuple[int, int] | None = None, + frigate_score: float | None = None, ) -> None: data = _load(filename) flat_key = _flat_key(filename) @@ -107,6 +115,8 @@ def _mark( entry["scores"][asset_id] = round(score, 4) if crop_dims is not None: entry["crop_dims"][asset_id] = [crop_dims[0], crop_dims[1]] + if frigate_score is not None: + entry["frigate_scores"][asset_id] = round(frigate_score, 4) by_person[person_name] = entry _save(filename, data) @@ -126,8 +136,9 @@ def mark_uploaded( person_name: str | None = None, score: float | None = None, crop_dims: tuple[int, int] | None = None, + frigate_score: float | None = None, ) -> None: - _mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims) + _mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims, frigate_score=frigate_score) logger.debug(f"Marked {asset_id} as uploaded ({person_name})") @@ -184,24 +195,56 @@ def get_tracked_frigate_filenames(person_name: str) -> set[str]: return set(entry["frigate_files"].keys()) +def has_frigate_scores(person_name: str) -> bool: + """Return True if any mapped file for this person has a stored Frigate recognition score.""" + data = _load(UPLOAD_TRACKER_FILE) + entry = _migrate_entry(data.get("by_person", {}).get(person_name, {})) + frigate_files = entry.get("frigate_files", {}) + frigate_scores = entry.get("frigate_scores", {}) + return any(asset_id in frigate_scores for asset_id in frigate_files.values()) + + def get_lowest_quality_mapped_file( person_name: str, exclude: set[str] | None = None ) -> tuple[str, str, float] | None: """Return (frigate_filename, asset_id, score) for the mapped file with the lowest quality score, or None if no mapped files with known scores exist. + Uses Frigate recognition scores (0-1) when any are present for this person, + treating files without a Frigate score as 0.0. Falls back to Laplacian blur + scores when no Frigate scores exist yet. + Pass `exclude` to skip files that failed to delete this run without removing them from the tracker — they remain candidates on the next run. """ data = _load(UPLOAD_TRACKER_FILE) entry = _migrate_entry(data.get("by_person", {}).get(person_name, {})) frigate_files = entry.get("frigate_files", {}) - scores = entry.get("scores", {}) - candidates = [ - (frigate_filename, asset_id, scores[asset_id]) - for frigate_filename, asset_id in frigate_files.items() - if asset_id in scores and (exclude is None or frigate_filename not in exclude) + blur_scores = entry.get("scores", {}) + frigate_scores = entry.get("frigate_scores", {}) + + mapped = [ + (ff, asset_id) + for ff, asset_id in frigate_files.items() + if exclude is None or ff not in exclude ] + if not mapped: + return None + + use_frigate = any(asset_id in frigate_scores for _, asset_id in mapped) + + if use_frigate: + candidates = [ + (ff, asset_id, frigate_scores.get(asset_id, 0.0)) + for ff, asset_id in mapped + ] + else: + candidates = [ + (ff, asset_id, blur_scores[asset_id]) + for ff, asset_id in mapped + if asset_id in blur_scores + ] + if not candidates: return None return min(candidates, key=lambda x: x[2]) @@ -220,6 +263,7 @@ def find_by_crop_dimension(size: int) -> list[dict]: scores = entry.get("scores", {}) frigate_files = entry.get("frigate_files", {}) asset_to_frigate = {v: k for k, v in frigate_files.items()} + frigate_scores = entry.get("frigate_scores", {}) for asset_id, dims in entry.get("crop_dims", {}).items(): w, h = dims[0], dims[1] if w == size or h == size: @@ -229,6 +273,7 @@ def find_by_crop_dimension(size: int) -> list[dict]: "width": w, "height": h, "blur_score": scores.get(asset_id), + "frigate_score": frigate_scores.get(asset_id), "frigate_filename": asset_to_frigate.get(asset_id), }) return results