feat: store Frigate recognition scores and use them for quality replacement

After each successful upload, call POST /api/faces/recognize to get
Frigate's own confidence score (0-1) for the uploaded crop. Store it
in the tracker as frigate_scores alongside the existing blur score.

When quality replacement activates and frigate_scores are present,
use them for the replacement comparison instead of blur scores — an
image Frigate recognizes poorly is a worse training image than one it
recognizes well, regardless of sharpness. Falls back to blur scores
on first run before any frigate_scores are populated.

Also surfaces frigate_score in TRACE_CROP_SIZE output.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

winnow/cli.py

@@ -41,6 +41,8 @@ def _handle_trace_crop(size_str: str) -> None:
             rprint(f"    Immich URL:   {immich_url}/photos/{m['asset_id']}")
         blur = m.get("blur_score")
         rprint(f"    Blur score:   {blur:.1f}" if blur is not None else "    Blur score:   unknown")
+        fscore = m.get("frigate_score")
+        rprint(f"    Frigate score: {fscore:.2f}" if fscore is not None else "    Frigate score: unknown")
         if m.get("frigate_filename"):
             rprint(f"    Frigate file: {m['frigate_filename']}")
         else:

winnow/executor.py

@@ -13,7 +13,7 @@ from rich import print as rprint
 from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
 
 from .config import Config, get_headers
-from .frigate_api import delete_frigate_person_files, get_frigate_person_files
+from .frigate_api import delete_frigate_person_files, get_frigate_person_files, recognize_face
 from .image_processing import process_face_mode, process_full_mode, process_object_mode
 from .immich_api import fetch_face_data, fetch_full_image
 from .log_config import console
@@ -22,6 +22,7 @@ from .upload_tracker import (
     get_lowest_quality_mapped_file,
     get_tracked_frigate_file_count,
     get_tracked_frigate_filenames,
+    has_frigate_scores,
     mark_rejected,
     mark_uploaded,
     record_frigate_file,
@@ -383,30 +384,45 @@ def upload_to_frigate(jobs: list[dict]) -> None:
                         progress.console.print(f"    [dim]⏭  {fname}: at cap, quality replacement disabled[/dim]")
                         progress.advance(upload_task)
                         continue
-                    new_score = score_map.get(fname)
-                    if new_score is None:
-                        progress.console.print(f"    [dim]⏭  {fname}: no confidence score, skipping replacement[/dim]")
-                        progress.advance(upload_task)
-                        continue
+                    using_fscore = has_frigate_scores(name)
+                    if using_fscore:
+                        candidate_score = recognize_face(fpath)
+                        if candidate_score is None:
+                            progress.console.print(
+                                f"    [dim]⏭  {fname}: Frigate recognize unavailable, skipping replacement[/dim]"
+                            )
+                            progress.advance(upload_task)
+                            continue
+                        score_label = "frigate"
+                    else:
+                        candidate_score = score_map.get(fname)
+                        if candidate_score is None:
+                            progress.console.print(
+                                f"    [dim]⏭  {fname}: no quality score, skipping replacement[/dim]"
+                            )
+                            progress.advance(upload_task)
+                            continue
+                        score_label = "blur"
                     worst = get_lowest_quality_mapped_file(name, exclude=failed_deletes)
-                    if worst is None or new_score <= worst[2]:
+                    if worst is None or candidate_score <= worst[2]:
                         worst_score_str = f"{worst[2]:.3f}" if worst is not None else "N/A"
                         progress.console.print(
-                            f"    [dim]⏭  {fname}: score {new_score:.3f} ≤ worst mapped"
+                            f"    [dim]⏭  {fname}: {score_label} {candidate_score:.3f} ≤ worst"
                             f" {worst_score_str}, skipping[/dim]"
                         )
                         progress.advance(upload_task)
                         continue
-                    # Delete the worst mapped file to make room for the better one
                     worst_frigate_file, _worst_asset_id, worst_score = worst
                     progress.console.print(
-                        f"    🔄 {fname}: score {new_score:.3f} > {worst_score:.3f},"
+                        f"    🔄 {fname}: {score_label} {candidate_score:.3f} > {worst_score:.3f},"
                         f" replacing {worst_frigate_file}"
                     )
                     if delete_frigate_person_files(name, [worst_frigate_file]):
                         remove_frigate_file(name, worst_frigate_file)
                         effective_count -= 1
-                        min_quality_score_for_slot = worst_score
+                        # Slot floor guard uses blur scores only — frigate_score mode
+                        # will re-evaluate the next candidate via recognize_face anyway.
+                        min_quality_score_for_slot = score_map.get(fname) if not using_fscore else None
                     else:
                         logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement")
                         failed_deletes.add(worst_frigate_file)
@@ -429,11 +445,13 @@ def upload_to_frigate(jobs: list[dict]) -> None:
 
                             asset_id = asset_map.get(fname)
                             if asset_id:
+                                post_fscore = recognize_face(fpath)
                                 mark_uploaded(
                                     asset_id,
                                     person_name=name,
                                     score=score_map.get(fname),
                                     crop_dims=dims_map.get(fname),
+                                    frigate_score=post_fscore,
                                 )
                                 actually_uploaded.append((fname, asset_id))
 

winnow/frigate_api.py

@@ -53,6 +53,33 @@ def get_frigate_person_files(person_name: str) -> list[str] | None:
     return files if isinstance(files, list) else []
 
 
+def recognize_face(file_path: str) -> float | None:
+    """Submit an image to Frigate's recognize endpoint and return the confidence score.
+
+    Returns None if FRIGATE_URL is unset, the API is unreachable, no face is
+    detected, or face recognition is not enabled in Frigate.
+    """
+    frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
+    if not frigate_url:
+        return None
+    try:
+        with open(file_path, "rb") as f:
+            resp = requests.post(
+                f"{frigate_url}/api/faces/recognize",
+                files={"file": (os.path.basename(file_path), f, "image/jpeg")},
+                timeout=15,
+            )
+        if not resp.ok:
+            return None
+        data = resp.json()
+        if data.get("success") and "score" in data:
+            return round(float(data["score"]), 4)
+        return None
+    except Exception as e:
+        logger.debug(f"Frigate recognize failed for {file_path}: {e}")
+        return None
+
+
 def delete_frigate_person_files(person_name: str, filenames: list[str]) -> bool:
     """Delete specific training files for a person from Frigate.
 

winnow/upload_tracker.py

@@ -11,13 +11,19 @@ Both are excluded from future candidate pools. To reset:
 
 by_person schema (frigate_uploaded_ids.json):
   {
-    "asset_ids":     ["immich-id-1", ...],                    # all assets we attempted to upload
-    "scores":        {"immich-id-1": 450.3},                   # Laplacian blur variance at upload time
-    "frigate_files": {"PersonName-123.webp": "immich-id-1"},  # Frigate filename → asset ID
-    "crop_dims":     {"immich-id-1": [640, 480]},             # crop pixel dimensions at upload time
-    "frigate_count": 42                                       # last known Frigate training image count
+    "asset_ids":      ["immich-id-1", ...],                    # all assets we attempted to upload
+    "scores":         {"immich-id-1": 450.3},                  # Laplacian blur variance at upload time
+    "frigate_scores": {"immich-id-1": 0.87},                   # Frigate recognition confidence (0-1) post-upload
+    "frigate_files":  {"PersonName-123.webp": "immich-id-1"},  # Frigate filename → asset ID
+    "crop_dims":      {"immich-id-1": [640, 480]},             # crop pixel dimensions at upload time
+    "frigate_count":  42                                       # last known Frigate training image count
   }
 
+frigate_scores uses the same 0-1 sigmoid-mapped cosine similarity that Frigate
+displays in its UI. When available, quality replacement uses frigate_scores in
+preference to blur scores — an image Frigate cannot recognize is a poor training
+image regardless of sharpness.
+
 frigate_files only contains files winnow uploaded — files added manually through
 Frigate's UI are never mapped here and are never touched by quality replacement.
 """
@@ -77,9 +83,10 @@ def _get_ids(entry: list | dict) -> list[str]:
 def _migrate_entry(entry: list | dict) -> dict:
     """Ensure by_person entry is in the current dict format."""
     if isinstance(entry, list):
-        return {"asset_ids": sorted(entry), "scores": {}, "frigate_files": {}, "crop_dims": {}}
+        return {"asset_ids": sorted(entry), "scores": {}, "frigate_scores": {}, "frigate_files": {}, "crop_dims": {}}
     entry.setdefault("asset_ids", [])
     entry.setdefault("scores", {})
+    entry.setdefault("frigate_scores", {})
     entry.setdefault("frigate_files", {})
     entry.setdefault("crop_dims", {})
     return entry
@@ -91,6 +98,7 @@ def _mark(
     person_name: str | None,
     score: float | None = None,
     crop_dims: tuple[int, int] | None = None,
+    frigate_score: float | None = None,
 ) -> None:
     data = _load(filename)
     flat_key = _flat_key(filename)
@@ -107,6 +115,8 @@ def _mark(
             entry["scores"][asset_id] = round(score, 4)
         if crop_dims is not None:
             entry["crop_dims"][asset_id] = [crop_dims[0], crop_dims[1]]
+        if frigate_score is not None:
+            entry["frigate_scores"][asset_id] = round(frigate_score, 4)
         by_person[person_name] = entry
     _save(filename, data)
 
@@ -126,8 +136,9 @@ def mark_uploaded(
     person_name: str | None = None,
     score: float | None = None,
     crop_dims: tuple[int, int] | None = None,
+    frigate_score: float | None = None,
 ) -> None:
-    _mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims)
+    _mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims, frigate_score=frigate_score)
     logger.debug(f"Marked {asset_id} as uploaded ({person_name})")
 
 
@@ -184,24 +195,56 @@ def get_tracked_frigate_filenames(person_name: str) -> set[str]:
     return set(entry["frigate_files"].keys())
 
 
+def has_frigate_scores(person_name: str) -> bool:
+    """Return True if any mapped file for this person has a stored Frigate recognition score."""
+    data = _load(UPLOAD_TRACKER_FILE)
+    entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
+    frigate_files = entry.get("frigate_files", {})
+    frigate_scores = entry.get("frigate_scores", {})
+    return any(asset_id in frigate_scores for asset_id in frigate_files.values())
+
+
 def get_lowest_quality_mapped_file(
     person_name: str, exclude: set[str] | None = None
 ) -> tuple[str, str, float] | None:
     """Return (frigate_filename, asset_id, score) for the mapped file with the lowest
     quality score, or None if no mapped files with known scores exist.
 
+    Uses Frigate recognition scores (0-1) when any are present for this person,
+    treating files without a Frigate score as 0.0. Falls back to Laplacian blur
+    scores when no Frigate scores exist yet.
+
     Pass `exclude` to skip files that failed to delete this run without removing
     them from the tracker — they remain candidates on the next run.
     """
     data = _load(UPLOAD_TRACKER_FILE)
     entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
     frigate_files = entry.get("frigate_files", {})
-    scores = entry.get("scores", {})
-    candidates = [
-        (frigate_filename, asset_id, scores[asset_id])
-        for frigate_filename, asset_id in frigate_files.items()
-        if asset_id in scores and (exclude is None or frigate_filename not in exclude)
+    blur_scores = entry.get("scores", {})
+    frigate_scores = entry.get("frigate_scores", {})
+
+    mapped = [
+        (ff, asset_id)
+        for ff, asset_id in frigate_files.items()
+        if exclude is None or ff not in exclude
     ]
+    if not mapped:
+        return None
+
+    use_frigate = any(asset_id in frigate_scores for _, asset_id in mapped)
+
+    if use_frigate:
+        candidates = [
+            (ff, asset_id, frigate_scores.get(asset_id, 0.0))
+            for ff, asset_id in mapped
+        ]
+    else:
+        candidates = [
+            (ff, asset_id, blur_scores[asset_id])
+            for ff, asset_id in mapped
+            if asset_id in blur_scores
+        ]
+
     if not candidates:
         return None
     return min(candidates, key=lambda x: x[2])
@@ -220,6 +263,7 @@ def find_by_crop_dimension(size: int) -> list[dict]:
         scores = entry.get("scores", {})
         frigate_files = entry.get("frigate_files", {})
         asset_to_frigate = {v: k for k, v in frigate_files.items()}
+        frigate_scores = entry.get("frigate_scores", {})
         for asset_id, dims in entry.get("crop_dims", {}).items():
             w, h = dims[0], dims[1]
             if w == size or h == size:
@@ -229,6 +273,7 @@ def find_by_crop_dimension(size: int) -> list[dict]:
                     "width": w,
                     "height": h,
                     "blur_score": scores.get(asset_id),
+                    "frigate_score": frigate_scores.get(asset_id),
                     "frigate_filename": asset_to_frigate.get(asset_id),
                 })
     return results