From 129c74720e0102e32b0e8da9c07a70617aabe559 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 03:44:36 +0000 Subject: [PATCH 01/10] Add quality replacement for Frigate face training images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a person is at MAX_AUTO_IMAGES, winnow now replaces the lowest-quality mapped training image in Frigate if a higher-confidence candidate is available, keeping the training set always optimised. Only files winnow uploaded (tracked via frigate_files mapping) are ever replaced — manually added Frigate training images are never touched. A concurrent-upload race condition is detected per-file: if N>1 new files appear after one upload, the mapping is skipped rather than guessed, logging at INFO level. The per-file snapshot approach is retained over a batch approach because wrong mappings (which a batch approach risks on race) are worse than no mapping. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 6 +++ README.md | 3 +- tests/test_upload_tracker.py | 70 ++++++++++++++++++++++++++++++++ winnow/config.py | 2 + winnow/executor.py | 63 ++++++++++++++++++++++++++++- winnow/frigate_api.py | 77 +++++++++++++++++++++++++++++------- winnow/jobs.py | 32 +++++++++------ winnow/upload_tracker.py | 59 ++++++++++++++++++++++++--- 8 files changed, 279 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eac3b3a..a9900cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Quality replacement**: when a person is at `MAX_AUTO_IMAGES`, winnow now checks each new candidate against the lowest-quality image already in Frigate and swaps it in if the new image scores higher. Only images winnow uploaded (tracked in `frigate_files`) are ever replaced — files added manually through Frigate's UI are left untouched permanently. Enabled by default; set `QUALITY_REPLACEMENT=false` to revert to the previous behaviour of skipping people at cap. +- **Frigate filename mapping**: each successful upload now records the mapping from Frigate's assigned filename to the originating Immich asset ID and face confidence score in the tracker (`frigate_files` field). This is the foundation for quality replacement and future management of the Frigate training set. +- **`QUALITY_REPLACEMENT` env var** (default `true`): controls whether at-cap people are eligible for quality replacement. When disabled, people at `MAX_AUTO_IMAGES` are skipped as before. + ## [0.2.12] - 2026-06-13 ### Added diff --git a/README.md b/README.md index 54177d6..2d5fd9b 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,8 @@ In scheduled mode the process (and loaded models) stays resident between runs. T | `USE_FULL_RESOLUTION` | `true` | Download full-resolution originals rather than preview thumbnails | | `MIN_CONFIDENCE` | `0.7` | Minimum Immich face detection confidence | | `BLUR_THRESHOLD` | `100.0` | Laplacian variance threshold — lower accepts more blur | -| `MAX_AUTO_IMAGES` | `80` | Maximum images selected in auto mode | +| `MAX_AUTO_IMAGES` | `80` | Maximum training images per person in Frigate | +| `QUALITY_REPLACEMENT` | `true` | When at cap, replace the lowest-quality mapped training image if a better candidate is found. Only affects images winnow uploaded — manually added Frigate training files are never touched. Set `false` to disable and skip people already at cap | ### GPU & Models diff --git a/tests/test_upload_tracker.py b/tests/test_upload_tracker.py index 5c59883..070a591 100644 --- a/tests/test_upload_tracker.py +++ b/tests/test_upload_tracker.py @@ -69,3 +69,73 @@ def test_duplicate_marks_are_idempotent(): mark_uploaded("dup", person_name="Alice") mark_uploaded("dup", person_name="Alice") assert filter_already_uploaded(["dup", "new"]) == ["new"] + + +# ── frigate_files mapping ───────────────────────────────────────────────────── + +def test_record_and_remove_frigate_file(): + from winnow.upload_tracker import get_person_summary, record_frigate_file, remove_frigate_file + record_frigate_file("Alice", "Alice-1000.webp", "asset-a1") + assert "Alice-1000.webp" in get_person_summary()["Alice"]["frigate_files"] + remove_frigate_file("Alice", "Alice-1000.webp") + assert "Alice-1000.webp" not in get_person_summary()["Alice"]["frigate_files"] + + +def test_remove_nonexistent_frigate_file_is_safe(): + from winnow.upload_tracker import remove_frigate_file + # Should not raise even if the file was never recorded + remove_frigate_file("Alice", "Alice-ghost.webp") + + +def test_remove_frigate_file_does_not_unmark_asset(): + """Deleting a Frigate file should not re-expose the source asset for upload.""" + from winnow.upload_tracker import ( + filter_already_uploaded, + mark_uploaded, + record_frigate_file, + remove_frigate_file, + ) + mark_uploaded("asset-a1", person_name="Alice") + record_frigate_file("Alice", "Alice-1000.webp", "asset-a1") + remove_frigate_file("Alice", "Alice-1000.webp") + # Asset must still be excluded — it was deliberately replaced, not lost + assert filter_already_uploaded(["asset-a1"]) == [] + + +def test_get_lowest_quality_mapped_file_none_when_empty(): + from winnow.upload_tracker import get_lowest_quality_mapped_file + assert get_lowest_quality_mapped_file("Alice") is None + + +def test_get_lowest_quality_mapped_file_returns_lowest(): + from winnow.upload_tracker import ( + get_lowest_quality_mapped_file, + mark_uploaded, + record_frigate_file, + ) + mark_uploaded("asset-hi", person_name="Alice", score=0.95) + mark_uploaded("asset-lo", person_name="Alice", score=0.71) + record_frigate_file("Alice", "Alice-1000.webp", "asset-hi") + record_frigate_file("Alice", "Alice-1001.webp", "asset-lo") + result = get_lowest_quality_mapped_file("Alice") + assert result is not None + frigate_filename, asset_id, score = result + assert frigate_filename == "Alice-1001.webp" + assert asset_id == "asset-lo" + assert score == pytest.approx(0.71, abs=0.001) + + +def test_get_lowest_quality_mapped_file_skips_unscored(): + """Files mapped without a score should not be returned as candidates.""" + from winnow.upload_tracker import ( + get_lowest_quality_mapped_file, + mark_uploaded, + record_frigate_file, + ) + mark_uploaded("asset-scored", person_name="Alice", score=0.85) + mark_uploaded("asset-noscr", person_name="Alice") + record_frigate_file("Alice", "Alice-1000.webp", "asset-scored") + record_frigate_file("Alice", "Alice-1001.webp", "asset-noscr") + result = get_lowest_quality_mapped_file("Alice") + assert result is not None + assert result[1] == "asset-scored" # only scored file is a candidate diff --git a/winnow/config.py b/winnow/config.py index 481f87f..ad3be2f 100644 --- a/winnow/config.py +++ b/winnow/config.py @@ -30,6 +30,7 @@ class _Config: BLUR_THRESHOLD: float = 100.0 MIN_CONFIDENCE: float = 0.7 MAX_AUTO_IMAGES: int = 80 + QUALITY_REPLACEMENT: bool = True # People filtering MIN_FACE_COUNT: int = 0 @@ -60,6 +61,7 @@ class _Config: self.BLUR_THRESHOLD = float(os.getenv("BLUR_THRESHOLD", "100.0")) self.MIN_CONFIDENCE = float(os.getenv("MIN_CONFIDENCE", "0.7")) self.MAX_AUTO_IMAGES = int(os.getenv("MAX_AUTO_IMAGES", "80")) + self.QUALITY_REPLACEMENT = os.getenv("QUALITY_REPLACEMENT", "true").lower() in ("true", "1", "yes") self.FACE_MARGIN = float(os.getenv("FACE_MARGIN", "0.15")) self.USE_FULL_RESOLUTION = os.getenv("USE_FULL_RESOLUTION", "true").lower() in ("true", "1", "yes") self.ENABLE_FACE_ALIGNMENT = os.getenv("ENABLE_FACE_ALIGNMENT", "true").lower() in ("true", "1", "yes") diff --git a/winnow/executor.py b/winnow/executor.py index 4bc608f..3da9f80 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -15,7 +15,14 @@ from .config import Config, get_headers from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image from .log_config import console -from .upload_tracker import mark_rejected, mark_uploaded +from .frigate_api import delete_frigate_person_files, get_frigate_person_files +from .upload_tracker import ( + get_lowest_quality_mapped_file, + mark_rejected, + mark_uploaded, + record_frigate_file, + remove_frigate_file, +) logger = logging.getLogger(__name__) @@ -246,8 +253,49 @@ def upload_to_frigate(jobs: list[dict]) -> None: person_uploaded = 0 person_failed = 0 + # Snapshot current Frigate filenames so we can identify which file + # each upload produces (Frigate assigns its own filename on ingest). + known_frigate_files: set[str] = set(get_frigate_person_files(name) or []) + quality_replacement = job.get("config", {}).get("quality_replacement", False) + for fname in person_files: fpath = os.path.join(person_dir, fname) + + # Quality replacement gate: when at cap, only upload if this image + # scores higher than the worst mapped file already in Frigate. + at_cap = len(known_frigate_files) >= Config.MAX_AUTO_IMAGES + if at_cap: + if not quality_replacement: + progress.console.print(f" [dim]⏭ {fname}: at cap, quality replacement disabled[/dim]") + progress.advance(upload_task) + continue + new_score = score_map.get(fname) + if new_score is None: + progress.console.print(f" [dim]⏭ {fname}: no confidence score, skipping replacement[/dim]") + progress.advance(upload_task) + continue + worst = get_lowest_quality_mapped_file(name) + if worst is None or new_score <= worst[2]: + progress.console.print( + f" [dim]⏭ {fname}: score {new_score:.3f} ≤ worst mapped" + f" {worst[2]:.3f if worst else 'N/A'}, skipping[/dim]" + ) + progress.advance(upload_task) + continue + # Delete the worst mapped file to make room for the better one + worst_frigate_file, _worst_asset_id, worst_score = worst + progress.console.print( + f" 🔄 {fname}: score {new_score:.3f} > {worst_score:.3f}," + f" replacing {worst_frigate_file}" + ) + if delete_frigate_person_files(name, [worst_frigate_file]): + remove_frigate_file(name, worst_frigate_file) + known_frigate_files.discard(worst_frigate_file) + else: + logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") + progress.advance(upload_task) + continue + for attempt in range(1, max_retries + 1): try: with open(fpath, "rb") as f: @@ -265,6 +313,19 @@ def upload_to_frigate(jobs: list[dict]) -> None: if asset_id: mark_uploaded(asset_id, person_name=name, score=score_map.get(fname)) + # Identify the Frigate filename assigned to this upload + # and record the mapping for future quality management. + current_files = set(get_frigate_person_files(name) or known_frigate_files) + new_files = current_files - known_frigate_files + if len(new_files) == 1 and asset_id: + record_frigate_file(name, next(iter(new_files)), asset_id) + elif len(new_files) > 1: + logger.info( + f"{name}: {len(new_files)} new Frigate files after uploading {fname}" + f" (concurrent upload detected) — skipping file mapping" + ) + known_frigate_files = current_files + break else: if attempt < max_retries: diff --git a/winnow/frigate_api.py b/winnow/frigate_api.py index 1988bb4..c51115e 100644 --- a/winnow/frigate_api.py +++ b/winnow/frigate_api.py @@ -8,26 +8,73 @@ import requests logger = logging.getLogger(__name__) -def get_frigate_face_counts() -> dict[str, int] | None: - """Return {person_name: training_image_count} from Frigate's train directory. - - Returns None if FRIGATE_URL is not set or the API is unreachable, so callers - can distinguish "API unavailable" from "person has 0 images." - """ +def _get_faces_data() -> dict | None: + """Fetch raw GET /api/faces response. Returns None if unavailable.""" frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/") if not frigate_url: return None try: resp = requests.get(f"{frigate_url}/api/faces", timeout=10) resp.raise_for_status() - data = resp.json() - # Response: {person_name: [file, ...], "train": [...], ...} - # "train" is a flat pending list, not a person — skip it. - return { - name: len(files) - for name, files in data.items() - if name != "train" and isinstance(files, list) - } + return resp.json() except Exception as e: - logger.warning(f"Could not query Frigate face counts: {e}") + logger.warning(f"Could not query Frigate faces API: {e}") return None + + +def get_frigate_face_counts() -> dict[str, int] | None: + """Return {person_name: training_image_count} from Frigate's train directory. + + Returns None if FRIGATE_URL is not set or the API is unreachable, so callers + can distinguish "API unavailable" from "person has 0 images." + """ + data = _get_faces_data() + if data is None: + return None + # Response: {person_name: [file, ...], "train": [...], ...} + # "train" is a flat pending list, not a person — skip it. + return { + name: len(files) + for name, files in data.items() + if name != "train" and isinstance(files, list) + } + + +def get_frigate_person_files(person_name: str) -> list[str] | None: + """Return the list of training filenames for a person in Frigate. + + Returns None if the API is unreachable. Returns an empty list if the + person exists but has no training images yet. + """ + data = _get_faces_data() + if data is None: + return None + files = data.get(person_name) + return files if isinstance(files, list) else [] + + +def delete_frigate_person_files(person_name: str, filenames: list[str]) -> bool: + """Delete specific training files for a person from Frigate. + + Uses POST /api/faces/{name}/delete with body {"ids": [filename, ...]}. + Returns True on success, False if unreachable or the request fails. + """ + frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/") + if not frigate_url or not filenames: + return False + from urllib.parse import quote + encoded = quote(person_name, safe="") + try: + resp = requests.post( + f"{frigate_url}/api/faces/{encoded}/delete", + json={"ids": filenames}, + timeout=10, + ) + if resp.ok: + logger.debug(f"Deleted {len(filenames)} Frigate file(s) for {person_name}") + return True + logger.warning(f"Frigate delete returned {resp.status_code} for {person_name}") + return False + except Exception as e: + logger.warning(f"Failed to delete Frigate files for {person_name}: {e}") + return False diff --git a/winnow/jobs.py b/winnow/jobs.py index ced0b04..954e66e 100644 --- a/winnow/jobs.py +++ b/winnow/jobs.py @@ -293,24 +293,34 @@ def auto_configure(people: list[dict]) -> list[dict]: already_uploaded = fc if fc is not None else person_summary.get("uploaded", 0) capacity = Config.MAX_AUTO_IMAGES - already_uploaded if capacity <= 0: + if not Config.QUALITY_REPLACEMENT: + rprint( + f" [dim]Skipping {name} (at cap:" + f" {already_uploaded}/{Config.MAX_AUTO_IMAGES}, quality replacement disabled).[/dim]" + ) + continue rprint( - f" [dim]Skipping {name} (at lifetime cap:" - f" {already_uploaded}/{Config.MAX_AUTO_IMAGES} trained).[/dim]" + f" [cyan]{name}: at cap ({already_uploaded}/{Config.MAX_AUTO_IMAGES})," + f" checking for quality improvements...[/cyan]" ) - continue + quality_replacement_only = True + else: + quality_replacement_only = False + + config["quality_replacement"] = quality_replacement_only or Config.QUALITY_REPLACEMENT has_embedding = is_embedding_available(entity_type) limit, selection_mode = _resolve_strategy(strategy, has_embedding) - # Cap selection to remaining capacity. - # For auto mode with partial training, keep "auto" so adaptive stopping - # still runs — just trim the result to the remaining capacity afterward. + # Cap selection to remaining capacity (no cap when replacement-only — executor + # decides per-image whether to swap; any candidate could be an improvement). auto_cap = None - if limit == "auto": - if already_uploaded > 0: - auto_cap = capacity - else: - limit = min(limit, capacity) + if not quality_replacement_only: + if limit == "auto": + if already_uploaded > 0: + auto_cap = capacity + else: + limit = min(limit, capacity) if selection_mode == "skip": continue diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index 0317639..ccaf97d 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -11,10 +11,14 @@ Both are excluded from future candidate pools. To reset: by_person schema (frigate_uploaded_ids.json): { - "asset_ids": ["immich-id-1", ...], # all assets we attempted to upload - "scores": {"immich-id-1": 0.953}, # Immich face confidence at upload time - "frigate_count": 42 # last known Frigate training image count + "asset_ids": ["immich-id-1", ...], # all assets we attempted to upload + "scores": {"immich-id-1": 0.953}, # Immich face confidence at upload time + "frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID + "frigate_count": 42 # last known Frigate training image count } + +frigate_files only contains files winnow uploaded — files added manually through +Frigate's UI are never mapped here and are never touched by quality replacement. """ import json @@ -72,9 +76,10 @@ def _get_ids(entry: list | dict) -> list[str]: def _migrate_entry(entry: list | dict) -> dict: """Ensure by_person entry is in the current dict format.""" if isinstance(entry, list): - return {"asset_ids": sorted(entry), "scores": {}} + return {"asset_ids": sorted(entry), "scores": {}, "frigate_files": {}} entry.setdefault("asset_ids", []) entry.setdefault("scores", {}) + entry.setdefault("frigate_files", {}) return entry @@ -116,6 +121,49 @@ def mark_rejected(asset_id: str, person_name: str | None = None) -> None: logger.debug(f"Marked {asset_id} as rejected ({person_name})") +def record_frigate_file(person_name: str, frigate_filename: str, asset_id: str) -> None: + """Record the mapping from a Frigate training filename to an Immich asset ID.""" + data = _load(UPLOAD_TRACKER_FILE) + by_person = data.setdefault("by_person", {}) + entry = _migrate_entry(by_person.get(person_name, {})) + entry["frigate_files"][frigate_filename] = asset_id + by_person[person_name] = entry + _save(UPLOAD_TRACKER_FILE, data) + logger.debug(f"Mapped Frigate file {frigate_filename} → {asset_id} ({person_name})") + + +def remove_frigate_file(person_name: str, frigate_filename: str) -> None: + """Remove a Frigate filename from the mapping after it has been deleted. + + Does NOT unmark the source asset_id — the deletion was deliberate and + we don't want to re-upload the inferior image on the next run. + """ + data = _load(UPLOAD_TRACKER_FILE) + by_person = data.get("by_person", {}) + entry = _migrate_entry(by_person.get(person_name, {})) + entry["frigate_files"].pop(frigate_filename, None) + by_person[person_name] = entry + _save(UPLOAD_TRACKER_FILE, data) + logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})") + + +def get_lowest_quality_mapped_file(person_name: str) -> tuple[str, str, float] | None: + """Return (frigate_filename, asset_id, score) for the mapped file with the lowest + confidence score, or None if no mapped files with known scores exist.""" + data = _load(UPLOAD_TRACKER_FILE) + entry = _migrate_entry(data.get("by_person", {}).get(person_name, {})) + frigate_files = entry.get("frigate_files", {}) + scores = entry.get("scores", {}) + candidates = [ + (frigate_filename, asset_id, scores[asset_id]) + for frigate_filename, asset_id in frigate_files.items() + if asset_id in scores + ] + if not candidates: + return None + return min(candidates, key=lambda x: x[2]) + + def update_frigate_count(person_name: str, count: int) -> None: """Record Frigate's authoritative training image count for a person.""" data = _load(UPLOAD_TRACKER_FILE) @@ -143,7 +191,7 @@ def reset_person(person_name: str) -> None: def get_person_summary() -> dict[str, dict]: - """Return {person_name: {uploaded, rejected, frigate_count, scores}} for display/capacity.""" + """Return {person_name: {uploaded, rejected, frigate_count, scores, frigate_files}} for display/capacity.""" uploaded_data = _load(UPLOAD_TRACKER_FILE).get("by_person", {}) rejected_data = _load(REJECT_TRACKER_FILE).get("by_person", {}) names = set(uploaded_data) | set(rejected_data) @@ -156,6 +204,7 @@ def get_person_summary() -> dict[str, dict]: "rejected": len(_get_ids(r_entry)), "frigate_count": u_entry.get("frigate_count") if isinstance(u_entry, dict) else None, "scores": u_entry.get("scores", {}) if isinstance(u_entry, dict) else {}, + "frigate_files": u_entry.get("frigate_files", {}) if isinstance(u_entry, dict) else {}, } return result From a0083cb9fec3d92c4400d4298e5f36c8a73fecbc Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 03:52:04 +0000 Subject: [PATCH 02/10] Fix three bugs found by code audit - Delete failure retry loop: when delete_frigate_person_files() fails, remove the file from the tracker so the next candidate targets a different worst file rather than re-attempting the same failed delete. - Interactive mode quality replacement: _configure_person() never set config["quality_replacement"], causing the executor to always default to False and silently skip all uploads for at-cap interactive jobs. Now mirrors auto_configure by reading Config.QUALITY_REPLACEMENT. - Silent mapping loss on API flap: after a successful upload, if the post-upload GET /api/faces returns None (transient API failure), the file was silently left unmapped. Now logs a warning so users know quality replacement won't target that file. Co-Authored-By: Claude Sonnet 4.6 --- winnow/executor.py | 28 +++++++++++++++++++--------- winnow/jobs.py | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/winnow/executor.py b/winnow/executor.py index 3da9f80..5a3a751 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -293,6 +293,9 @@ def upload_to_frigate(jobs: list[dict]) -> None: known_frigate_files.discard(worst_frigate_file) else: logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") + # Remove from tracker so the next candidate targets a different file. + # The file stays in Frigate (unmapped, like a manually-added file). + remove_frigate_file(name, worst_frigate_file) progress.advance(upload_task) continue @@ -315,16 +318,23 @@ def upload_to_frigate(jobs: list[dict]) -> None: # Identify the Frigate filename assigned to this upload # and record the mapping for future quality management. - current_files = set(get_frigate_person_files(name) or known_frigate_files) - new_files = current_files - known_frigate_files - if len(new_files) == 1 and asset_id: - record_frigate_file(name, next(iter(new_files)), asset_id) - elif len(new_files) > 1: - logger.info( - f"{name}: {len(new_files)} new Frigate files after uploading {fname}" - f" (concurrent upload detected) — skipping file mapping" + fresh = get_frigate_person_files(name) + if fresh is None: + logger.warning( + f"{name}: Frigate API unreachable after uploading {fname}" + f" — file mapping skipped, quality replacement won't target this file" ) - known_frigate_files = current_files + else: + current_files = set(fresh) + new_files = current_files - known_frigate_files + if len(new_files) == 1 and asset_id: + record_frigate_file(name, next(iter(new_files)), asset_id) + elif len(new_files) > 1: + logger.info( + f"{name}: {len(new_files)} new Frigate files after uploading {fname}" + f" (concurrent upload detected) — skipping file mapping" + ) + known_frigate_files = current_files break else: diff --git a/winnow/jobs.py b/winnow/jobs.py index 954e66e..369b566 100644 --- a/winnow/jobs.py +++ b/winnow/jobs.py @@ -139,7 +139,7 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None: mode_choice = Prompt.ask("Choice", choices=["1", "2"], default="1") entity_type = "face" if mode_choice == "1" else "object" - config = {"name": name, "mode": entity_type} + config = {"name": name, "mode": entity_type, "quality_replacement": Config.QUALITY_REPLACEMENT} if entity_type == "object": config["object_class"] = Prompt.ask("Enter Object Class (e.g. dog, cat, car)", default="dog") From 615c3c3cc6e22fee6a38a5f79599141689c23d09 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 03:53:28 +0000 Subject: [PATCH 03/10] Fix ruff import ordering in executor.py Co-Authored-By: Claude Sonnet 4.6 --- winnow/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winnow/executor.py b/winnow/executor.py index 5a3a751..9313403 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -12,10 +12,10 @@ from rich import print as rprint from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn from .config import Config, get_headers +from .frigate_api import delete_frigate_person_files, get_frigate_person_files from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image from .log_config import console -from .frigate_api import delete_frigate_person_files, get_frigate_person_files from .upload_tracker import ( get_lowest_quality_mapped_file, mark_rejected, From 9f92e1c91900717c80e8ef0b6c86ed1b2d925684 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 04:08:22 +0000 Subject: [PATCH 04/10] Fix Intel GPU Docker build: libze-intel-gpu1 renamed to level-zero Intel renamed libze-intel-gpu1 to level-zero in their graphics repository, breaking the amd64 Intel GPU image build. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8bc352a..201586d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,8 +91,9 @@ RUN if [ "$VARIANT" = "gpu" ]; then \ # Intel: install GPU compute runtime so OpenVINO EP can target Intel Arc / iGPU. # onnxruntime-openvino bundles OpenVINO itself; only the userspace GPU driver # (OpenCL ICD + Level Zero) is needed from the OS. -# libze-intel-gpu1 / intel-level-zero-gpu aren't in Ubuntu 22.04 main, so this -# block adds Intel's official graphics repo first, then installs. +# These packages aren't in Ubuntu 22.04 main, so this block adds Intel's +# official GPU repo first, then installs. libze-intel-gpu1 was renamed to +# level-zero in Intel's repo. RUN if [ "$VARIANT" = "intel" ]; then \ apt-get update \ && apt-get install -y --no-install-recommends curl gnupg \ @@ -103,7 +104,7 @@ https://repositories.intel.com/graphics/ubuntu jammy flex" \ > /etc/apt/sources.list.d/intel-graphics.list \ && apt-get update \ && apt-get install -y --no-install-recommends \ - intel-opencl-icd intel-level-zero-gpu libze-intel-gpu1 \ + intel-opencl-icd intel-level-zero-gpu level-zero \ && apt-get remove -y --autoremove curl gnupg \ && rm -rf /var/lib/apt/lists/*; \ fi From 4c8c2195981922038d6b568d858ae0e85f709f7b Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 04:08:22 +0000 Subject: [PATCH 05/10] Fix Intel GPU Docker build: libze-intel-gpu1 renamed to level-zero Intel renamed libze-intel-gpu1 to level-zero in their graphics repository, breaking the amd64 Intel GPU image build. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8bc352a..201586d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,8 +91,9 @@ RUN if [ "$VARIANT" = "gpu" ]; then \ # Intel: install GPU compute runtime so OpenVINO EP can target Intel Arc / iGPU. # onnxruntime-openvino bundles OpenVINO itself; only the userspace GPU driver # (OpenCL ICD + Level Zero) is needed from the OS. -# libze-intel-gpu1 / intel-level-zero-gpu aren't in Ubuntu 22.04 main, so this -# block adds Intel's official graphics repo first, then installs. +# These packages aren't in Ubuntu 22.04 main, so this block adds Intel's +# official GPU repo first, then installs. libze-intel-gpu1 was renamed to +# level-zero in Intel's repo. RUN if [ "$VARIANT" = "intel" ]; then \ apt-get update \ && apt-get install -y --no-install-recommends curl gnupg \ @@ -103,7 +104,7 @@ https://repositories.intel.com/graphics/ubuntu jammy flex" \ > /etc/apt/sources.list.d/intel-graphics.list \ && apt-get update \ && apt-get install -y --no-install-recommends \ - intel-opencl-icd intel-level-zero-gpu libze-intel-gpu1 \ + intel-opencl-icd intel-level-zero-gpu level-zero \ && apt-get remove -y --autoremove curl gnupg \ && rm -rf /var/lib/apt/lists/*; \ fi From 11e7d4aad7ecdc8fac35972ffa2c8c18145575b2 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 04:59:12 +0000 Subject: [PATCH 06/10] Implement quality score tracking and batch Frigate file mapping - Track laplacian blur score through quality filtering pipeline (quality.py: blur_score on QualityResult; diversity.py: store on asset; executor.py: read via quality_score key) - Replace per-file polling with post-person batch reconciliation: after all uploads for a person complete, poll Frigate (up to 15s) until the expected number of new files appear, then map by filename timestamp order (Frigate FIFO queue = upload order = timestamp order) - Document race condition limitation: concurrent external uploads cause the batch to be skipped entirely (safe but files go unmapped); noted in code as requiring a Frigate API fix (return filename on upload) - Add two assess_quality integration tests for blur_score Co-Authored-By: Claude Sonnet 4.6 --- tests/test_quality.py | 10 +++++ winnow/diversity.py | 1 + winnow/executor.py | 102 +++++++++++++++++++++++++++++++----------- winnow/quality.py | 10 +++-- 4 files changed, 93 insertions(+), 30 deletions(-) diff --git a/tests/test_quality.py b/tests/test_quality.py index 13f3e6d..323d916 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -134,6 +134,16 @@ def test_assess_quality_passes_good_image(): img = _noisy_color_image() result = assess_quality(img, face_bbox=(10, 10, 110, 110), confidence=0.9) assert result.passed + assert result.blur_score is not None + assert result.blur_score > 0 + + +def test_assess_quality_blur_score_is_low_for_flat_image(): + from winnow.quality import assess_quality + flat = _rgb_image(128, 128, 128) + result = assess_quality(flat) + assert result.blur_score is not None + assert result.blur_score < 1.0 def test_assess_quality_collects_multiple_failures(): diff --git a/winnow/diversity.py b/winnow/diversity.py index 0e47144..711846c 100644 --- a/winnow/diversity.py +++ b/winnow/diversity.py @@ -258,6 +258,7 @@ def _select_by_embedding( logger.debug(f"Quality filtered {asset['id']}: {quality.reason}") continue + asset["quality_score"] = quality.blur_score face_crop = _crop_face_from_thumbnail(img, asset, person_id=person_id) embed_img = face_crop if face_crop is not None else img else: diff --git a/winnow/executor.py b/winnow/executor.py index 9313403..36250dd 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -3,6 +3,7 @@ import logging import os import shutil +import time from io import BytesIO from urllib.parse import quote @@ -27,6 +28,68 @@ from .upload_tracker import ( logger = logging.getLogger(__name__) +def _reconcile_frigate_mappings( + person_name: str, + known_files_before: set[str], + uploaded: list[tuple[str, str | None]], +) -> None: + """Map Frigate filenames to asset IDs after a batch of uploads. + + Polls until all expected new files appear in the Frigate API, then maps + them to asset IDs by filename timestamp order (Frigate processes the + upload queue in FIFO order, so earlier uploads get earlier timestamps). + + KNOWN LIMITATION — race condition with external uploads: + If another client uploads a face file for this person concurrently, the + count of new files will exceed `len(uploaded)` and we bail out entirely + (the "> target" branch). That's safe — we never record a wrong mapping — + but those uploads become permanently unmapped (they won't be eligible for + quality replacement). The right fix is a Frigate API that returns the + filename in the upload response, removing the need for any post-upload + diffing. Until then, the external-upload guard keeps mappings correct at + the cost of occasionally missing them when another client is active. + """ + target = len(uploaded) + current_files: set[str] = set() + + for delay in (1, 2, 4, 8): + time.sleep(delay) + fresh = get_frigate_person_files(person_name) + if fresh is None: + logger.warning( + f"{person_name}: Frigate API unreachable during mapping reconciliation" + " — quality replacement won't target these files" + ) + return + current_files = set(fresh) + if len(current_files - known_files_before) >= target: + break + + new_files = current_files - known_files_before + + if len(new_files) == target: + def _ts(fname: str) -> float: + try: + return float(fname.rsplit("_", 1)[-1].replace(".webp", "")) + except (ValueError, IndexError): + return 0.0 + + for (fname, asset_id), frigate_file in zip(uploaded, sorted(new_files, key=_ts)): + if asset_id: + record_frigate_file(person_name, frigate_file, asset_id) + logger.debug(f"{person_name}: batch-mapped {target} Frigate file(s)") + elif len(new_files) > target: + logger.info( + f"{person_name}: {len(new_files)} new Frigate files for {target} uploads" + " (external upload detected) — skipping file mapping" + ) + else: + logger.warning( + f"{person_name}: only {len(new_files)} of {target} expected Frigate files" + " appeared after reconciliation — mapping skipped" + ) + + def _enrich_asset_with_face_data(asset: dict, person: dict) -> dict: """Enrich an asset dict with face bounding box data from the Immich faces API. @@ -141,7 +204,7 @@ def execute_jobs(jobs: list[dict]) -> None: # Record which asset produced which output file filename = f"{count}.jpg" asset_map[filename] = asset["id"] - score_map[filename] = asset.get("face_confidence") + score_map[filename] = asset.get("quality_score") or asset.get("face_confidence") # Also record object-mode variant filenames if mode == "object": for f in sorted(os.listdir(person_dir)): @@ -253,17 +316,16 @@ def upload_to_frigate(jobs: list[dict]) -> None: person_uploaded = 0 person_failed = 0 - # Snapshot current Frigate filenames so we can identify which file - # each upload produces (Frigate assigns its own filename on ingest). known_frigate_files: set[str] = set(get_frigate_person_files(name) or []) + known_frigate_files_at_start = set(known_frigate_files) + effective_count = len(known_frigate_files) quality_replacement = job.get("config", {}).get("quality_replacement", False) + actually_uploaded: list[tuple[str, str | None]] = [] for fname in person_files: fpath = os.path.join(person_dir, fname) - # Quality replacement gate: when at cap, only upload if this image - # scores higher than the worst mapped file already in Frigate. - at_cap = len(known_frigate_files) >= Config.MAX_AUTO_IMAGES + at_cap = effective_count >= Config.MAX_AUTO_IMAGES if at_cap: if not quality_replacement: progress.console.print(f" [dim]⏭ {fname}: at cap, quality replacement disabled[/dim]") @@ -291,6 +353,7 @@ def upload_to_frigate(jobs: list[dict]) -> None: if delete_frigate_person_files(name, [worst_frigate_file]): remove_frigate_file(name, worst_frigate_file) known_frigate_files.discard(worst_frigate_file) + effective_count -= 1 else: logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") # Remove from tracker so the next candidate targets a different file. @@ -310,31 +373,12 @@ def upload_to_frigate(jobs: list[dict]) -> None: if resp.status_code == 200: uploaded += 1 person_uploaded += 1 + effective_count += 1 - # Mark this asset as uploaded so it's skipped on future runs asset_id = asset_map.get(fname) if asset_id: mark_uploaded(asset_id, person_name=name, score=score_map.get(fname)) - - # Identify the Frigate filename assigned to this upload - # and record the mapping for future quality management. - fresh = get_frigate_person_files(name) - if fresh is None: - logger.warning( - f"{name}: Frigate API unreachable after uploading {fname}" - f" — file mapping skipped, quality replacement won't target this file" - ) - else: - current_files = set(fresh) - new_files = current_files - known_frigate_files - if len(new_files) == 1 and asset_id: - record_frigate_file(name, next(iter(new_files)), asset_id) - elif len(new_files) > 1: - logger.info( - f"{name}: {len(new_files)} new Frigate files after uploading {fname}" - f" (concurrent upload detected) — skipping file mapping" - ) - known_frigate_files = current_files + actually_uploaded.append((fname, asset_id)) break else: @@ -391,6 +435,10 @@ def upload_to_frigate(jobs: list[dict]) -> None: progress.advance(upload_task) + # Batch-map Frigate filenames to asset IDs now that all uploads are done + if actually_uploaded: + _reconcile_frigate_mappings(name, known_frigate_files_at_start, actually_uploaded) + # Per-person summary if person_failed == 0: progress.console.print( diff --git a/winnow/quality.py b/winnow/quality.py index 6d4ad3e..4ccc2d4 100644 --- a/winnow/quality.py +++ b/winnow/quality.py @@ -20,6 +20,7 @@ class QualityResult: passed: bool reasons: list[str] = field(default_factory=list) + blur_score: float | None = None @property def reason(self) -> str: @@ -113,9 +114,12 @@ def assess_quality( img_np = np.asarray(img) reasons = [] - # Run all checks, collect failures + # Compute laplacian variance once (used by check_blur and stored as blur_score) + gray = cv2.cvtColor(img_np, cv2.COLOR_RGB2GRAY) if img_np.ndim == 3 else img_np + blur_score = float(cv2.Laplacian(gray, cv2.CV_64F).var()) + checks = [ - check_blur(img_np, blur_threshold), + (blur_score >= blur_threshold, f"Blurry (laplacian={blur_score:.1f}, threshold={blur_threshold})" if blur_score < blur_threshold else ""), check_grayscale(img_np), check_exposure(img_np), check_confidence(confidence, min_confidence), @@ -129,5 +133,5 @@ def assess_quality( if not passed: reasons.append(reason) - return QualityResult(passed=len(reasons) == 0, reasons=reasons) + return QualityResult(passed=len(reasons) == 0, reasons=reasons, blur_score=blur_score) From 12bccaa6313595db872e8047f93559cdeb8feaab Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 05:14:15 +0000 Subject: [PATCH 07/10] Fix TypeError when worst mapped file is None in quality replacement When no mapped files exist for a person, worst is None and the old f-string tried to subscript it before the conditional was evaluated. Extracted worst_score_str as a local variable to avoid the crash. Co-Authored-By: Claude Sonnet 4.6 --- winnow/executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/winnow/executor.py b/winnow/executor.py index 36250dd..5e2f4b3 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -338,9 +338,10 @@ def upload_to_frigate(jobs: list[dict]) -> None: continue worst = get_lowest_quality_mapped_file(name) if worst is None or new_score <= worst[2]: + worst_score_str = f"{worst[2]:.3f}" if worst is not None else "N/A" progress.console.print( f" [dim]⏭ {fname}: score {new_score:.3f} ≤ worst mapped" - f" {worst[2]:.3f if worst else 'N/A'}, skipping[/dim]" + f" {worst_score_str}, skipping[/dim]" ) progress.advance(upload_task) continue From 785ac3cbc6a97d2c35dac9206c6c4881aa2a29c5 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 05:18:15 +0000 Subject: [PATCH 08/10] Add QUALITY_REPLACEMENT to config tests; expand CI disk cleanup Config tests now verify QUALITY_REPLACEMENT defaults to True and respects the QUALITY_REPLACEMENT=false env override. CI: replace minimal disk cleanup with more aggressive removal (Android SDK ~14GB, Swift, CodeQL, docker system prune) so the NVIDIA GPU image build no longer exhausts runner disk space. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/docker-publish.yml | 40 ++++++++++++++-------------- tests/test_config.py | 3 +++ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9571693..2f83e0e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,11 +36,11 @@ jobs: steps: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Disk space freed." + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift + sudo rm -rf "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" + docker system prune -af + df -h - name: Checkout repository uses: actions/checkout@v6 @@ -148,11 +148,11 @@ jobs: steps: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Disk space freed." + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift + sudo rm -rf "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" + docker system prune -af + df -h - name: Checkout repository uses: actions/checkout@v6 @@ -210,11 +210,11 @@ jobs: steps: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Disk space freed." + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift + sudo rm -rf "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" + docker system prune -af + df -h - name: Checkout repository uses: actions/checkout@v6 @@ -272,11 +272,11 @@ jobs: steps: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Disk space freed." + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift + sudo rm -rf "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" + docker system prune -af + df -h - name: Checkout repository uses: actions/checkout@v6 diff --git a/tests/test_config.py b/tests/test_config.py index 20c28bf..6e247ed 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -22,6 +22,7 @@ def test_config_loads_defaults(monkeypatch): assert cfg.BLUR_THRESHOLD == 100.0 assert cfg.MIN_CONFIDENCE == 0.7 assert cfg.MAX_AUTO_IMAGES == 80 + assert cfg.QUALITY_REPLACEMENT is True assert cfg.FACE_MARGIN == 0.15 assert cfg.USE_FULL_RESOLUTION is True assert cfg.ENABLE_FACE_ALIGNMENT is True @@ -39,6 +40,7 @@ def test_config_env_overrides(monkeypatch): monkeypatch.setenv("BLUR_THRESHOLD", "50.0") monkeypatch.setenv("MIN_CONFIDENCE", "0.9") monkeypatch.setenv("MAX_AUTO_IMAGES", "40") + monkeypatch.setenv("QUALITY_REPLACEMENT", "false") monkeypatch.setenv("FACE_MARGIN", "0.2") monkeypatch.setenv("USE_FULL_RESOLUTION", "false") monkeypatch.setenv("ENABLE_FACE_ALIGNMENT", "false") @@ -54,6 +56,7 @@ def test_config_env_overrides(monkeypatch): assert cfg.BLUR_THRESHOLD == 50.0 assert cfg.MIN_CONFIDENCE == 0.9 assert cfg.MAX_AUTO_IMAGES == 40 + assert cfg.QUALITY_REPLACEMENT is False assert cfg.FACE_MARGIN == 0.2 assert cfg.USE_FULL_RESOLUTION is False assert cfg.ENABLE_FACE_ALIGNMENT is False From 650dadd102d50bdb7d3b0ccd37601aa2db8558d1 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 05:35:26 +0000 Subject: [PATCH 09/10] Cap quality replacement against tracked files only, not total Frigate count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, effective_count and the jobs.py cap check used the total Frigate file count (including manually-added files), so any file a user curated by hand ate into winnow's managed quota. Now: - get_tracked_frigate_file_count() returns len(frigate_files) from the tracker — only files winnow uploaded and reconciled - effective_count in the upload loop uses this tracker count so manually-added files are invisible to the cap - jobs.py capacity check uses len(frigate_files) instead of the live Frigate API count or cached frigate_count - Frigate API call for known_frigate_files_at_start is now only used for the post-upload reconciliation diff, not for cap enforcement Side-effect: fixes audit bug #1 — an unreachable Frigate GET no longer zeroes effective_count and bypasses the cap, because the cap is now read from the always-available local tracker. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_upload_tracker.py | 17 +++++++++++++++++ winnow/executor.py | 10 ++++++---- winnow/jobs.py | 11 ++++------- winnow/upload_tracker.py | 11 +++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/test_upload_tracker.py b/tests/test_upload_tracker.py index 070a591..b5b6acd 100644 --- a/tests/test_upload_tracker.py +++ b/tests/test_upload_tracker.py @@ -102,6 +102,23 @@ def test_remove_frigate_file_does_not_unmark_asset(): assert filter_already_uploaded(["asset-a1"]) == [] +def test_get_tracked_frigate_file_count_zero_when_empty(): + from winnow.upload_tracker import get_tracked_frigate_file_count + assert get_tracked_frigate_file_count("Alice") == 0 + + +def test_get_tracked_frigate_file_count_counts_only_mapped(): + """Only files explicitly recorded via record_frigate_file count toward the cap.""" + from winnow.upload_tracker import get_tracked_frigate_file_count, mark_uploaded, record_frigate_file + mark_uploaded("asset-a", person_name="Alice") + mark_uploaded("asset-b", person_name="Alice") + record_frigate_file("Alice", "Alice-1000.webp", "asset-a") + # asset-b is uploaded but not yet mapped — does not count + assert get_tracked_frigate_file_count("Alice") == 1 + record_frigate_file("Alice", "Alice-1001.webp", "asset-b") + assert get_tracked_frigate_file_count("Alice") == 2 + + def test_get_lowest_quality_mapped_file_none_when_empty(): from winnow.upload_tracker import get_lowest_quality_mapped_file assert get_lowest_quality_mapped_file("Alice") is None diff --git a/winnow/executor.py b/winnow/executor.py index 5e2f4b3..fbf6e40 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -19,6 +19,7 @@ from .immich_api import fetch_face_data, fetch_full_image from .log_config import console from .upload_tracker import ( get_lowest_quality_mapped_file, + get_tracked_frigate_file_count, mark_rejected, mark_uploaded, record_frigate_file, @@ -316,9 +317,11 @@ def upload_to_frigate(jobs: list[dict]) -> None: person_uploaded = 0 person_failed = 0 - known_frigate_files: set[str] = set(get_frigate_person_files(name) or []) - known_frigate_files_at_start = set(known_frigate_files) - effective_count = len(known_frigate_files) + # Snapshot live Frigate files for post-upload reconciliation diff only. + # effective_count is sourced from the tracker (mapped files) so that + # manually-added Frigate files don't consume winnow's managed quota. + known_frigate_files_at_start: set[str] = set(get_frigate_person_files(name) or []) + effective_count = get_tracked_frigate_file_count(name) quality_replacement = job.get("config", {}).get("quality_replacement", False) actually_uploaded: list[tuple[str, str | None]] = [] @@ -353,7 +356,6 @@ def upload_to_frigate(jobs: list[dict]) -> None: ) if delete_frigate_person_files(name, [worst_frigate_file]): remove_frigate_file(name, worst_frigate_file) - known_frigate_files.discard(worst_frigate_file) effective_count -= 1 else: logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") diff --git a/winnow/jobs.py b/winnow/jobs.py index 369b566..fda3fd8 100644 --- a/winnow/jobs.py +++ b/winnow/jobs.py @@ -283,14 +283,11 @@ def auto_configure(people: list[dict]) -> list[dict]: rprint(f" [dim]Skipping {name} (0 new images after dedup).[/dim]") continue - # Enforce MAX_AUTO_IMAGES as a lifetime cap per person. - # Priority: live Frigate count → last cached Frigate count → local uploaded count. + # Enforce MAX_AUTO_IMAGES against the tracked file count only. + # Manually-added Frigate files are invisible to this cap so users can + # curate their own files without shrinking winnow's managed quota. person_summary = upload_summary.get(name, {}) - if frigate_counts is not None: - already_uploaded = frigate_counts.get(name, 0) - else: - fc = person_summary.get("frigate_count") - already_uploaded = fc if fc is not None else person_summary.get("uploaded", 0) + already_uploaded = len(person_summary.get("frigate_files", {})) capacity = Config.MAX_AUTO_IMAGES - already_uploaded if capacity <= 0: if not Config.QUALITY_REPLACEMENT: diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index ccaf97d..3b6de12 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -147,6 +147,17 @@ def remove_frigate_file(person_name: str, frigate_filename: str) -> None: logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})") +def get_tracked_frigate_file_count(person_name: str) -> int: + """Return the number of Frigate training files winnow has mapped for this person. + + Used as the cap baseline so that manually-added Frigate files do not + consume slots from winnow's managed quota. + """ + data = _load(UPLOAD_TRACKER_FILE) + entry = _migrate_entry(data.get("by_person", {}).get(person_name, {})) + return len(entry["frigate_files"]) + + def get_lowest_quality_mapped_file(person_name: str) -> tuple[str, str, float] | None: """Return (frigate_filename, asset_id, score) for the mapped file with the lowest confidence score, or None if no mapped files with known scores exist.""" From 0ce67a757009863b82fb501f0ba5d868c9e1db34 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 05:55:51 +0000 Subject: [PATCH 10/10] Add NOTICES file for if_curator MIT attribution; bump to 0.2.13 Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 3 +++ NOTICES | 23 +++++++++++++++++++++++ pyproject-cpu.toml | 2 +- pyproject-intel.toml | 2 +- pyproject-rocm.toml | 2 +- pyproject.toml | 2 +- 6 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 NOTICES diff --git a/CHANGELOG.md b/CHANGELOG.md index a9900cc..ec2b670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.13] - 2026-06-13 + ### Added - **Quality replacement**: when a person is at `MAX_AUTO_IMAGES`, winnow now checks each new candidate against the lowest-quality image already in Frigate and swaps it in if the new image scores higher. Only images winnow uploaded (tracked in `frigate_files`) are ever replaced — files added manually through Frigate's UI are left untouched permanently. Enabled by default; set `QUALITY_REPLACEMENT=false` to revert to the previous behaviour of skipping people at cap. - **Frigate filename mapping**: each successful upload now records the mapping from Frigate's assigned filename to the originating Immich asset ID and face confidence score in the tracker (`frigate_files` field). This is the foundation for quality replacement and future management of the Frigate training set. - **`QUALITY_REPLACEMENT` env var** (default `true`): controls whether at-cap people are eligible for quality replacement. When disabled, people at `MAX_AUTO_IMAGES` are skipped as before. +- **NOTICES file**: third-party attribution for if_curator (MIT, Copyright © 2026 Sebastian) added to satisfy upstream license requirements. ## [0.2.12] - 2026-06-13 diff --git a/NOTICES b/NOTICES new file mode 100644 index 0000000..a839356 --- /dev/null +++ b/NOTICES @@ -0,0 +1,23 @@ +winnow incorporates portions of if_curator (https://github.com/ds-sebastian/if_curator). + + MIT License + + Copyright (c) 2026 Sebastian + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/pyproject-cpu.toml b/pyproject-cpu.toml index 99133be..7a72006 100644 --- a/pyproject-cpu.toml +++ b/pyproject-cpu.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.2.12" +version = "0.2.13" description = "Immich to Frigate training sets" license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/pyproject-intel.toml b/pyproject-intel.toml index d8c9039..67b7e66 100644 --- a/pyproject-intel.toml +++ b/pyproject-intel.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.2.12" +version = "0.2.13" description = "Immich to Frigate training sets" license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/pyproject-rocm.toml b/pyproject-rocm.toml index d1a8c1d..ab0d46b 100644 --- a/pyproject-rocm.toml +++ b/pyproject-rocm.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.2.12" +version = "0.2.13" description = "Immich to Frigate training sets" license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/pyproject.toml b/pyproject.toml index 8de4cbe..e7dfe93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.2.12" +version = "0.2.13" description = "Immich to Frigate training sets" license = "AGPL-3.0-or-later" requires-python = ">=3.13"