From 650dadd102d50bdb7d3b0ccd37601aa2db8558d1 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 05:35:26 +0000 Subject: [PATCH] Cap quality replacement against tracked files only, not total Frigate count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, effective_count and the jobs.py cap check used the total Frigate file count (including manually-added files), so any file a user curated by hand ate into winnow's managed quota. Now: - get_tracked_frigate_file_count() returns len(frigate_files) from the tracker — only files winnow uploaded and reconciled - effective_count in the upload loop uses this tracker count so manually-added files are invisible to the cap - jobs.py capacity check uses len(frigate_files) instead of the live Frigate API count or cached frigate_count - Frigate API call for known_frigate_files_at_start is now only used for the post-upload reconciliation diff, not for cap enforcement Side-effect: fixes audit bug #1 — an unreachable Frigate GET no longer zeroes effective_count and bypasses the cap, because the cap is now read from the always-available local tracker. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_upload_tracker.py | 17 +++++++++++++++++ winnow/executor.py | 10 ++++++---- winnow/jobs.py | 11 ++++------- winnow/upload_tracker.py | 11 +++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/test_upload_tracker.py b/tests/test_upload_tracker.py index 070a591..b5b6acd 100644 --- a/tests/test_upload_tracker.py +++ b/tests/test_upload_tracker.py @@ -102,6 +102,23 @@ def test_remove_frigate_file_does_not_unmark_asset(): assert filter_already_uploaded(["asset-a1"]) == [] +def test_get_tracked_frigate_file_count_zero_when_empty(): + from winnow.upload_tracker import get_tracked_frigate_file_count + assert get_tracked_frigate_file_count("Alice") == 0 + + +def test_get_tracked_frigate_file_count_counts_only_mapped(): + """Only files explicitly recorded via record_frigate_file count toward the cap.""" + from winnow.upload_tracker import get_tracked_frigate_file_count, mark_uploaded, record_frigate_file + mark_uploaded("asset-a", person_name="Alice") + mark_uploaded("asset-b", person_name="Alice") + record_frigate_file("Alice", "Alice-1000.webp", "asset-a") + # asset-b is uploaded but not yet mapped — does not count + assert get_tracked_frigate_file_count("Alice") == 1 + record_frigate_file("Alice", "Alice-1001.webp", "asset-b") + assert get_tracked_frigate_file_count("Alice") == 2 + + def test_get_lowest_quality_mapped_file_none_when_empty(): from winnow.upload_tracker import get_lowest_quality_mapped_file assert get_lowest_quality_mapped_file("Alice") is None diff --git a/winnow/executor.py b/winnow/executor.py index 5e2f4b3..fbf6e40 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -19,6 +19,7 @@ from .immich_api import fetch_face_data, fetch_full_image from .log_config import console from .upload_tracker import ( get_lowest_quality_mapped_file, + get_tracked_frigate_file_count, mark_rejected, mark_uploaded, record_frigate_file, @@ -316,9 +317,11 @@ def upload_to_frigate(jobs: list[dict]) -> None: person_uploaded = 0 person_failed = 0 - known_frigate_files: set[str] = set(get_frigate_person_files(name) or []) - known_frigate_files_at_start = set(known_frigate_files) - effective_count = len(known_frigate_files) + # Snapshot live Frigate files for post-upload reconciliation diff only. + # effective_count is sourced from the tracker (mapped files) so that + # manually-added Frigate files don't consume winnow's managed quota. + known_frigate_files_at_start: set[str] = set(get_frigate_person_files(name) or []) + effective_count = get_tracked_frigate_file_count(name) quality_replacement = job.get("config", {}).get("quality_replacement", False) actually_uploaded: list[tuple[str, str | None]] = [] @@ -353,7 +356,6 @@ def upload_to_frigate(jobs: list[dict]) -> None: ) if delete_frigate_person_files(name, [worst_frigate_file]): remove_frigate_file(name, worst_frigate_file) - known_frigate_files.discard(worst_frigate_file) effective_count -= 1 else: logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement") diff --git a/winnow/jobs.py b/winnow/jobs.py index 369b566..fda3fd8 100644 --- a/winnow/jobs.py +++ b/winnow/jobs.py @@ -283,14 +283,11 @@ def auto_configure(people: list[dict]) -> list[dict]: rprint(f" [dim]Skipping {name} (0 new images after dedup).[/dim]") continue - # Enforce MAX_AUTO_IMAGES as a lifetime cap per person. - # Priority: live Frigate count → last cached Frigate count → local uploaded count. + # Enforce MAX_AUTO_IMAGES against the tracked file count only. + # Manually-added Frigate files are invisible to this cap so users can + # curate their own files without shrinking winnow's managed quota. person_summary = upload_summary.get(name, {}) - if frigate_counts is not None: - already_uploaded = frigate_counts.get(name, 0) - else: - fc = person_summary.get("frigate_count") - already_uploaded = fc if fc is not None else person_summary.get("uploaded", 0) + already_uploaded = len(person_summary.get("frigate_files", {})) capacity = Config.MAX_AUTO_IMAGES - already_uploaded if capacity <= 0: if not Config.QUALITY_REPLACEMENT: diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index ccaf97d..3b6de12 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -147,6 +147,17 @@ def remove_frigate_file(person_name: str, frigate_filename: str) -> None: logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})") +def get_tracked_frigate_file_count(person_name: str) -> int: + """Return the number of Frigate training files winnow has mapped for this person. + + Used as the cap baseline so that manually-added Frigate files do not + consume slots from winnow's managed quota. + """ + data = _load(UPLOAD_TRACKER_FILE) + entry = _migrate_entry(data.get("by_person", {}).get(person_name, {})) + return len(entry["frigate_files"]) + + def get_lowest_quality_mapped_file(person_name: str) -> tuple[str, str, float] | None: """Return (frigate_filename, asset_id, score) for the mapped file with the lowest confidence score, or None if no mapped files with known scores exist."""