From 405413490bacf995dbff42be8543b80bf256e2b2 Mon Sep 17 00:00:00 2001 From: Holden Date: Sat, 13 Jun 2026 19:58:08 +0000 Subject: [PATCH] fix: RESET_PERSON now deletes managed Frigate files before clearing tracker Previously reset_person wiped the local tracker but left existing Frigate training files as orphans, causing the next run to upload a full new batch on top of them. Now deletes all winnow-managed files from Frigate first so the next run starts truly clean. Manually-added Frigate files are never touched. Also fixes a spurious warning when FRIGATE_URL is unset: the deletion step is now skipped at info level rather than logging a misleading error. Moves the deferred import to top-level and eliminates a double disk read. Bumps to 0.4.1. Also fixes ruff lint violations in executor.py (import sort, line length) and promotes the "winnow only touches files it uploaded" callout to the README intro. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 +++++++ README.md | 4 +++- pyproject.toml | 2 +- winnow/executor.py | 10 ++++++++-- winnow/upload_tracker.py | 39 ++++++++++++++++++++++++++++++++------- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 484bcd9..4e25717 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.1] - 2026-06-13 + +### Fixed + +- **`RESET_PERSON` no longer creates duplicate Frigate files**: previously, resetting a person only wiped the local tracker — existing Frigate training files were left as unmanaged orphans, causing the next run to upload a full new batch on top of them. `reset_person` now deletes all winnow-managed files for that person from Frigate before clearing the tracker. Manually-added Frigate files are unaffected. +- **No spurious warning when `FRIGATE_URL` is unset and `RESET_PERSON` is used**: the deletion step is now skipped silently at info level rather than logging a misleading "could not delete" warning. + ## [0.4.0] - 2026-06-13 ### Added diff --git a/README.md b/README.md index 5641f39..4c9af18 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ Frigate's face recognition is only as good as its training data — and the key quality metric is **diversity**, not volume. A hundred photos from the same week teach the model one lighting condition. What you need is a spread: different years, different angles, different lighting, different contexts. Your photo library already has that data. winnow finds and delivers the right subset automatically. +> **winnow only touches files it uploaded.** Faces added to Frigate manually through its UI are never deleted, replaced, or modified — not by quality replacement, not by `RESET_PERSON`, not by stale cleanup. If you have a curated training set you want to keep, it is safe. + --- ## How It Works @@ -222,7 +224,7 @@ In scheduled mode the process (and loaded models) stays resident between runs. T | :--- | :--- | :--- | | `DRY_RUN` | `false` | Preview selection without downloading or uploading | | `RETRY_REJECTED` | `false` | Re-attempt assets previously rejected by Frigate | -| `RESET_PERSON` | *(unset)* | Clear upload and rejection history for one person by name | +| `RESET_PERSON` | *(unset)* | Clear upload history for one person and delete their winnow-managed Frigate training files so the next run starts fresh. Manually added Frigate files are never touched | ### Scheduling diff --git a/pyproject.toml b/pyproject.toml index e65e885..6765705 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.4.0" +version = "0.4.1" description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification." license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/winnow/executor.py b/winnow/executor.py index c54d10a..e6279a9 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -13,7 +13,12 @@ from rich import print as rprint from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn from .config import Config, get_headers -from .frigate_api import delete_frigate_person_files, get_all_frigate_person_files, get_frigate_person_files, recognize_face +from .frigate_api import ( + delete_frigate_person_files, + get_all_frigate_person_files, + get_frigate_person_files, + recognize_face, +) from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image from .log_config import console @@ -479,7 +484,8 @@ def upload_to_frigate(jobs: list[dict]) -> None: remove_frigate_file(name, target_frigate_file) person_has_fscores = has_frigate_scores(name) effective_count -= 1 - min_quality_score_for_slot = None # clear any blur-mode slot floor — Frigate uses a different score metric + # clear any blur-mode slot floor — Frigate uses a different score metric + min_quality_score_for_slot = None else: logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement") failed_deletes.add(target_frigate_file) diff --git a/winnow/upload_tracker.py b/winnow/upload_tracker.py index a6d84e3..28f8a1a 100644 --- a/winnow/upload_tracker.py +++ b/winnow/upload_tracker.py @@ -29,8 +29,11 @@ Frigate's UI are never mapped here and are never touched by quality replacement. import json import logging +import os from pathlib import Path +from .frigate_api import delete_frigate_person_files + logger = logging.getLogger(__name__) UPLOAD_TRACKER_FILE = "frigate_uploaded_ids.json" @@ -297,19 +300,41 @@ def update_frigate_count(person_name: str, count: int) -> None: def reset_person(person_name: str) -> None: - """Remove all uploaded and rejected records for a given person.""" - for filename in (UPLOAD_TRACKER_FILE, REJECT_TRACKER_FILE): - data = _load(filename) + """Remove all uploaded and rejected records for a given person. + + Also deletes winnow-managed Frigate training files so the next run starts + clean rather than uploading on top of orphaned files. Manually-added Frigate + files (not in frigate_files) are never touched. Proceeds with tracker reset + even if Frigate is unreachable. + """ + upload_data = _load(UPLOAD_TRACKER_FILE) + entry = _migrate_entry(upload_data.get("by_person", {}).get(person_name, {})) + frigate_filenames = list(entry.get("frigate_files", {}).keys()) + if frigate_filenames: + if not os.environ.get("FRIGATE_URL", "").strip(): + logger.info(f"FRIGATE_URL not set — skipping Frigate file deletion for {person_name}") + elif delete_frigate_person_files(person_name, frigate_filenames): + logger.info(f"Deleted {len(frigate_filenames)} Frigate file(s) for {person_name}") + else: + logger.warning(f"Could not delete Frigate files for {person_name} — tracker reset proceeding anyway") + + changed = False + tracker_files = ((UPLOAD_TRACKER_FILE, upload_data), (REJECT_TRACKER_FILE, _load(REJECT_TRACKER_FILE))) + for filename, data in tracker_files: flat_key = _flat_key(filename) by_person = data.get("by_person", {}) - entry = by_person.pop(person_name, None) - if entry is not None: - person_ids = set(_get_ids(entry)) + tracker_entry = by_person.pop(person_name, None) + if tracker_entry is not None: + person_ids = set(_get_ids(tracker_entry)) flat = set(data.get(flat_key, [])) - person_ids data[flat_key] = sorted(flat) data["by_person"] = by_person _save(filename, data) - logger.info(f"Reset tracking data for {person_name}") + changed = True + if changed: + logger.info(f"Reset tracking data for {person_name}") + else: + logger.debug(f"reset_person: no tracking data found for {person_name}") def get_person_summary() -> dict[str, dict]: