From a4571f59f6952edfb728c10c02411a96ad940037 Mon Sep 17 00:00:00 2001 From: Holden Date: Sun, 14 Jun 2026 21:01:41 +0000 Subject: [PATCH] =?UTF-8?q?chore:=20release=200.5.0=20=E2=80=94=20version?= =?UTF-8?q?=20bump,=20changelog,=20clean=20up=20.env.example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 18 +++++++----------- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 9aa6d33..3daa6d6 100644 --- a/.env.example +++ b/.env.example @@ -8,19 +8,16 @@ FRIGATE_URL=http://192.168.1.10:5000 # Set AUTO_MODE=true to force auto mode even in an interactive terminal. # AUTO_MODE=true # VERBOSE=true # Enable DEBUG-level console output (log file is always DEBUG) -# TRAINING_MODE: face = upload to Frigate face recognition API -# object = save crops to output dir for manual Frigate placement -TRAINING_MODE=face -# STRATEGY: auto = objective diversity (recommended), standard = 30 imgs, broad = 100 imgs -STRATEGY=auto +# STRATEGY: adaptive = embedding diversity (recommended), standard = 30 imgs, broad = 100 imgs +STRATEGY=adaptive # LIMIT=50 # Custom image count; overrides STRATEGY preset -# OBJECT_CLASS=dog # Object label for object mode (e.g. dog, cat, car) # ── People Filtering ────────────────────────────────────────────────────────── # ONLY_PEOPLE=John,Jane # Comma-separated; process only these people # SKIP_PEOPLE=Unknown # Comma-separated; skip these people -# MIN_FACE_COUNT=5 # Skip people with fewer than N assets in Immich +# MIN_FACE_COUNT=3 # Skip people with fewer than N assets in Immich (default: 3) # YEARS_FILTER=10 # Only include images from the last N years (default: 10) +# MERGE_DUPLICATE_PEOPLE=false # Merge duplicate Immich person records permanently (default: false — warn and skip) # ── Image Quality ───────────────────────────────────────────────────────────── # MIN_FACE_WIDTH=90 # Minimum face width in pixels (default: 90, guarantees ≥8,100px crop) @@ -29,22 +26,21 @@ STRATEGY=auto # USE_FULL_RESOLUTION=true # Use full-res images vs thumbnails (default: true) # MIN_CONFIDENCE=0.7 # Minimum face detection confidence (default: 0.7) # BLUR_THRESHOLD=120.0 # Laplacian blur threshold; lower = accept more blur (default: 120.0) -# MAX_AUTO_IMAGES=80 # Hard cap on auto-diversity selection (default: 80) +# MAX_AUTO_IMAGES=20 # Hard cap on auto-diversity selection (default: 20) # QUALITY_REPLACEMENT=true # At cap, replace a weaker tracked image with a better candidate (default: true) -# FRIGATE_SCORE_CEILING=0.0 # Skip uploads already well-covered (pre-upload score > ceiling = redundant; 0 = disabled; requires at least one prior run) +# FRIGATE_SCORE_CEILING= # Below-cap novelty gate: unset = dynamic (default), 0 = disabled, e.g. 0.85 = fixed ceiling # ENABLE_FRIGATE_SCORES=true # Call Frigate's recognize endpoint pre-upload to store diversity scores (default: true; adds ~200ms per upload) # ── Caching & Models ────────────────────────────────────────────────────────── # FORCE_CPU=true # Disable GPU, fall back to CPU # ENABLE_CACHE=false # Disable embedding cache (default: true) CACHE_DIR=/app/.if_cache -HF_HOME=/models/huggingface INSIGHTFACE_HOME=/models/.insightface # ── Tracker overrides (one-shot — remove after use) ─────────────────────────── # DRY_RUN=true # Preview selection without downloading/uploading # RETRY_REJECTED=true # Re-attempt previously rejected images -# RESET_PERSON=John # Clear uploaded+rejected history for one person +# RESET_PERSON=John # Clear uploaded+rejected history for one person (use * for all) # ── Scheduling ──────────────────────────────────────────────────────────────── # CRON_SCHEDULE controls container lifetime: diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b8aa5..7f9687c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.5.0] - 2026-06-14 + +### Changed + +- **SQLite upload tracker**: `upload_tracker.py` is fully rewritten on top of SQLite (stdlib `sqlite3`). The JSON pair (`frigate_uploaded_ids.json` / `frigate_rejected_ids.json`) is replaced by a single `winnow_tracker.db` (WAL journal, `check_same_thread=False`). Existing JSON files are migrated atomically on first run and renamed to `.json.bak`. No user action required; the tracker API (`mark_uploaded`, `mark_rejected`, `filter_already_uploaded`, `get_person_summary`, etc.) is unchanged. + +- **Config lazy singleton**: `_Config` now uses `__getattr__` to defer all I/O until the first attribute access. `load_dotenv()` no longer runs at module import time — it runs on the first access to any `Config` attribute. Empty-string env vars (`IMMICH_URL=`, `OUTPUT_DIR=`) are now correctly distinguished from unset ones so a `.env` file value never silently overrides an explicit `""` set in the environment. `Config.reset()` clears the loaded state for clean test isolation. + +- **Reconcile module extracted**: `reconcile_frigate_mappings` and `enrich_asset_with_face_data` are extracted from `executor.py` into a new `winnow/reconcile.py` module. No behaviour change; reduces `executor.py` length and clarifies responsibility boundaries. + +- **Single lockfile**: `pyproject-gpu.toml`, `pyproject-cpu.toml`, `pyproject-rocm.toml`, `pyproject-intel.toml` and their separate lockfiles are removed. GPU/ROCm/Intel/CPU variant deps are now declared as `[project.optional-dependencies]` extras in `pyproject.toml` with `[tool.uv] conflicts` for mutual exclusion. A single `uv.lock` covers all variants. The Dockerfile selects the correct extra via `uv sync --extra $VARIANT`. + +- **Ubuntu base bumped**: amd64 GPU base updated from `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04` to `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04`. amd64 ROCm and CPU bases updated from Ubuntu 22.04 to Ubuntu 26.04. arm64 bases remain Ubuntu 24.04. + +### Fixed + +- **Frigate API unreachable at upload start no longer crashes reconciliation**: when the Frigate `GET /api/faces` call fails at upload start, reconciliation is now skipped entirely for that batch (`_skip_reconcile = True`). Previously, falling back to the tracker's known filenames as the pre-upload baseline caused the `> target` guard to fire on unmapped manual files, silently dropping all mappings. + +- **Polling `== target` guards against wrong-file mapping**: the reconcile poll loop now breaks on `len(new_files) == target` and sets an "external upload detected" flag when `> target`. The old `>= target` break would have proceeded with an incorrect file set when a concurrent external upload was present, causing wrong asset-ID mappings. The poll loop now also exits early on `> target` rather than exhausting all four retry intervals (up to 15 s wasted per person with a concurrent external uploader). + +- **`auto_cap` post-selection truncation removed**: the diversity selector now receives the correct upper bound (`capacity` or `min(limit, capacity)`) directly instead of selecting up to `MAX_AUTO_IMAGES` and then silently truncating the result list. The old approach produced a selection biased toward the first `capacity` items in embedding space rather than the globally optimal diverse subset. + +- **Dockerfile unknown VARIANT now fails loudly**: added an explicit `elif [ "$VARIANT" = "gpu" ]` branch and an `else … exit 1` for unrecognised values. Previously, any unknown variant silently fell through to the `cpu` branch. + +- **JSON migration partial-rename data loss**: if the rename of one of the two JSON files failed (e.g. a `PermissionError`), the other file's data was committed to SQLite but the `COUNT(*) > 0` guard on the next run would skip re-migration of the remaining file, permanently losing its data. The guard is removed (idempotent `INSERT OR IGNORE` makes re-running safe). Each rename is now wrapped in its own `try/except OSError` so a failure on one file is logged and does not prevent the other from completing. + +- **SQL column allowlist in `_pick_mapped_file`**: the `score_col` f-string interpolation into SQL is now guarded by a `frozenset` allowlist at the function boundary, raising `ValueError` on any value outside `{"blur_score", "frigate_score"}`. + +- **`load_dotenv` no longer runs at import time**: moving `load_dotenv()` to the first line of `_load()` prevents side-effects during module import (which could interfere with test environment setup) and makes the load order deterministic relative to `os.environ` overrides. + +- **Empty-string env var priority fix**: `if self.IMMICH_URL or …` treated `IMMICH_URL=""` as falsy and silently fell through to the config file. Changed to `if self.IMMICH_URL is None` so an empty-string explicit env var is respected. + +### Added + +- **Diversity test suite expanded** (PR #11): 33 new tests covering k-medoids clustering, farthest-point sampling, adaptive threshold computation, near-duplicate deduplication, and time-spread selection. Total: 93 tests (was 60). + +- **Known-limitation annotations** (PR #12): `TODO(frigate-api)` comments placed at each FIFO-ordering assumption, manual-file-invisibility note, and async-rebuild limitation in `executor.py` and `reconcile.py`. These mark spots where a richer Frigate API would allow a deeper fix. + ## [0.4.11] - 2026-06-14 ### Removed diff --git a/pyproject.toml b/pyproject.toml index d5ec3eb..5ccbf97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.4.11" +version = "0.5.0" description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition." license = "AGPL-3.0-or-later" requires-python = ">=3.13"