diff --git a/.env.example b/.env.example index 9aa6d33..3daa6d6 100644 --- a/.env.example +++ b/.env.example @@ -8,19 +8,16 @@ FRIGATE_URL=http://192.168.1.10:5000 # Set AUTO_MODE=true to force auto mode even in an interactive terminal. # AUTO_MODE=true # VERBOSE=true # Enable DEBUG-level console output (log file is always DEBUG) -# TRAINING_MODE: face = upload to Frigate face recognition API -# object = save crops to output dir for manual Frigate placement -TRAINING_MODE=face -# STRATEGY: auto = objective diversity (recommended), standard = 30 imgs, broad = 100 imgs -STRATEGY=auto +# STRATEGY: adaptive = embedding diversity (recommended), standard = 30 imgs, broad = 100 imgs +STRATEGY=adaptive # LIMIT=50 # Custom image count; overrides STRATEGY preset -# OBJECT_CLASS=dog # Object label for object mode (e.g. dog, cat, car) # ── People Filtering ────────────────────────────────────────────────────────── # ONLY_PEOPLE=John,Jane # Comma-separated; process only these people # SKIP_PEOPLE=Unknown # Comma-separated; skip these people -# MIN_FACE_COUNT=5 # Skip people with fewer than N assets in Immich +# MIN_FACE_COUNT=3 # Skip people with fewer than N assets in Immich (default: 3) # YEARS_FILTER=10 # Only include images from the last N years (default: 10) +# MERGE_DUPLICATE_PEOPLE=false # Merge duplicate Immich person records permanently (default: false — warn and skip) # ── Image Quality ───────────────────────────────────────────────────────────── # MIN_FACE_WIDTH=90 # Minimum face width in pixels (default: 90, guarantees ≥8,100px crop) @@ -29,22 +26,21 @@ STRATEGY=auto # USE_FULL_RESOLUTION=true # Use full-res images vs thumbnails (default: true) # MIN_CONFIDENCE=0.7 # Minimum face detection confidence (default: 0.7) # BLUR_THRESHOLD=120.0 # Laplacian blur threshold; lower = accept more blur (default: 120.0) -# MAX_AUTO_IMAGES=80 # Hard cap on auto-diversity selection (default: 80) +# MAX_AUTO_IMAGES=20 # Hard cap on auto-diversity selection (default: 20) # QUALITY_REPLACEMENT=true # At cap, replace a weaker tracked image with a better candidate (default: true) -# FRIGATE_SCORE_CEILING=0.0 # Skip uploads already well-covered (pre-upload score > ceiling = redundant; 0 = disabled; requires at least one prior run) +# FRIGATE_SCORE_CEILING= # Below-cap novelty gate: unset = dynamic (default), 0 = disabled, e.g. 0.85 = fixed ceiling # ENABLE_FRIGATE_SCORES=true # Call Frigate's recognize endpoint pre-upload to store diversity scores (default: true; adds ~200ms per upload) # ── Caching & Models ────────────────────────────────────────────────────────── # FORCE_CPU=true # Disable GPU, fall back to CPU # ENABLE_CACHE=false # Disable embedding cache (default: true) CACHE_DIR=/app/.if_cache -HF_HOME=/models/huggingface INSIGHTFACE_HOME=/models/.insightface # ── Tracker overrides (one-shot — remove after use) ─────────────────────────── # DRY_RUN=true # Preview selection without downloading/uploading # RETRY_REJECTED=true # Re-attempt previously rejected images -# RESET_PERSON=John # Clear uploaded+rejected history for one person +# RESET_PERSON=John # Clear uploaded+rejected history for one person (use * for all) # ── Scheduling ──────────────────────────────────────────────────────────────── # CRON_SCHEDULE controls container lifetime: diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b8aa5..7f9687c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.5.0] - 2026-06-14 + +### Changed + +- **SQLite upload tracker**: `upload_tracker.py` is fully rewritten on top of SQLite (stdlib `sqlite3`). The JSON pair (`frigate_uploaded_ids.json` / `frigate_rejected_ids.json`) is replaced by a single `winnow_tracker.db` (WAL journal, `check_same_thread=False`). Existing JSON files are migrated atomically on first run and renamed to `.json.bak`. No user action required; the tracker API (`mark_uploaded`, `mark_rejected`, `filter_already_uploaded`, `get_person_summary`, etc.) is unchanged. + +- **Config lazy singleton**: `_Config` now uses `__getattr__` to defer all I/O until the first attribute access. `load_dotenv()` no longer runs at module import time — it runs on the first access to any `Config` attribute. Empty-string env vars (`IMMICH_URL=`, `OUTPUT_DIR=`) are now correctly distinguished from unset ones so a `.env` file value never silently overrides an explicit `""` set in the environment. `Config.reset()` clears the loaded state for clean test isolation. + +- **Reconcile module extracted**: `reconcile_frigate_mappings` and `enrich_asset_with_face_data` are extracted from `executor.py` into a new `winnow/reconcile.py` module. No behaviour change; reduces `executor.py` length and clarifies responsibility boundaries. + +- **Single lockfile**: `pyproject-gpu.toml`, `pyproject-cpu.toml`, `pyproject-rocm.toml`, `pyproject-intel.toml` and their separate lockfiles are removed. GPU/ROCm/Intel/CPU variant deps are now declared as `[project.optional-dependencies]` extras in `pyproject.toml` with `[tool.uv] conflicts` for mutual exclusion. A single `uv.lock` covers all variants. The Dockerfile selects the correct extra via `uv sync --extra $VARIANT`. + +- **Ubuntu base bumped**: amd64 GPU base updated from `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04` to `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04`. amd64 ROCm and CPU bases updated from Ubuntu 22.04 to Ubuntu 26.04. arm64 bases remain Ubuntu 24.04. + +### Fixed + +- **Frigate API unreachable at upload start no longer crashes reconciliation**: when the Frigate `GET /api/faces` call fails at upload start, reconciliation is now skipped entirely for that batch (`_skip_reconcile = True`). Previously, falling back to the tracker's known filenames as the pre-upload baseline caused the `> target` guard to fire on unmapped manual files, silently dropping all mappings. + +- **Polling `== target` guards against wrong-file mapping**: the reconcile poll loop now breaks on `len(new_files) == target` and sets an "external upload detected" flag when `> target`. The old `>= target` break would have proceeded with an incorrect file set when a concurrent external upload was present, causing wrong asset-ID mappings. The poll loop now also exits early on `> target` rather than exhausting all four retry intervals (up to 15 s wasted per person with a concurrent external uploader). + +- **`auto_cap` post-selection truncation removed**: the diversity selector now receives the correct upper bound (`capacity` or `min(limit, capacity)`) directly instead of selecting up to `MAX_AUTO_IMAGES` and then silently truncating the result list. The old approach produced a selection biased toward the first `capacity` items in embedding space rather than the globally optimal diverse subset. + +- **Dockerfile unknown VARIANT now fails loudly**: added an explicit `elif [ "$VARIANT" = "gpu" ]` branch and an `else … exit 1` for unrecognised values. Previously, any unknown variant silently fell through to the `cpu` branch. + +- **JSON migration partial-rename data loss**: if the rename of one of the two JSON files failed (e.g. a `PermissionError`), the other file's data was committed to SQLite but the `COUNT(*) > 0` guard on the next run would skip re-migration of the remaining file, permanently losing its data. The guard is removed (idempotent `INSERT OR IGNORE` makes re-running safe). Each rename is now wrapped in its own `try/except OSError` so a failure on one file is logged and does not prevent the other from completing. + +- **SQL column allowlist in `_pick_mapped_file`**: the `score_col` f-string interpolation into SQL is now guarded by a `frozenset` allowlist at the function boundary, raising `ValueError` on any value outside `{"blur_score", "frigate_score"}`. + +- **`load_dotenv` no longer runs at import time**: moving `load_dotenv()` to the first line of `_load()` prevents side-effects during module import (which could interfere with test environment setup) and makes the load order deterministic relative to `os.environ` overrides. + +- **Empty-string env var priority fix**: `if self.IMMICH_URL or …` treated `IMMICH_URL=""` as falsy and silently fell through to the config file. Changed to `if self.IMMICH_URL is None` so an empty-string explicit env var is respected. + +### Added + +- **Diversity test suite expanded** (PR #11): 33 new tests covering k-medoids clustering, farthest-point sampling, adaptive threshold computation, near-duplicate deduplication, and time-spread selection. Total: 93 tests (was 60). + +- **Known-limitation annotations** (PR #12): `TODO(frigate-api)` comments placed at each FIFO-ordering assumption, manual-file-invisibility note, and async-rebuild limitation in `executor.py` and `reconcile.py`. These mark spots where a richer Frigate API would allow a deeper fix. + ## [0.4.11] - 2026-06-14 ### Removed diff --git a/pyproject.toml b/pyproject.toml index d5ec3eb..5ccbf97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.4.11" +version = "0.5.0" description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition." license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/uv.lock b/uv.lock index 009739f..22056b0 100644 --- a/uv.lock +++ b/uv.lock @@ -96,7 +96,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly" }, + { name = "humanfriendly", marker = "platform_machine != 's390x'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -128,7 +128,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "platform_machine != 's390x' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -322,7 +322,7 @@ name = "nvidia-cublas-cu12" version = "12.9.2.10" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc-cu12" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine != 's390x'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f7/a2/c96163a0fff1839c0c9548bbdeae7b853b867009e33b9b9264adc238b1cf/nvidia_cublas_cu12-12.9.2.10-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:5572131a59c3eebeeb1c4c8144f772d49372c20124916e072a0e3fc30df421d5", size = 575012079, upload-time = "2026-04-08T18:51:47.303Z" }, @@ -345,7 +345,7 @@ name = "nvidia-cudnn-cu12" version = "9.23.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine != 's390x'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/10/30/ecca1c8194c8077c4b57a3d96b56d96f15852551b01b919bae6429d92218/nvidia_cudnn_cu12-9.23.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6dbc18f05aab2a323a4ffd43d985410608f7db7db9a8596e189cddbd3e527441", size = 778220760, upload-time = "2026-06-09T19:38:19.281Z" }, @@ -415,10 +415,10 @@ name = "onnxruntime-gpu" version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, + { name = "flatbuffers", marker = "platform_machine != 's390x'" }, + { name = "numpy", marker = "platform_machine != 's390x'" }, + { name = "packaging", marker = "platform_machine != 's390x'" }, + { name = "protobuf", marker = "platform_machine != 's390x'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/dd/97/fe8979f44b9275654b42f7bb556e30789b71a1b22998c83b540df2b1b774/onnxruntime_gpu-1.26.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfda2fad535595bfc3e570eb588092717711dcb2957656d814695e0c9ceb1508", size = 276974871, upload-time = "2026-05-08T19:15:58.052Z" }, @@ -434,11 +434,11 @@ name = "onnxruntime-openvino" version = "1.24.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "flatbuffers", marker = "platform_machine != 's390x'" }, + { name = "numpy", marker = "platform_machine != 's390x'" }, + { name = "packaging", marker = "platform_machine != 's390x'" }, + { name = "protobuf", marker = "platform_machine != 's390x'" }, + { name = "sympy", marker = "platform_machine != 's390x'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/08/07/f225999919f56506b603aaa3ff837ad563ab26f86906ed7fa7e5abcd849e/onnxruntime_openvino-1.24.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:2c3bb73e68ac27f4891af8a595c1faf574ec68b772e6583c90a0b997a1822782", size = 84433183, upload-time = "2026-02-26T13:44:50.254Z" }, @@ -450,12 +450,12 @@ name = "onnxruntime-rocm" version = "1.22.2.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "platform_machine != 's390x'" }, + { name = "flatbuffers", marker = "platform_machine != 's390x'" }, + { name = "numpy", marker = "platform_machine != 's390x'" }, + { name = "packaging", marker = "platform_machine != 's390x'" }, + { name = "protobuf", marker = "platform_machine != 's390x'" }, + { name = "sympy", marker = "platform_machine != 's390x'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/33/d0/e6f011c8e01853a8a4a56b48b1421257d89767fa76e8a273eeb2c54eb725/onnxruntime_rocm-1.22.2.post1-cp313-cp313-manylinux_2_35_x86_64.whl", hash = "sha256:19b56e9e41da3c7042dc97223ef46976ed8bbdf0ffe43646129f773647794b44", size = 217937775, upload-time = "2025-09-11T16:40:34.403Z" }, @@ -811,7 +811,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "platform_machine != 's390x'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -862,7 +862,7 @@ wheels = [ [[package]] name = "winnow" -version = "0.4.11" +version = "0.5.0" source = { editable = "." } dependencies = [ { name = "croniter" },