Merge pull request #15 from sudolulo/release/0.5.0

chore: release 0.5.0
2026-06-14 17:03:23 -04:00
parent 51ea7bd43c fa4fc8984c
commit d6cb9c6ab8
4 changed files with 67 additions and 33 deletions
@@ -8,19 +8,16 @@ FRIGATE_URL=http://192.168.1.10:5000
 # Set AUTO_MODE=true to force auto mode even in an interactive terminal.
 # AUTO_MODE=true
 # VERBOSE=true                # Enable DEBUG-level console output (log file is always DEBUG)
-# TRAINING_MODE: face = upload to Frigate face recognition API
-#                object = save crops to output dir for manual Frigate placement
-TRAINING_MODE=face
-# STRATEGY: auto = objective diversity (recommended), standard = 30 imgs, broad = 100 imgs
-STRATEGY=auto
+# STRATEGY: adaptive = embedding diversity (recommended), standard = 30 imgs, broad = 100 imgs
+STRATEGY=adaptive
 # LIMIT=50                    # Custom image count; overrides STRATEGY preset
-# OBJECT_CLASS=dog            # Object label for object mode (e.g. dog, cat, car)

 # ── People Filtering ──────────────────────────────────────────────────────────
 # ONLY_PEOPLE=John,Jane       # Comma-separated; process only these people
 # SKIP_PEOPLE=Unknown         # Comma-separated; skip these people
-# MIN_FACE_COUNT=5            # Skip people with fewer than N assets in Immich
+# MIN_FACE_COUNT=3            # Skip people with fewer than N assets in Immich (default: 3)
 # YEARS_FILTER=10             # Only include images from the last N years (default: 10)
+# MERGE_DUPLICATE_PEOPLE=false  # Merge duplicate Immich person records permanently (default: false — warn and skip)

 # ── Image Quality ─────────────────────────────────────────────────────────────
 # MIN_FACE_WIDTH=90           # Minimum face width in pixels (default: 90, guarantees ≥8,100px crop)
@@ -29,22 +26,21 @@ STRATEGY=auto
 # USE_FULL_RESOLUTION=true    # Use full-res images vs thumbnails (default: true)
 # MIN_CONFIDENCE=0.7          # Minimum face detection confidence (default: 0.7)
 # BLUR_THRESHOLD=120.0        # Laplacian blur threshold; lower = accept more blur (default: 120.0)
-# MAX_AUTO_IMAGES=80          # Hard cap on auto-diversity selection (default: 80)
+# MAX_AUTO_IMAGES=20          # Hard cap on auto-diversity selection (default: 20)
 # QUALITY_REPLACEMENT=true    # At cap, replace a weaker tracked image with a better candidate (default: true)
-# FRIGATE_SCORE_CEILING=0.0   # Skip uploads already well-covered (pre-upload score > ceiling = redundant; 0 = disabled; requires at least one prior run)
+# FRIGATE_SCORE_CEILING=      # Below-cap novelty gate: unset = dynamic (default), 0 = disabled, e.g. 0.85 = fixed ceiling
 # ENABLE_FRIGATE_SCORES=true  # Call Frigate's recognize endpoint pre-upload to store diversity scores (default: true; adds ~200ms per upload)

 # ── Caching & Models ──────────────────────────────────────────────────────────
 # FORCE_CPU=true              # Disable GPU, fall back to CPU
 # ENABLE_CACHE=false          # Disable embedding cache (default: true)
 CACHE_DIR=/app/.if_cache
-HF_HOME=/models/huggingface
 INSIGHTFACE_HOME=/models/.insightface

 # ── Tracker overrides (one-shot — remove after use) ───────────────────────────
 # DRY_RUN=true                # Preview selection without downloading/uploading
 # RETRY_REJECTED=true         # Re-attempt previously rejected images
-# RESET_PERSON=John           # Clear uploaded+rejected history for one person
+# RESET_PERSON=John           # Clear uploaded+rejected history for one person (use * for all)

 # ── Scheduling ────────────────────────────────────────────────────────────────
 # CRON_SCHEDULE controls container lifetime:
@@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased]

+## [0.5.0] - 2026-06-14
+
+### Changed
+
+- **SQLite upload tracker**: `upload_tracker.py` is fully rewritten on top of SQLite (stdlib `sqlite3`). The JSON pair (`frigate_uploaded_ids.json` / `frigate_rejected_ids.json`) is replaced by a single `winnow_tracker.db` (WAL journal, `check_same_thread=False`). Existing JSON files are migrated atomically on first run and renamed to `.json.bak`. No user action required; the tracker API (`mark_uploaded`, `mark_rejected`, `filter_already_uploaded`, `get_person_summary`, etc.) is unchanged.
+
+- **Config lazy singleton**: `_Config` now uses `__getattr__` to defer all I/O until the first attribute access. `load_dotenv()` no longer runs at module import time — it runs on the first access to any `Config` attribute. Empty-string env vars (`IMMICH_URL=`, `OUTPUT_DIR=`) are now correctly distinguished from unset ones so a `.env` file value never silently overrides an explicit `""` set in the environment. `Config.reset()` clears the loaded state for clean test isolation.
+
+- **Reconcile module extracted**: `reconcile_frigate_mappings` and `enrich_asset_with_face_data` are extracted from `executor.py` into a new `winnow/reconcile.py` module. No behaviour change; reduces `executor.py` length and clarifies responsibility boundaries.
+
+- **Single lockfile**: `pyproject-gpu.toml`, `pyproject-cpu.toml`, `pyproject-rocm.toml`, `pyproject-intel.toml` and their separate lockfiles are removed. GPU/ROCm/Intel/CPU variant deps are now declared as `[project.optional-dependencies]` extras in `pyproject.toml` with `[tool.uv] conflicts` for mutual exclusion. A single `uv.lock` covers all variants. The Dockerfile selects the correct extra via `uv sync --extra $VARIANT`.
+
+- **Ubuntu base bumped**: amd64 GPU base updated from `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04` to `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04`. amd64 ROCm and CPU bases updated from Ubuntu 22.04 to Ubuntu 26.04. arm64 bases remain Ubuntu 24.04.
+
+### Fixed
+
+- **Frigate API unreachable at upload start no longer crashes reconciliation**: when the Frigate `GET /api/faces` call fails at upload start, reconciliation is now skipped entirely for that batch (`_skip_reconcile = True`). Previously, falling back to the tracker's known filenames as the pre-upload baseline caused the `> target` guard to fire on unmapped manual files, silently dropping all mappings.
+
+- **Polling `== target` guards against wrong-file mapping**: the reconcile poll loop now breaks on `len(new_files) == target` and sets an "external upload detected" flag when `> target`. The old `>= target` break would have proceeded with an incorrect file set when a concurrent external upload was present, causing wrong asset-ID mappings. The poll loop now also exits early on `> target` rather than exhausting all four retry intervals (up to 15 s wasted per person with a concurrent external uploader).
+
+- **`auto_cap` post-selection truncation removed**: the diversity selector now receives the correct upper bound (`capacity` or `min(limit, capacity)`) directly instead of selecting up to `MAX_AUTO_IMAGES` and then silently truncating the result list. The old approach produced a selection biased toward the first `capacity` items in embedding space rather than the globally optimal diverse subset.
+
+- **Dockerfile unknown VARIANT now fails loudly**: added an explicit `elif [ "$VARIANT" = "gpu" ]` branch and an `else … exit 1` for unrecognised values. Previously, any unknown variant silently fell through to the `cpu` branch.
+
+- **JSON migration partial-rename data loss**: if the rename of one of the two JSON files failed (e.g. a `PermissionError`), the other file's data was committed to SQLite but the `COUNT(*) > 0` guard on the next run would skip re-migration of the remaining file, permanently losing its data. The guard is removed (idempotent `INSERT OR IGNORE` makes re-running safe). Each rename is now wrapped in its own `try/except OSError` so a failure on one file is logged and does not prevent the other from completing.
+
+- **SQL column allowlist in `_pick_mapped_file`**: the `score_col` f-string interpolation into SQL is now guarded by a `frozenset` allowlist at the function boundary, raising `ValueError` on any value outside `{"blur_score", "frigate_score"}`.
+
+- **`load_dotenv` no longer runs at import time**: moving `load_dotenv()` to the first line of `_load()` prevents side-effects during module import (which could interfere with test environment setup) and makes the load order deterministic relative to `os.environ` overrides.
+
+- **Empty-string env var priority fix**: `if self.IMMICH_URL or …` treated `IMMICH_URL=""` as falsy and silently fell through to the config file. Changed to `if self.IMMICH_URL is None` so an empty-string explicit env var is respected.
+
+### Added
+
+- **Diversity test suite expanded** (PR #11): 33 new tests covering k-medoids clustering, farthest-point sampling, adaptive threshold computation, near-duplicate deduplication, and time-spread selection. Total: 93 tests (was 60).
+
+- **Known-limitation annotations** (PR #12): `TODO(frigate-api)` comments placed at each FIFO-ordering assumption, manual-file-invisibility note, and async-rebuild limitation in `executor.py` and `reconcile.py`. These mark spots where a richer Frigate API would allow a deeper fix.
+
 ## [0.4.11] - 2026-06-14

 ### Removed
@@ -1,6 +1,6 @@
 [project]
 name = "winnow"
-version = "0.4.11"
+version = "0.5.0"
 description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
 license = "AGPL-3.0-or-later"
 requires-python = ">=3.13"
@@ -96,7 +96,7 @@ name = "coloredlogs"
 version = "15.0.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "humanfriendly" },
+    { name = "humanfriendly", marker = "platform_machine != 's390x'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" }
 wheels = [
@@ -128,7 +128,7 @@ name = "humanfriendly"
 version = "10.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyreadline3", marker = "sys_platform == 'win32'" },
+    { name = "pyreadline3", marker = "platform_machine != 's390x' and sys_platform == 'win32'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" }
 wheels = [
@@ -322,7 +322,7 @@ name = "nvidia-cublas-cu12"
 version = "12.9.2.10"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cuda-nvrtc-cu12" },
+    { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine != 's390x'" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/f7/a2/c96163a0fff1839c0c9548bbdeae7b853b867009e33b9b9264adc238b1cf/nvidia_cublas_cu12-12.9.2.10-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:5572131a59c3eebeeb1c4c8144f772d49372c20124916e072a0e3fc30df421d5", size = 575012079, upload-time = "2026-04-08T18:51:47.303Z" },
@@ -345,7 +345,7 @@ name = "nvidia-cudnn-cu12"
 version = "9.23.1.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12" },
+    { name = "nvidia-cublas-cu12", marker = "platform_machine != 's390x'" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/10/30/ecca1c8194c8077c4b57a3d96b56d96f15852551b01b919bae6429d92218/nvidia_cudnn_cu12-9.23.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6dbc18f05aab2a323a4ffd43d985410608f7db7db9a8596e189cddbd3e527441", size = 778220760, upload-time = "2026-06-09T19:38:19.281Z" },
@@ -415,10 +415,10 @@ name = "onnxruntime-gpu"
 version = "1.26.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "flatbuffers" },
-    { name = "numpy" },
-    { name = "packaging" },
-    { name = "protobuf" },
+    { name = "flatbuffers", marker = "platform_machine != 's390x'" },
+    { name = "numpy", marker = "platform_machine != 's390x'" },
+    { name = "packaging", marker = "platform_machine != 's390x'" },
+    { name = "protobuf", marker = "platform_machine != 's390x'" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/dd/97/fe8979f44b9275654b42f7bb556e30789b71a1b22998c83b540df2b1b774/onnxruntime_gpu-1.26.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfda2fad535595bfc3e570eb588092717711dcb2957656d814695e0c9ceb1508", size = 276974871, upload-time = "2026-05-08T19:15:58.052Z" },
@@ -434,11 +434,11 @@ name = "onnxruntime-openvino"
 version = "1.24.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "flatbuffers" },
-    { name = "numpy" },
-    { name = "packaging" },
-    { name = "protobuf" },
-    { name = "sympy" },
+    { name = "flatbuffers", marker = "platform_machine != 's390x'" },
+    { name = "numpy", marker = "platform_machine != 's390x'" },
+    { name = "packaging", marker = "platform_machine != 's390x'" },
+    { name = "protobuf", marker = "platform_machine != 's390x'" },
+    { name = "sympy", marker = "platform_machine != 's390x'" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/08/07/f225999919f56506b603aaa3ff837ad563ab26f86906ed7fa7e5abcd849e/onnxruntime_openvino-1.24.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:2c3bb73e68ac27f4891af8a595c1faf574ec68b772e6583c90a0b997a1822782", size = 84433183, upload-time = "2026-02-26T13:44:50.254Z" },
@@ -450,12 +450,12 @@ name = "onnxruntime-rocm"
 version = "1.22.2.post1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "coloredlogs" },
-    { name = "flatbuffers" },
-    { name = "numpy" },
-    { name = "packaging" },
-    { name = "protobuf" },
-    { name = "sympy" },
+    { name = "coloredlogs", marker = "platform_machine != 's390x'" },
+    { name = "flatbuffers", marker = "platform_machine != 's390x'" },
+    { name = "numpy", marker = "platform_machine != 's390x'" },
+    { name = "packaging", marker = "platform_machine != 's390x'" },
+    { name = "protobuf", marker = "platform_machine != 's390x'" },
+    { name = "sympy", marker = "platform_machine != 's390x'" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/33/d0/e6f011c8e01853a8a4a56b48b1421257d89767fa76e8a273eeb2c54eb725/onnxruntime_rocm-1.22.2.post1-cp313-cp313-manylinux_2_35_x86_64.whl", hash = "sha256:19b56e9e41da3c7042dc97223ef46976ed8bbdf0ffe43646129f773647794b44", size = 217937775, upload-time = "2025-09-11T16:40:34.403Z" },
@@ -811,7 +811,7 @@ name = "sympy"
 version = "1.14.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "mpmath" },
+    { name = "mpmath", marker = "platform_machine != 's390x'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" }
 wheels = [
@@ -862,7 +862,7 @@ wheels = [

 [[package]]
 name = "winnow"
-version = "0.4.11"
+version = "0.5.0"
 source = { editable = "." }
 dependencies = [
    { name = "croniter" },