release: merge dev → main for 0.4.4

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.4] - 2026-06-14
+
+### Added
+
+- **`RESET_PERSON=*` bulk reset**: resets every tracked person at once (deletes their Frigate training files and clears tracker data). Any other value still resets that specific person by name. If a person is literally named `*` they are reset as part of the bulk operation, and a warning is printed to clarify this.
+- **Near-duplicate removal before diversity selection**: a greedy dedup pass now runs after embedding collection and before clustering. Candidates within 0.20 cosine distance of a higher-quality image are dropped, eliminating burst shots and same-event lookalike photos that produce redundant training images. The best-quality frame from each near-identical group is kept. Dropped count is logged per person.
+
+### Fixed
+
+- **HTTP 500 upload errors no longer show Frigate's misleading "Try restarting Frigate" message**: the response body is now logged at debug level only. HTTP 400 detail (e.g. "No face was detected") is still shown since it is actionable.
+- **`RuntimeWarning: Mean of empty slice`** when a person has only one image after quality filtering: `_compute_adaptive_threshold` now returns the floor value immediately when there are no pairwise distances to sample, and the k-medoids cluster count is floored at 1 to prevent `k=0`.
+- **Path traversal guard on output directory**: person names with `../` sequences or absolute paths (e.g. `/etc`) are now rejected before any filesystem operation, logging an error and skipping the job rather than writing outside the output tree.
+
 ## [0.4.3] - 2026-06-14
 
 ### Added

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "winnow"
-version = "0.4.3"
+version = "0.4.4"
 description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
 license = "AGPL-3.0-or-later"
 requires-python = ">=3.13"

winnow/cli.py

@@ -156,11 +156,27 @@ def main() -> None:
         rprint(f"Server: [dim]{Config.IMMICH_URL}[/dim]")
         rprint(f"Output: [dim]{Config.OUTPUT_DIR}[/dim]")
 
-        # Handle RESET_PERSON before anything else
+        # Handle RESET_PERSON before anything else.
+        # RESET_PERSON=* resets every tracked person; any other value resets
+        # that specific person by name.
         reset_person_name = os.environ.get("RESET_PERSON", "").strip()
         if reset_person_name:
-            reset_person(reset_person_name)
-            rprint(f"[bold yellow]Reset tracking data for: {reset_person_name}[/bold yellow]")
+            if reset_person_name == "*":
+                names = list(get_person_summary().keys())
+                if "*" in names:
+                    rprint(
+                        "[yellow]Note: a person literally named '*' exists in the tracker "
+                        "and will be reset along with everyone else.[/yellow]"
+                    )
+                if names:
+                    for name in names:
+                        reset_person(name)
+                    rprint(f"[bold yellow]Reset tracking data for all {len(names)} people.[/bold yellow]")
+                else:
+                    rprint("[dim]No tracking data to reset.[/dim]")
+            else:
+                reset_person(reset_person_name)
+                rprint(f"[bold yellow]Reset tracking data for: {reset_person_name}[/bold yellow]")
 
         # Show per-person tracker summary if data exists
         summary = get_person_summary()

winnow/diversity.py

@@ -281,7 +281,16 @@ def _select_by_embedding(
         logger.warning(f"Only {len(valid_candidates)} valid embeddings. Returning all.")
         return valid_candidates
 
-    # --- Phase 5: Cluster-aware selection ---
+    # --- Phase 5: Near-duplicate removal ---
+    # Burst shots and repeated near-identical photos produce embeddings that are
+    # close but not identical, so FPS doesn't filter them out on its own.
+    # Greedily drop any candidate within DEDUP_THRESHOLD cosine distance of a
+    # higher-quality image already in the kept set.
+    embeddings, valid_candidates, confidence_scores = _dedup_embeddings(
+        embeddings, valid_candidates, confidence_scores
+    )
+
+    # --- Phase 6: Cluster-aware selection ---
     return _cluster_aware_selection(
         embeddings,
         valid_candidates,
@@ -291,6 +300,59 @@ def _select_by_embedding(
     )
 
 
+# =============================================================================
+# Near-Duplicate Removal
+# =============================================================================
+
+_DEDUP_THRESHOLD = 0.20  # cosine distance — burst shots ~0.01-0.05, same-event similar shots ~0.10-0.20
+
+
+def _dedup_embeddings(
+    embeddings: list,
+    candidates: list,
+    confidence_scores: list,
+) -> tuple[list, list, list]:
+    """Greedy near-duplicate removal before clustering.
+
+    Sorts by quality score descending (best first), then for each candidate
+    drops it if any already-kept embedding is within _DEDUP_THRESHOLD cosine
+    distance. This eliminates burst-shot near-duplicates while preserving the
+    highest-quality representative from each near-identical group.
+    """
+    if len(embeddings) < 2:
+        return embeddings, candidates, confidence_scores
+
+    emb_matrix = np.vstack(embeddings)
+    norms = np.linalg.norm(emb_matrix, axis=1, keepdims=True)
+    emb_normed = emb_matrix / np.maximum(norms, 1e-8)
+
+    # Sort by quality descending so the best image in each near-duplicate group wins
+    quality_scores = [c.get("quality_score") or 0.0 for c in candidates]
+    order = sorted(range(len(candidates)), key=lambda i: quality_scores[i], reverse=True)
+
+    kept_indices = []
+    kept_normed = []
+
+    for i in order:
+        if kept_normed:
+            kept_stack = np.vstack(kept_normed)
+            sims = emb_normed[i] @ kept_stack.T
+            if np.any(sims > 1 - _DEDUP_THRESHOLD):
+                continue
+        kept_indices.append(i)
+        kept_normed.append(emb_normed[i])
+
+    dropped = len(embeddings) - len(kept_indices)
+    if dropped:
+        logger.info(f"Near-duplicate removal dropped {dropped} images (threshold {_DEDUP_THRESHOLD}).")
+
+    return (
+        [embeddings[i] for i in kept_indices],
+        [candidates[i] for i in kept_indices],
+        [confidence_scores[i] for i in kept_indices],
+    )
+
+
 # =============================================================================
 # K-Medoids (Lightweight Implementation)
 # =============================================================================
@@ -373,6 +435,8 @@ def _compute_adaptive_threshold(emb_normed: np.ndarray, entity_type: str) -> flo
     # Compute pairwise cosine distances for the sample
     pairwise = 1 - sample @ sample.T
     upper_tri = pairwise[np.triu_indices(len(sample), k=1)]
+    if len(upper_tri) == 0:
+        return 0.05
     median_dist = float(np.median(upper_tri))
 
     # Faces: 20% of median (tighter — want fewer, more distinct images)
@@ -421,7 +485,7 @@ def _cluster_aware_selection(
     target = Config.MAX_AUTO_IMAGES if limit == "auto" else limit
 
     # --- Stage 1: K-Medoids clustering ---
-    k = min(max(5, target // 4), n // 3, n)  # e.g., 5-20 clusters
+    k = min(max(5, target // 4), max(1, n // 3), n)  # e.g., 1-20 clusters
     logger.debug(f"Clustering {n} embeddings into {k} groups (K-Medoids)...")
 
     # Compute full cosine distance matrix

winnow/executor.py

@@ -38,6 +38,19 @@ from .upload_tracker import (
 logger = logging.getLogger(__name__)
 
 
+def _safe_person_dir(output_dir: str, person_name: str) -> str:
+    """Return the output subdirectory for a person, raising ValueError on path traversal.
+
+    os.path.join silently discards output_dir when person_name is absolute,
+    and '../..' sequences resolve outside the tree. Both are rejected here.
+    """
+    candidate = os.path.realpath(os.path.join(output_dir, person_name))
+    base = os.path.realpath(output_dir)
+    if not candidate.startswith(base + os.sep) and candidate != base:
+        raise ValueError(f"Person name {person_name!r} escapes output directory — skipping")
+    return candidate
+
+
 def _reconcile_frigate_mappings(
     person_name: str,
     known_files_before: set[str],
@@ -182,7 +195,11 @@ def execute_jobs(jobs: list[dict]) -> None:
             name, mode = person["name"], config.get("mode", "face")
 
             job_task = progress.add_task(f"Processing {name}...", total=len(assets))
-            person_dir = os.path.join(Config.OUTPUT_DIR, name)
+            try:
+                person_dir = _safe_person_dir(Config.OUTPUT_DIR, name)
+            except ValueError as e:
+                logger.error(str(e))
+                continue
             # Face crops are transient (uploaded then discarded); wipe before each run.
             # Object crops are the deliverable; preserve them across runs.
             if mode == "face" and os.path.isdir(person_dir):
@@ -305,7 +322,11 @@ def upload_to_frigate(jobs: list[dict]) -> None:
     object_jobs = [j for j in jobs if j["config"].get("mode") == "object"]
     for job in object_jobs:
         name = job["person"]["name"]
-        person_dir = os.path.join(Config.OUTPUT_DIR, name)
+        try:
+            person_dir = _safe_person_dir(Config.OUTPUT_DIR, name)
+        except ValueError as e:
+            logger.error(str(e))
+            continue
         rprint(f"  [dim]📁 {name} (object): crops saved to {person_dir} — copy to Frigate manually[/dim]")
 
     frigate_url = os.environ.get("FRIGATE_URL", "")
@@ -355,7 +376,11 @@ def upload_to_frigate(jobs: list[dict]) -> None:
             if " " in name:
                 progress.console.print(f"  ℹ️  URL-encoded name for Frigate API: '{name}' → '{encoded_name}'")
 
-            person_dir = os.path.join(Config.OUTPUT_DIR, name)
+            try:
+                person_dir = _safe_person_dir(Config.OUTPUT_DIR, name)
+            except ValueError as e:
+                logger.error(str(e))
+                continue
             if not os.path.isdir(person_dir):
                 progress.console.print(f"  [dim]⏭️  {name}: no output directory, skipping[/dim]")
                 continue
@@ -580,10 +605,12 @@ def upload_to_frigate(jobs: list[dict]) -> None:
                             )
                             try:
                                 error_detail = resp.json().get("message", resp.text[:100])
-                                progress.console.print(f"      [dim]{error_detail}[/dim]")
                             except Exception:
                                 error_detail = resp.text[:100]
+                            if resp.status_code == 400:
                                 progress.console.print(f"      [dim]{error_detail}[/dim]")
+                            else:
+                                logger.debug(f"{fname} HTTP {resp.status_code}: {error_detail}")
                             if resp.status_code == 400 and "face" in error_detail.lower():
                                 asset_id = asset_map.get(fname)
                                 if asset_id: