Merge pull request #39 from sudolulo/fix/codebase-audit-r2

fix: codebase audit rounds 2-5 (correctness, cli, jobs)

pyproject.toml

@@ -29,6 +29,9 @@ dependencies = [
 gpu   = [
     "onnxruntime-gpu>=1.23.2; sys_platform == 'linux' and platform_machine == 'x86_64'",
     "nvidia-cudnn-cu12>=9.0.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
+    "nvidia-cuda-runtime-cu12>=12.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
+    "nvidia-cufft-cu12>=11.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
+    "nvidia-curand-cu12>=10.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
 ]
 rocm  = ["onnxruntime-rocm>=1.16.0; sys_platform == 'linux' and platform_machine == 'x86_64'"]
 intel = ["onnxruntime-openvino>=1.20.0; sys_platform == 'linux' and platform_machine == 'x86_64'"]

uv.lock

@@ -340,6 +340,16 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/52/de/823919be3b9d0ccbf1f784035423c5f18f4267fb0123558d58b813c6ec86/nvidia_cuda_nvrtc_cu12-12.9.86-py3-none-win_amd64.whl", hash = "sha256:72972ebdcf504d69462d3bcd67e7b81edd25d0fb85a2c46d3ea3517666636349", size = 76408187, upload-time = "2025-06-05T20:12:27.819Z" },
 ]
 
+[[package]]
+name = "nvidia-cuda-runtime-cu12"
+version = "12.9.79"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/e0/0279bd94539fda525e0c8538db29b72a5a8495b0c12173113471d28bce78/nvidia_cuda_runtime_cu12-12.9.79-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:83469a846206f2a733db0c42e223589ab62fd2fabac4432d2f8802de4bded0a4", size = 3515012, upload-time = "2025-06-05T20:00:35.519Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/46/a92db19b8309581092a3add7e6fceb4c301a3fd233969856a8cbf042cd3c/nvidia_cuda_runtime_cu12-12.9.79-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25bba2dfb01d48a9b59ca474a1ac43c6ebf7011f1b0b8cc44f54eb6ac48a96c3", size = 3493179, upload-time = "2025-06-05T20:00:53.735Z" },
+    { url = "https://files.pythonhosted.org/packages/59/df/e7c3a360be4f7b93cee39271b792669baeb3846c58a4df6dfcf187a7ffab/nvidia_cuda_runtime_cu12-12.9.79-py3-none-win_amd64.whl", hash = "sha256:8e018af8fa02363876860388bd10ccb89eb9ab8fb0aa749aaf58430a9f7c4891", size = 3591604, upload-time = "2025-06-05T20:11:17.036Z" },
+]
+
 [[package]]
 name = "nvidia-cudnn-cu12"
 version = "9.23.1.3"
@@ -353,6 +363,39 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/75/ec/62b56fc5e8219a268c6f62c4e9fb1369ebec049512328e650d1a9a28bcc8/nvidia_cudnn_cu12-9.23.1.3-py3-none-win_amd64.whl", hash = "sha256:b874af5bfab5e1010ae88bfead14bf8e9da6b20283582288f1c05f056090a398", size = 689996767, upload-time = "2026-06-09T19:44:25.343Z" },
 ]
 
+[[package]]
+name = "nvidia-cufft-cu12"
+version = "11.4.1.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 's390x'" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/2b/76445b0af890da61b501fde30650a1a4bd910607261b209cccb5235d3daa/nvidia_cufft_cu12-11.4.1.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1a28c9b12260a1aa7a8fd12f5ebd82d027963d635ba82ff39a1acfa7c4c0fbcf", size = 200822453, upload-time = "2025-06-05T20:05:27.889Z" },
+    { url = "https://files.pythonhosted.org/packages/95/f4/61e6996dd20481ee834f57a8e9dca28b1869366a135e0d42e2aa8493bdd4/nvidia_cufft_cu12-11.4.1.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c67884f2a7d276b4b80eb56a79322a95df592ae5e765cf1243693365ccab4e28", size = 200877592, upload-time = "2025-06-05T20:05:45.862Z" },
+    { url = "https://files.pythonhosted.org/packages/20/ee/29955203338515b940bd4f60ffdbc073428f25ef9bfbce44c9a066aedc5c/nvidia_cufft_cu12-11.4.1.4-py3-none-win_amd64.whl", hash = "sha256:8e5bfaac795e93f80611f807d42844e8e27e340e0cde270dcb6c65386d795b80", size = 200067309, upload-time = "2025-06-05T20:13:59.762Z" },
+]
+
+[[package]]
+name = "nvidia-curand-cu12"
+version = "10.3.10.19"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/14/1c/2a45afc614d99558d4a773fa740d8bb5471c8398eeed925fc0fcba020173/nvidia_curand_cu12-10.3.10.19-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:de663377feb1697e1d30ed587b07d5721fdd6d2015c738d7528a6002a6134d37", size = 68292066, upload-time = "2025-05-01T19:39:13.595Z" },
+    { url = "https://files.pythonhosted.org/packages/31/44/193a0e171750ca9f8320626e8a1f2381e4077a65e69e2fb9708bd479e34a/nvidia_curand_cu12-10.3.10.19-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:49b274db4780d421bd2ccd362e1415c13887c53c214f0d4b761752b8f9f6aa1e", size = 68295626, upload-time = "2025-05-01T19:39:38.885Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/98/1bd66fd09cbe1a5920cb36ba87029d511db7cca93979e635fd431ad3b6c0/nvidia_curand_cu12-10.3.10.19-py3-none-win_amd64.whl", hash = "sha256:e8129e6ac40dc123bd948e33d3e11b4aa617d87a583fa2f21b3210e90c743cde", size = 68774847, upload-time = "2025-05-01T19:48:52.93Z" },
+]
+
+[[package]]
+name = "nvidia-nvjitlink-cu12"
+version = "12.9.86"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/46/0c/c75bbfb967457a0b7670b8ad267bfc4fffdf341c074e0a80db06c24ccfd4/nvidia_nvjitlink_cu12-12.9.86-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:e3f1171dbdc83c5932a45f0f4c99180a70de9bd2718c1ab77d14104f6d7147f9", size = 39748338, upload-time = "2025-06-05T20:10:25.613Z" },
+    { url = "https://files.pythonhosted.org/packages/97/bc/2dcba8e70cf3115b400fef54f213bcd6715a3195eba000f8330f11e40c45/nvidia_nvjitlink_cu12-12.9.86-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:994a05ef08ef4b0b299829cde613a424382aff7efb08a7172c1fa616cc3af2ca", size = 39514880, upload-time = "2025-06-05T20:10:04.89Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/7e/2eecb277d8a98184d881fb98a738363fd4f14577a4d2d7f8264266e82623/nvidia_nvjitlink_cu12-12.9.86-py3-none-win_amd64.whl", hash = "sha256:cc6fcec260ca843c10e34c936921a1c426b351753587fdd638e8cff7b16bb9db", size = 35584936, upload-time = "2025-06-05T20:16:08.525Z" },
+]
+
 [[package]]
 name = "onnx"
 version = "1.21.0"
@@ -862,7 +905,7 @@ wheels = [
 
 [[package]]
 name = "winnow"
-version = "0.6.2"
+version = "0.6.4"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },
@@ -880,7 +923,10 @@ cpu = [
     { name = "onnxruntime" },
 ]
 gpu = [
+    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
     { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
     { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
 ]
 intel = [
@@ -901,7 +947,10 @@ requires-dist = [
     { name = "croniter", specifier = ">=5.0.2" },
     { name = "insightface", specifier = ">=0.7.3" },
     { name = "numpy", specifier = ">=2.2.6" },
+    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'gpu'", specifier = ">=12.0" },
     { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'gpu'", specifier = ">=9.0.0" },
+    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'gpu'", specifier = ">=11.0" },
+    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'gpu'", specifier = ">=10.0" },
     { name = "onnxruntime", marker = "extra == 'cpu'", specifier = ">=1.23.2" },
     { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'gpu'", specifier = ">=1.23.2" },
     { name = "onnxruntime-openvino", marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'intel'", specifier = ">=1.20.0" },

winnow/cli.py

@@ -7,6 +7,7 @@ import sys
 from rich import print as rprint
 from rich.prompt import Confirm
 
+from . import __version__
 from .config import Config, _getenv_bool
 from .executor import execute_jobs, upload_to_frigate
 from .immich_api import get_immich_version, get_people, merge_people
@@ -71,7 +72,7 @@ def _handle_duplicate_people(people: list[dict]) -> list[dict]:
     by_name: dict[str, list[dict]] = defaultdict(list)
     for p in people:
         name = (p.get("name") or "").strip()
-        if name:
+        if name and p.get("id"):
             by_name[name].append(p)
 
     duplicates = {name: ps for name, ps in by_name.items() if len(ps) > 1}
@@ -81,19 +82,23 @@ def _handle_duplicate_people(people: list[dict]) -> list[dict]:
     def _smaller_duplicate_ids(groups: dict) -> set[str]:
         """IDs of all but the largest person in each duplicate group."""
         return {
-            p.get("id")
+            pid
             for ps in groups.values()
             for p in sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)[1:]
+            if (pid := p.get("id"))
         }
 
     skip_ids = _smaller_duplicate_ids(duplicates)
 
+    def _excl(lst: list[dict]) -> list[dict]:
+        return [p for p in lst if p.get("id") not in skip_ids]
+
     if not Config.MERGE_DUPLICATE_PEOPLE:
         rprint("\n[bold yellow]⚠  Duplicate person names detected in Immich:[/bold yellow]")
         for name, ps in sorted(duplicates.items()):
             ordered = sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)
             entries = ", ".join(
-                f"[dim]{p['id'][:8]}…[/dim] ({p.get('assetCount', 0)} assets)"
+                f"[dim]{(p.get('id') or '?')[:8]}…[/dim] ({p.get('assetCount', 0)} assets)"
                 for p in ordered
             )
             rprint(f"  [yellow]{name}[/yellow] → {len(ps)} people: {entries}")
@@ -109,20 +114,21 @@ def _handle_duplicate_people(people: list[dict]) -> list[dict]:
         )
         # Return deduplicated list — keep only the largest per name so that
         # downstream job creation never runs two jobs for the same Frigate folder.
-        return [p for p in people if p.get("id") not in skip_ids]
+        return _excl(people)
 
     # Auto-merge: survivor = largest asset count, rest merge into it inside Immich
     merged_any = False
     for name, ps in sorted(duplicates.items()):
         ordered = sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)
         survivor = ordered[0]
-        merge_ids = [p["id"] for p in ordered[1:]]
+        survivor_id = survivor.get("id")
+        merge_ids = [pid for p in ordered[1:] if (pid := p.get("id")) is not None]
         rprint(
             f"  [cyan]Merging {name!r} inside Immich:[/cyan] keeping "
-            f"[dim]{survivor['id'][:8]}…[/dim] ({survivor.get('assetCount', 0)} assets), "
+            f"[dim]{survivor_id[:8]}…[/dim] ({survivor.get('assetCount', 0)} assets), "
             f"absorbing {len(merge_ids)} smaller duplicate(s)..."
         )
-        if merge_people(survivor["id"], merge_ids):
+        if merge_people(survivor_id, merge_ids):
             rprint(f"  [green]✓ Merged {name!r}[/green]")
             merged_any = True
         else:
@@ -141,12 +147,12 @@ def _handle_duplicate_people(people: list[dict]) -> list[dict]:
                 " — possible transient error or expired API key;"
                 " proceeding with pre-merge list. Check IMMICH_API_KEY if this recurs."
             )
-            return [p for p in people if p.get("id") not in skip_ids]
+            return _excl(people)
         # Filter out the smaller duplicate from any group whose merge failed — those
         # IDs still exist in Immich and would produce two jobs for the same folder.
         # IDs from groups that merged successfully are already gone from Immich, so
         # this filter is a no-op for them.
-        return [p for p in fresh if p.get("id") not in skip_ids]
+        return _excl(fresh)
 
     # All merges failed — fall back to local deduplication (keep largest per name) so
     # downstream job creation never runs two jobs for the same Frigate folder.
@@ -154,7 +160,7 @@ def _handle_duplicate_people(people: list[dict]) -> list[dict]:
         "  [yellow]All merges failed — applying local deduplication"
         " to avoid overwriting output.[/yellow]"
     )
-    return [p for p in people if p.get("id") not in skip_ids]
+    return _excl(people)
 
 
 _UNSUPPORTED_VARS = [
@@ -179,8 +185,8 @@ def main() -> None:
         if trace_size:
             _handle_trace_crop(trace_size)
 
-        console.print(r"""
+        console.print(f"""
-    [bold blue]winnow[/bold blue]
+    [bold blue]winnow[/bold blue] [dim]v{__version__}[/dim]
     [dim]Immich -> Frigate Training Data Curator[/dim]
         """)
 

winnow/diversity.py

@@ -303,6 +303,9 @@ def _select_by_embedding(
 
             emb = get_embedding(embed_img, asset_id=asset["id"])
             if emb is not None:
+                if np.linalg.norm(emb) < 1e-6:
+                    logger.debug("Zero-norm embedding for asset %s, skipping", asset["id"])
+                    continue
                 embeddings.append(emb)
                 valid_candidates.append(asset)
                 confidence_scores.append(confidence)

winnow/embeddings.py

@@ -218,9 +218,14 @@ def get_face_embedding(img_pil: Image.Image) -> np.ndarray | None:
         if not faces:
             return None
 
-        # Return embedding of largest face
+        # Return embedding of the face nearest the crop centre; a large margin can pull
-        largest = max(faces, key=lambda f: (f.bbox[2] - f.bbox[0]) * (f.bbox[3] - f.bbox[1]))
+        # a bigger neighbouring face into frame, and max-by-area would pick the wrong person.
-        return largest.embedding
+        cx, cy = img_pil.width / 2, img_pil.height / 2
+        nearest = min(
+            faces,
+            key=lambda f: ((f.bbox[0] + f.bbox[2]) / 2 - cx) ** 2 + ((f.bbox[1] + f.bbox[3]) / 2 - cy) ** 2,
+        )
+        return nearest.embedding
     except Exception as e:
         logger.error("Error getting face embedding: %s", e)
         return None

winnow/executor.py

@@ -514,8 +514,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
                                 uploaded += 1
                                 person_uploaded += 1
                                 effective_count += 1
-                                min_quality_score_for_slot = None
+                                min_quality_score_for_slot = None  # for/else rollback mirrors this pair
-
                                 asset_id = asset_map.get(fname)
                                 if asset_id:
                                     try:
@@ -608,6 +607,16 @@ def upload_to_frigate(jobs: list[dict]) -> None:
                             progress.console.print(
                                 f"    [red]✗ {fname}: {type(e).__name__} - {e} (after {max_retries} attempts)[/red]"
                             )
+                    else:
+                        # All retries exhausted without a successful upload.
+                        # Restore the slot freed by the preceding delete so the next
+                        # candidate still sees at_cap=True and must beat the replacement gate.
+                        # Also clear the quality floor — the deleted file's score no longer
+                        # represents any live Frigate file, and leaving it blocks the next
+                        # candidate from filling the restored slot.
+                        if at_cap:
+                            effective_count += 1
+                            min_quality_score_for_slot = None
 
                     progress.advance(upload_task)
 

winnow/immich_api.py

@@ -39,7 +39,11 @@ def get_immich_version() -> tuple[int, int, int] | None:
         )
         if resp.ok:
             data = resp.json()
-            return (int(data["major"]), int(data["minor"]), int(data["patch"]))
+            major, minor, patch = data.get("major"), data.get("minor"), data.get("patch")
+            if major is None or minor is None or patch is None:
+                logger.debug("Unexpected Immich version schema: %s", data)
+                return None
+            return (int(major), int(minor), int(patch))
         return None
     except Exception:
         return None

winnow/jobs.py

@@ -192,13 +192,20 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None:
     return job
 
 
+def _valid_people(people: list[dict]) -> list[dict]:
+    return sorted(
+        [p for p in people if (p.get("name") or "").strip() and p.get("id")],
+        key=lambda x: x["name"],
+    )
+
+
 def interactive_configure(people: list[dict]) -> list[dict]:
     """Interactive phase: select person(s), mode, and configure training strategy.
 
     Supports multi-person batch mode — after configuring one person,
     prompts to add another.
     """
-    valid_people = sorted([p for p in people if p.get("name")], key=lambda x: x["name"])
+    valid_people = _valid_people(people)
 
     if not valid_people:
         rprint("[red]No people found with names in Immich.[/red]")
@@ -209,9 +216,9 @@ def interactive_configure(people: list[dict]) -> list[dict]:
     while True:
         # Select person
         console.print("\n[bold cyan]Select Person to Train:[/bold cyan]")
+        queued_ids = {j["person"]["id"] for j in jobs}
         for idx, p in enumerate(valid_people, 1):
-            # Mark already-queued people
+            marker = " [dim](queued)[/dim]" if p.get("id") in queued_ids else ""
-            marker = " [dim](queued)[/dim]" if any(j["person"]["id"] == p["id"] for j in jobs) else ""
             console.print(f"  [bold]{idx}.[/bold] {p['name']}{marker}")
 
         p_choice = IntPrompt.ask("Enter Number", choices=[str(i) for i in range(1, len(valid_people) + 1)])
@@ -230,7 +237,7 @@ def interactive_configure(people: list[dict]) -> list[dict]:
 
 def auto_configure(people: list[dict]) -> list[dict]:
     """Non-interactive: configure jobs for all named people automatically."""
-    valid_people = sorted([p for p in people if p.get("name")], key=lambda x: x["name"])
+    valid_people = _valid_people(people)
 
     if not valid_people:
         rprint("[red]No people found with names in Immich.[/red]")