fix: implement fixable limitations from annotation pass
cache.py — model fingerprint auto-invalidation: Replace hardcoded "buffalo_l_v1" version string with a fingerprint derived from buffalo_l .onnx file sizes and mtimes. EmbeddingCache now computes this at init time; stale embeddings from replaced or updated model files are automatically invalidated. Falls back to the static string before the model is downloaded. Note: existing caches built against the old key will miss on the first run after upgrade and recompute cleanly. frigate_api.py — Frigate version check: Add get_frigate_version() (GET /api/version). Called at the start of upload_to_frigate(); warns if below v0.16 where the face training API endpoints don't exist. immich_api.py + cli.py — Immich version check: Add get_immich_version() (GET /api/server/version). Called at startup before get_people(); warns if below v1.106 where the face data and merge APIs winnow depends on aren't guaranteed present. Remaining TODO(frigate-api) annotations are left in place — they require Frigate to expose per-file embeddings or a rebuild-complete signal before they can be addressed.
This commit is contained in:
+31
-13
@@ -7,43 +7,62 @@ recomputing on reruns. Uses numpy binary format for fast I/O.
|
|||||||
import hashlib
|
import hashlib
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Model versions — bump these when the upstream model changes.
|
|
||||||
# LIMITATION — no automatic invalidation: if the user replaces the buffalo_l
|
|
||||||
# model files on disk (e.g. custom weights, InsightFace update) without
|
|
||||||
# changing INSIGHTFACE_HOME, the version string here stays "buffalo_l_v1" and
|
|
||||||
# stale embeddings from the old model are served from cache indefinitely.
|
|
||||||
# TODO: derive the version from a checksum of the model files, or expose a
|
|
||||||
# --clear-cache / CLEAR_EMBEDDING_CACHE flag so users can force invalidation.
|
|
||||||
MODEL_VERSIONS = {
|
MODEL_VERSIONS = {
|
||||||
"insightface": "buffalo_l_v1",
|
|
||||||
"immich": "immich_buffalo_l_v1",
|
"immich": "immich_buffalo_l_v1",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _insightface_model_fingerprint() -> str:
|
||||||
|
"""Derive a version string from buffalo_l .onnx file sizes and mtimes.
|
||||||
|
|
||||||
|
Changes automatically when model files are replaced or updated, preventing
|
||||||
|
stale embeddings from a previous model being served from cache.
|
||||||
|
Falls back to a static string before the model is downloaded (first run).
|
||||||
|
"""
|
||||||
|
insightface_home = os.environ.get("INSIGHTFACE_HOME", os.path.expanduser("~/.insightface"))
|
||||||
|
model_dir = Path(insightface_home) / "models" / "buffalo_l"
|
||||||
|
if not model_dir.exists():
|
||||||
|
return "buffalo_l_v1"
|
||||||
|
onnx_files = sorted(model_dir.glob("*.onnx"))
|
||||||
|
if not onnx_files:
|
||||||
|
return "buffalo_l_v1"
|
||||||
|
fingerprint = "|".join(
|
||||||
|
f"{f.name}:{f.stat().st_size}:{int(f.stat().st_mtime)}"
|
||||||
|
for f in onnx_files
|
||||||
|
)
|
||||||
|
return hashlib.sha256(fingerprint.encode()).hexdigest()[:12]
|
||||||
|
|
||||||
|
|
||||||
class EmbeddingCache:
|
class EmbeddingCache:
|
||||||
"""Simple disk-based embedding cache.
|
"""Simple disk-based embedding cache.
|
||||||
|
|
||||||
Embeddings are stored as .npy files in a flat directory,
|
Embeddings are stored as .npy files in a flat directory,
|
||||||
keyed by a hash of (asset_id, model_version).
|
keyed by a hash of (asset_id, model_version). The InsightFace version
|
||||||
|
is derived from buffalo_l model file metadata so the cache auto-invalidates
|
||||||
|
when model files are replaced or updated.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cache_dir: str = ".if_cache") -> None:
|
def __init__(self, cache_dir: str = ".if_cache") -> None:
|
||||||
self.cache_dir = cache_dir
|
self.cache_dir = cache_dir
|
||||||
self._ensured = False
|
self._ensured = False
|
||||||
|
self._model_versions = {
|
||||||
|
**MODEL_VERSIONS,
|
||||||
|
"insightface": _insightface_model_fingerprint(),
|
||||||
|
}
|
||||||
|
|
||||||
def _ensure_dir(self) -> None:
|
def _ensure_dir(self) -> None:
|
||||||
if not self._ensured:
|
if not self._ensured:
|
||||||
os.makedirs(self.cache_dir, exist_ok=True)
|
os.makedirs(self.cache_dir, exist_ok=True)
|
||||||
self._ensured = True
|
self._ensured = True
|
||||||
|
|
||||||
@staticmethod
|
def _key(self, asset_id: str, model: str) -> str:
|
||||||
def _key(asset_id: str, model: str) -> str:
|
version = self._model_versions.get(model, model)
|
||||||
version = MODEL_VERSIONS.get(model, model)
|
|
||||||
raw = f"{asset_id}:{version}"
|
raw = f"{asset_id}:{version}"
|
||||||
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
@@ -96,4 +115,3 @@ def get_cache(cache_dir: str = ".if_cache") -> EmbeddingCache:
|
|||||||
if _cache is None:
|
if _cache is None:
|
||||||
_cache = EmbeddingCache(cache_dir)
|
_cache = EmbeddingCache(cache_dir)
|
||||||
return _cache
|
return _cache
|
||||||
|
|
||||||
|
|||||||
+8
-1
@@ -9,7 +9,7 @@ from rich.prompt import Confirm
|
|||||||
|
|
||||||
from .config import Config, ConfigManager
|
from .config import Config, ConfigManager
|
||||||
from .executor import execute_jobs, upload_to_frigate
|
from .executor import execute_jobs, upload_to_frigate
|
||||||
from .immich_api import get_people, merge_people
|
from .immich_api import get_immich_version, get_people, merge_people
|
||||||
from .jobs import _show_preview, auto_configure, interactive_configure
|
from .jobs import _show_preview, auto_configure, interactive_configure
|
||||||
from .log_config import console, setup_logging
|
from .log_config import console, setup_logging
|
||||||
from .upload_tracker import find_by_crop_dimension, get_person_summary, reset_person
|
from .upload_tracker import find_by_crop_dimension, get_person_summary, reset_person
|
||||||
@@ -216,6 +216,13 @@ def main() -> None:
|
|||||||
f" {counts['rejected']} rejected{frigate_part}[/dim]"
|
f" {counts['rejected']} rejected{frigate_part}[/dim]"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_immich_version = get_immich_version()
|
||||||
|
if _immich_version is not None and _immich_version < (1, 106, 0):
|
||||||
|
rprint(
|
||||||
|
f" [yellow]⚠ Immich {'.'.join(str(x) for x in _immich_version)} detected — "
|
||||||
|
"winnow requires v1.106+. Some features may not work.[/yellow]"
|
||||||
|
)
|
||||||
|
|
||||||
people = get_people()
|
people = get_people()
|
||||||
if not people:
|
if not people:
|
||||||
rprint("[bold red]Could not fetch people from Immich. Check URL/Key.[/bold red]")
|
rprint("[bold red]Could not fetch people from Immich. Check URL/Key.[/bold red]")
|
||||||
|
|||||||
+12
-4
@@ -17,6 +17,7 @@ from .frigate_api import (
|
|||||||
delete_frigate_person_files,
|
delete_frigate_person_files,
|
||||||
get_all_frigate_person_files,
|
get_all_frigate_person_files,
|
||||||
get_frigate_person_files,
|
get_frigate_person_files,
|
||||||
|
get_frigate_version,
|
||||||
recognize_face,
|
recognize_face,
|
||||||
)
|
)
|
||||||
from .image_processing import process_face_mode
|
from .image_processing import process_face_mode
|
||||||
@@ -311,10 +312,17 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
rprint("[yellow]⚠️ FRIGATE_URL not set, skipping upload.[/yellow]")
|
rprint("[yellow]⚠️ FRIGATE_URL not set, skipping upload.[/yellow]")
|
||||||
return
|
return
|
||||||
|
|
||||||
# TODO: verify Frigate version at startup (GET /api/version) and warn if
|
_frigate_version = get_frigate_version()
|
||||||
# below v0.16. The face training API (/api/faces/{name}/register,
|
if _frigate_version is not None:
|
||||||
# /api/faces/{name}/delete, /api/faces/recognize) was introduced in v0.16;
|
try:
|
||||||
# older versions return 404s that surface as opaque upload failures.
|
parts = [int(x) for x in _frigate_version.split("-")[0].split(".") if x.isdigit()]
|
||||||
|
if len(parts) >= 2 and (parts[0], parts[1]) < (0, 16):
|
||||||
|
rprint(
|
||||||
|
f" [yellow]⚠ Frigate {_frigate_version} detected — "
|
||||||
|
"face training API requires v0.16+. Uploads may fail.[/yellow]"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
rprint("\n[bold cyan]📤 Uploading to Frigate[/bold cyan]")
|
rprint("\n[bold cyan]📤 Uploading to Frigate[/bold cyan]")
|
||||||
rprint(f" Target: [dim]{frigate_url}[/dim]")
|
rprint(f" Target: [dim]{frigate_url}[/dim]")
|
||||||
|
|||||||
@@ -8,6 +8,24 @@ import requests
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def get_frigate_version() -> str | None:
|
||||||
|
"""Fetch Frigate's version string from GET /api/version.
|
||||||
|
|
||||||
|
Returns the version string (e.g. "0.16.0-beta4") or None if FRIGATE_URL
|
||||||
|
is unset, the endpoint is unreachable, or the response is not parseable.
|
||||||
|
"""
|
||||||
|
frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
|
||||||
|
if not frigate_url:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
resp = requests.get(f"{frigate_url}/api/version", timeout=5)
|
||||||
|
if resp.ok:
|
||||||
|
return resp.text.strip().strip('"')
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _get_faces_data() -> dict | None:
|
def _get_faces_data() -> dict | None:
|
||||||
"""Fetch raw GET /api/faces response. Returns None if unavailable."""
|
"""Fetch raw GET /api/faces response. Returns None if unavailable."""
|
||||||
frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
|
frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
|
||||||
|
|||||||
@@ -26,6 +26,25 @@ class FaceData:
|
|||||||
image_height: int
|
image_height: int
|
||||||
|
|
||||||
|
|
||||||
|
def get_immich_version() -> tuple[int, int, int] | None:
|
||||||
|
"""Fetch Immich server version from GET /api/server/version.
|
||||||
|
|
||||||
|
Returns (major, minor, patch) or None if unreachable or unparseable.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
resp = requests.get(
|
||||||
|
f"{Config.IMMICH_URL}/api/server/version",
|
||||||
|
headers=get_headers(),
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
if resp.ok:
|
||||||
|
data = resp.json()
|
||||||
|
return (int(data["major"]), int(data["minor"]), int(data["patch"]))
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_people() -> list[dict]:
|
def get_people() -> list[dict]:
|
||||||
"""Fetch all people from Immich."""
|
"""Fetch all people from Immich."""
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user