fix: implement fixable limitations from annotation pass
cache.py — model fingerprint auto-invalidation: Replace hardcoded "buffalo_l_v1" version string with a fingerprint derived from buffalo_l .onnx file sizes and mtimes. EmbeddingCache now computes this at init time; stale embeddings from replaced or updated model files are automatically invalidated. Falls back to the static string before the model is downloaded. Note: existing caches built against the old key will miss on the first run after upgrade and recompute cleanly. frigate_api.py — Frigate version check: Add get_frigate_version() (GET /api/version). Called at the start of upload_to_frigate(); warns if below v0.16 where the face training API endpoints don't exist. immich_api.py + cli.py — Immich version check: Add get_immich_version() (GET /api/server/version). Called at startup before get_people(); warns if below v1.106 where the face data and merge APIs winnow depends on aren't guaranteed present. Remaining TODO(frigate-api) annotations are left in place — they require Frigate to expose per-file embeddings or a rebuild-complete signal before they can be addressed.
This commit is contained in:
+31
-13
@@ -7,43 +7,62 @@ recomputing on reruns. Uses numpy binary format for fast I/O.
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Model versions — bump these when the upstream model changes.
|
||||
# LIMITATION — no automatic invalidation: if the user replaces the buffalo_l
|
||||
# model files on disk (e.g. custom weights, InsightFace update) without
|
||||
# changing INSIGHTFACE_HOME, the version string here stays "buffalo_l_v1" and
|
||||
# stale embeddings from the old model are served from cache indefinitely.
|
||||
# TODO: derive the version from a checksum of the model files, or expose a
|
||||
# --clear-cache / CLEAR_EMBEDDING_CACHE flag so users can force invalidation.
|
||||
MODEL_VERSIONS = {
|
||||
"insightface": "buffalo_l_v1",
|
||||
"immich": "immich_buffalo_l_v1",
|
||||
}
|
||||
|
||||
|
||||
def _insightface_model_fingerprint() -> str:
|
||||
"""Derive a version string from buffalo_l .onnx file sizes and mtimes.
|
||||
|
||||
Changes automatically when model files are replaced or updated, preventing
|
||||
stale embeddings from a previous model being served from cache.
|
||||
Falls back to a static string before the model is downloaded (first run).
|
||||
"""
|
||||
insightface_home = os.environ.get("INSIGHTFACE_HOME", os.path.expanduser("~/.insightface"))
|
||||
model_dir = Path(insightface_home) / "models" / "buffalo_l"
|
||||
if not model_dir.exists():
|
||||
return "buffalo_l_v1"
|
||||
onnx_files = sorted(model_dir.glob("*.onnx"))
|
||||
if not onnx_files:
|
||||
return "buffalo_l_v1"
|
||||
fingerprint = "|".join(
|
||||
f"{f.name}:{f.stat().st_size}:{int(f.stat().st_mtime)}"
|
||||
for f in onnx_files
|
||||
)
|
||||
return hashlib.sha256(fingerprint.encode()).hexdigest()[:12]
|
||||
|
||||
|
||||
class EmbeddingCache:
|
||||
"""Simple disk-based embedding cache.
|
||||
|
||||
Embeddings are stored as .npy files in a flat directory,
|
||||
keyed by a hash of (asset_id, model_version).
|
||||
keyed by a hash of (asset_id, model_version). The InsightFace version
|
||||
is derived from buffalo_l model file metadata so the cache auto-invalidates
|
||||
when model files are replaced or updated.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: str = ".if_cache") -> None:
|
||||
self.cache_dir = cache_dir
|
||||
self._ensured = False
|
||||
self._model_versions = {
|
||||
**MODEL_VERSIONS,
|
||||
"insightface": _insightface_model_fingerprint(),
|
||||
}
|
||||
|
||||
def _ensure_dir(self) -> None:
|
||||
if not self._ensured:
|
||||
os.makedirs(self.cache_dir, exist_ok=True)
|
||||
self._ensured = True
|
||||
|
||||
@staticmethod
|
||||
def _key(asset_id: str, model: str) -> str:
|
||||
version = MODEL_VERSIONS.get(model, model)
|
||||
def _key(self, asset_id: str, model: str) -> str:
|
||||
version = self._model_versions.get(model, model)
|
||||
raw = f"{asset_id}:{version}"
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
||||
|
||||
@@ -96,4 +115,3 @@ def get_cache(cache_dir: str = ".if_cache") -> EmbeddingCache:
|
||||
if _cache is None:
|
||||
_cache = EmbeddingCache(cache_dir)
|
||||
return _cache
|
||||
|
||||
|
||||
Reference in New Issue
Block a user