From b9b734507b21e263965c6cb8b041625986746253 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Jun 2026 22:25:47 -0400 Subject: [PATCH] automated update: 2026-06-10 22:25:47 --- Dockerfile | 6 ++- compose.yml | 3 +- if_curator/__main__.py | 2 +- if_curator/cache.py | 9 +++- if_curator/cli.py | 92 ++++++++++++++++++++-------------- if_curator/config.py | 61 +++++++++++++++++++--- if_curator/embeddings.py | 53 +++++++++++++++++--- if_curator/image_processing.py | 11 ++-- if_curator/immich_api.py | 5 +- if_curator/logging.py | 9 +++- if_curator/quality.py | 5 +- pyproject.toml | 13 +++-- scheduler.py | 53 ++++++++++++++++---- 13 files changed, 243 insertions(+), 79 deletions(-) diff --git a/Dockerfile b/Dockerfile index 72fc7e2..f65e714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,10 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh \ && cp /root/.local/bin/uv /usr/local/bin/uv WORKDIR /app -RUN git clone --depth 1 https://github.com/sudolulo/if_curator_headless.git . \ - && uv sync --extra gpu && uv add croniter && uv cache clean +# Copy project files instead of git clone for reproducible builds +COPY pyproject.toml uv.lock* ./ +COPY if_curator/ if_curator/ +RUN uv sync --extra gpu --extra object && uv add croniter && uv cache clean COPY entrypoint.sh scheduler.py /app/ RUN chmod +x /app/entrypoint.sh diff --git a/compose.yml b/compose.yml index 942be30..038a2c9 100644 --- a/compose.yml +++ b/compose.yml @@ -13,7 +13,7 @@ services: - ENABLE_CACHE=true - CACHE_DIR=/app/.if_cache - HF_HOME=/models/huggingface - - INSIGHTFACE_HOME=/models/insightface + - INSIGHTFACE_HOME=/models # - ONLY_PEOPLE=John,Jane # - SKIP_PEOPLE=Unknown # - MIN_FACE_COUNT=5 @@ -37,3 +37,4 @@ services: - driver: nvidia count: all capabilities: [gpu] + diff --git a/if_curator/__main__.py b/if_curator/__main__.py index 9ae637f..eacf94b 100644 --- a/if_curator/__main__.py +++ b/if_curator/__main__.py @@ -1,4 +1,4 @@ -from .cli import main +from if_curator.cli import main if __name__ == "__main__": main() diff --git a/if_curator/cache.py b/if_curator/cache.py index 21a36be..5d6e529 100644 --- a/if_curator/cache.py +++ b/if_curator/cache.py @@ -80,8 +80,15 @@ _cache: EmbeddingCache | None = None def get_cache(cache_dir: str = ".if_cache") -> EmbeddingCache: - """Get or create the singleton cache instance.""" + """Get or create the singleton cache instance. + + Note: The ``cache_dir`` parameter is only used when creating the + singleton for the first time. Subsequent calls return the existing + instance regardless of ``cache_dir``. If you need a cache with a + different directory, instantiate ``EmbeddingCache`` directly. + """ global _cache if _cache is None: _cache = EmbeddingCache(cache_dir) return _cache + diff --git a/if_curator/cli.py b/if_curator/cli.py index 5c0abce..d6888d1 100644 --- a/if_curator/cli.py +++ b/if_curator/cli.py @@ -14,7 +14,7 @@ from rich.table import Table from .config import Config, ConfigManager from .diversity import select_diverse_assets -from .embeddings import is_embedding_available +from .embeddings import is_embedding_available, load_embedding_model from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_all_assets, fetch_full_image, filter_recent_assets, get_people from .logging import console, setup_logging @@ -146,6 +146,7 @@ def interactive_configure(people: list[dict]) -> list[dict]: return jobs + def auto_configure(people: list[dict]) -> list[dict]: """Non-interactive: configure jobs for all named people automatically.""" valid_people = sorted([p for p in people if p.get("name")], key=lambda x: x["name"]) @@ -164,6 +165,13 @@ def auto_configure(people: list[dict]) -> list[dict]: if skip: valid_people = [p for p in valid_people if p["name"] not in skip] + # Filter by minimum face count (Issue #6: previously unimplemented) + min_face_count = Config.MIN_FACE_COUNT + if min_face_count > 0: + valid_people = [p for p in valid_people if p.get("assetCount", 0) >= min_face_count] + if valid_people: + rprint(f" Filtered to {len(valid_people)} people with ≥{min_face_count} assets (MIN_FACE_COUNT={min_face_count})") + jobs = [] for person in valid_people: name = person["name"] @@ -239,6 +247,7 @@ def upload_to_frigate(jobs: list[dict]) -> None: rprint(f" People: [bold]{len(jobs)}[/bold], Total images: [bold]{total_files}[/bold]") uploaded, failed = 0, 0 + max_retries = 2 with Progress( SpinnerColumn(), @@ -276,45 +285,54 @@ def upload_to_frigate(jobs: list[dict]) -> None: for fname in person_files: fpath = os.path.join(person_dir, fname) - try: - with open(fpath, "rb") as f: - resp = requests.post( - f"{frigate_url}/api/faces/train/{encoded_name}/classify", - files={"file": (fname, f, "image/jpeg")}, - timeout=30, + success = False + + for attempt in range(1, max_retries + 1): + try: + with open(fpath, "rb") as f: + resp = requests.post( + f"{frigate_url}/api/faces/train/{encoded_name}/classify", + files={"file": (fname, f, "image/jpeg")}, + timeout=30, + ) + if resp.status_code == 200: + uploaded += 1 + person_uploaded += 1 + success = True + break + else: + if attempt < max_retries: + logger.warning(f"Upload attempt {attempt}/{max_retries} for {fname}: HTTP {resp.status_code}, retrying...") + continue + failed += 1 + person_failed += 1 + progress.console.print( + f" [red]✗ {fname}: HTTP {resp.status_code} (after {max_retries} attempts)[/red]" + ) + try: + error_detail = resp.json().get("message", resp.text[:100]) + progress.console.print(f" [dim]{error_detail}[/dim]") + except Exception: + progress.console.print(f" [dim]{resp.text[:100]}[/dim]") + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + if attempt < max_retries: + logger.warning(f"Upload attempt {attempt}/{max_retries} for {fname}: {type(exc).__name__}, retrying...") + continue + failed += 1 + person_failed += 1 + label = "Connection refused" if isinstance(exc, requests.exceptions.ConnectionError) else "Request timed out (30s)" + progress.console.print( + f" [red]✗ {fname}: {label} (after {max_retries} attempts)[/red]" ) - if resp.status_code == 200: - uploaded += 1 - person_uploaded += 1 - else: + except Exception as e: + if attempt < max_retries: + logger.warning(f"Upload attempt {attempt}/{max_retries} for {fname}: {type(e).__name__}, retrying...") + continue failed += 1 person_failed += 1 progress.console.print( - f" [red]✗ {fname}: HTTP {resp.status_code}[/red]" + f" [red]✗ {fname}: {type(e).__name__} - {e} (after {max_retries} attempts)[/red]" ) - try: - error_detail = resp.json().get("message", resp.text[:100]) - progress.console.print(f" [dim]{error_detail}[/dim]") - except Exception: - progress.console.print(f" [dim]{resp.text[:100]}[/dim]") - except requests.exceptions.ConnectionError: - failed += 1 - person_failed += 1 - progress.console.print( - f" [red]✗ {fname}: Connection refused[/red]" - ) - except requests.exceptions.Timeout: - failed += 1 - person_failed += 1 - progress.console.print( - f" [red]✗ {fname}: Request timed out (30s)[/red]" - ) - except Exception as e: - failed += 1 - person_failed += 1 - progress.console.print( - f" [red]✗ {fname}: {type(e).__name__} - {e}[/red]" - ) progress.advance(upload_task) @@ -349,8 +367,8 @@ def _perform_selection(assets: list, limit: int | str, name: str, selection_mode model_display = "InsightFace (face embeddings)" if entity_type == "face" else "SigLIP (visual embeddings)" rprint(f"\n[cyan]Using {model_display} for diversity analysis...[/cyan]") - # Pre-load model to avoid interference with progress bar - is_embedding_available(entity_type) + # Pre-load model explicitly (separate from availability check) + load_embedding_model(entity_type) with Progress( SpinnerColumn(), diff --git a/if_curator/config.py b/if_curator/config.py index 4a87b41..7397a98 100644 --- a/if_curator/config.py +++ b/if_curator/config.py @@ -14,10 +14,10 @@ load_dotenv() CONFIG_FILE = Path(".immich_config.json") -class Config: +class _Config: """Singleton configuration with uppercase attribute access for backward compatibility.""" - _instance: ClassVar["Config | None"] = None + _instance: ClassVar["_Config | None"] = None # Configuration values IMMICH_URL: str | None = None @@ -26,11 +26,14 @@ class Config: YEARS_FILTER: int = 10 # Quality filtering - MIN_FACE_WIDTH: int = 100 + MIN_FACE_WIDTH: int = 50 BLUR_THRESHOLD: float = 100.0 MIN_CONFIDENCE: float = 0.7 MAX_AUTO_IMAGES: int = 80 + # People filtering + MIN_FACE_COUNT: int = 0 + # Output quality FACE_MARGIN: float = 0.15 USE_FULL_RESOLUTION: bool = True @@ -40,7 +43,7 @@ class Config: ENABLE_CACHE: bool = False CACHE_DIR: str = ".if_cache" - def __new__(cls) -> "Config": + def __new__(cls) -> "_Config": if cls._instance is None: cls._instance = super().__new__(cls) cls._instance._load() @@ -54,6 +57,9 @@ class Config: self.OUTPUT_DIR = os.getenv("OUTPUT_DIR", "./frigate_train") self.YEARS_FILTER = int(os.getenv("YEARS_FILTER", "10")) self.MIN_FACE_WIDTH = int(os.getenv("MIN_FACE_WIDTH", "50")) + self.MIN_FACE_COUNT = int(os.getenv("MIN_FACE_COUNT", "0")) + self.ENABLE_CACHE = os.getenv("ENABLE_CACHE", "false").lower() in ("true", "1", "yes") + self.CACHE_DIR = os.getenv("CACHE_DIR", ".if_cache") # Fall back to config file for missing values if CONFIG_FILE.exists(): @@ -66,6 +72,11 @@ class Config: except (json.JSONDecodeError, OSError) as e: logging.warning(f"Failed to load config file: {e}") + @classmethod + def reset(cls) -> None: + """Reset the singleton — mainly useful for testing or delayed env setup.""" + cls._instance = None + def save(self) -> None: """Persist configuration to file.""" try: @@ -105,11 +116,47 @@ class Config: raise ValueError("Missing Immich URL or API Key.") -# Singleton instance and backward-compatible aliases -Config = Config() # type: ignore[misc] -ConfigManager = type("ConfigManager", (), {"get": staticmethod(lambda: Config)}) +# Singleton instance — use a lazy property pattern to avoid import-time side effects +# when env vars aren't yet set. Call Config.instance() or just access attributes on +# the module-level `Config` (which delegates to the singleton). +class _ConfigAccessor: + """Lazy accessor that defers singleton creation until first attribute access. + + This avoids reading .env and config files at import time, so environment + variables set after importing the module are properly picked up. + """ + + def __getattr__(self, name: str): + return getattr(_Config(), name) + + def __setattr__(self, name: str, value): + if name.startswith("_"): + super().__setattr__(name, value) + else: + setattr(_Config(), name, value) + + def reset(self) -> None: + """Reset the underlying singleton.""" + _Config.reset() + + def interactive_setup(self) -> None: + """Delegate to the singleton.""" + _Config().interactive_setup() + + def validate(self) -> None: + """Delegate to the singleton.""" + _Config().validate() + + def save(self) -> None: + """Delegate to the singleton.""" + _Config().save() + + +Config = _ConfigAccessor() +ConfigManager = type("ConfigManager", (), {"get": staticmethod(lambda: _Config())}) def get_headers() -> dict[str, str]: """Return HTTP headers for Immich API requests.""" return {"x-api-key": Config.API_KEY or "", "Accept": "application/json"} + diff --git a/if_curator/embeddings.py b/if_curator/embeddings.py index c21b660..9b6feef 100644 --- a/if_curator/embeddings.py +++ b/if_curator/embeddings.py @@ -7,6 +7,7 @@ Unified embedding interface for faces and objects. """ import contextlib +import importlib import logging import os import warnings @@ -21,8 +22,10 @@ logger = logging.getLogger(__name__) # Lazy-loaded singletons _insightface_app = None +_insightface_loaded = False _siglip_model = None _siglip_processor = None +_siglip_loaded = False def _is_force_cpu() -> bool: @@ -37,9 +40,10 @@ def _is_force_cpu() -> bool: def get_insightface_app(): """Singleton for InsightFace app with automatic GPU/CPU fallback.""" - global _insightface_app - if _insightface_app is not None: + global _insightface_app, _insightface_loaded + if _insightface_loaded: return _insightface_app + _insightface_loaded = True try: import onnxruntime as ort @@ -120,9 +124,10 @@ def get_face_embedding(img_pil: Image.Image) -> np.ndarray | None: def get_siglip_model(): """Singleton for SigLIP model and processor with GPU auto-detection.""" - global _siglip_model, _siglip_processor - if _siglip_model is not None: + global _siglip_model, _siglip_processor, _siglip_loaded + if _siglip_loaded: return _siglip_model, _siglip_processor + _siglip_loaded = True try: import warnings @@ -264,9 +269,45 @@ def get_embedding( return emb -def is_embedding_available(entity_type: str = "face") -> bool: - """Check if embedding model is available for the given entity type.""" +def _is_module_available(module_name: str) -> bool: + """Check if a Python module is importable without importing it fully.""" + try: + importlib.util.find_spec(module_name) + return True + except (ModuleNotFoundError, ValueError): + return False + + +def is_embedding_available(entity_type: str = "face", *, load: bool = False) -> bool: + """Check if embedding model is available for the given entity type. + + By default this performs a lightweight import-check only (no model loading). + Pass ``load=True`` to actually load the model (expensive, hundreds of MB). + + Args: + entity_type: 'face' or 'object' + load: If True, fully load the model to verify. If False (default), + only check that the required packages are importable. + """ + if load: + if entity_type == "face": + return get_insightface_app() is not None + model, _ = get_siglip_model() + return model is not None + + # Lightweight check: just verify the packages are importable + if entity_type == "face": + return _is_module_available("insightface") and _is_module_available("onnxruntime") + return _is_module_available("transformers") and _is_module_available("torch") + + +def load_embedding_model(entity_type: str = "face") -> bool: + """Explicitly load the embedding model for the given entity type. + + Returns True if the model loaded successfully. + """ if entity_type == "face": return get_insightface_app() is not None model, _ = get_siglip_model() return model is not None + diff --git a/if_curator/image_processing.py b/if_curator/image_processing.py index 66a926f..df9c72c 100644 --- a/if_curator/image_processing.py +++ b/if_curator/image_processing.py @@ -140,15 +140,17 @@ def process_object_mode( results = model(img, verbose=False, device=device) found = False - for idx, (box, cls_id, conf) in enumerate( - (box, int(box.cls[0]), float(box.conf[0])) for r in results for box in r.boxes - ): + class_idx = 0 # Sequential counter per target class (Issue #10) + for box in (box for r in results for box in r.boxes): + cls_id = int(box.cls[0]) + conf = float(box.conf[0]) if 0 <= cls_id < len(model.names) and model.names[cls_id] == target_class and conf > 0.5: x1, y1, x2, y2 = box.xyxy[0].tolist() img.crop((x1, y1, x2, y2)).save( - os.path.join(output_dir, f"{count}_{idx}.jpg"), + os.path.join(output_dir, f"{count}_{class_idx}.jpg"), format="JPEG", ) + class_idx += 1 found = True return found @@ -161,3 +163,4 @@ def process_full_mode(img: Image.Image, output_dir: str, count: int) -> bool: """Save full image.""" img.save(os.path.join(output_dir, f"{count}.jpg"), format="JPEG") return True + diff --git a/if_curator/immich_api.py b/if_curator/immich_api.py index b81a563..c49a7cd 100644 --- a/if_curator/immich_api.py +++ b/if_curator/immich_api.py @@ -13,6 +13,8 @@ from .config import Config, get_headers logger = logging.getLogger(__name__) +MAX_PAGES = 1000 # Safety limit for pagination + @dataclass class FaceData: @@ -50,7 +52,7 @@ def fetch_all_assets(person: dict) -> list[dict]: logger.info(f"Fetching assets for {name}...") assets = [] - for page in range(1, 1000): # Safety limit + for page in range(1, MAX_PAGES + 1): try: resp = requests.post( url, @@ -209,3 +211,4 @@ def filter_recent_assets(assets: list[dict], years: int | None = None) -> list[d logger.info(f"Retained {len(recent)} assets (filtered {skipped} old assets).") return recent + diff --git a/if_curator/logging.py b/if_curator/logging.py index 870cce5..1ce402d 100644 --- a/if_curator/logging.py +++ b/if_curator/logging.py @@ -1,6 +1,7 @@ """Logging configuration for if-curator.""" import logging +import os import warnings from rich.console import Console @@ -33,8 +34,11 @@ def setup_logging(verbose: bool = False) -> logging.Logger: # Rich console handler - uses shared console to avoid breaking progress bars root.addHandler(RichHandler(rich_tracebacks=True, markup=True, console=console)) - # File handler (always debug level) - file_handler = logging.FileHandler("immich_export.log") + # File handler (always debug level) — log file respects OUTPUT_DIR if set + log_dir = os.environ.get("OUTPUT_DIR", ".") + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, "immich_export.log") + file_handler = logging.FileHandler(log_path) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) root.addHandler(file_handler) @@ -48,3 +52,4 @@ def setup_logging(verbose: bool = False) -> logging.Logger: warnings.filterwarnings("ignore", category=FutureWarning, module="transformers") return root + diff --git a/if_curator/quality.py b/if_curator/quality.py index 477391a..6d4ad3e 100644 --- a/if_curator/quality.py +++ b/if_curator/quality.py @@ -70,7 +70,7 @@ def check_exposure(img_np: np.ndarray, lo: float = 30.0, hi: float = 225.0) -> t return True, "" -def check_face_size(face_width: float, face_height: float, min_px: int = 100) -> tuple[bool, str]: +def check_face_size(face_width: float, face_height: float, min_px: int = 50) -> tuple[bool, str]: """Validate face crop is large enough for meaningful features.""" if face_width < min_px or face_height < min_px: return False, f"Face too small ({face_width:.0f}x{face_height:.0f}, min={min_px}px)" @@ -94,7 +94,7 @@ def assess_quality( face_bbox: tuple[float, float, float, float] | None = None, confidence: float | None = None, blur_threshold: float = 100.0, - min_face_px: int = 100, + min_face_px: int = 50, min_confidence: float = 0.7, ) -> QualityResult: """Run all quality checks on an image. @@ -130,3 +130,4 @@ def assess_quality( reasons.append(reason) return QualityResult(passed=len(reasons) == 0, reasons=reasons) + diff --git a/pyproject.toml b/pyproject.toml index d770981..1532714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Immich to Frigate training sets" readme = "README.md" license = "MIT" requires-python = ">=3.12" -authors = [{ name = "ds-sebastian" }] +authors = [{ name = "sudolulo" }] keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"] classifiers = [ "Development Status :: 3 - Alpha", @@ -23,21 +23,23 @@ dependencies = [ "python-dotenv>=1.2.1", "requests>=2.32.5", "rich>=14.2.0", - "torch>=2.9.1", - "transformers>=4.57.6", - "ultralytics>=8.3.252", ] [project.scripts] if-curator = "if_curator.cli:main" [project.urls] -Repository = "https://github.com/ds-sebastian/if_curator" +Repository = "https://github.com/sudolulo/if_curator_headless" [project.optional-dependencies] gpu = [ "onnxruntime-gpu>=1.23.2", ] +object = [ + "torch>=2.9.1", + "transformers>=4.57.6", + "ultralytics>=8.3.252", +] [dependency-groups] dev = [ @@ -75,3 +77,4 @@ DEP002 = ["onnxruntime-gpu"] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" + diff --git a/scheduler.py b/scheduler.py index b2d3e5a..8f104ec 100644 --- a/scheduler.py +++ b/scheduler.py @@ -3,23 +3,56 @@ import os import sys import subprocess import time -from croniter import croniter +import logging +from pathlib import Path + +try: + from croniter import croniter +except ImportError: + print("❌ croniter not installed. Run: uv add croniter") + sys.exit(1) + +SCHEDULE = os.environ["CRON_SCHEDULE"] +MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface") +INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models") -SCHEDULE = os.environ.get("CRON_SCHEDULE", "0 3 * * SUN") RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"} -def run_curator(): - print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting if-curator...", flush=True) - subprocess.run(["uv", "run", "python", "-m", "if_curator.cli"], env=RUN_ENV) +logger = logging.getLogger(__name__) -now = time.time() -cron = croniter(SCHEDULE, now) + +def check_models(): + """Log model status before each run.""" + print("📦 Checking models...", flush=True) + buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l" + if buffalo.exists(): + print(" ✅ InsightFace Buffalo_L: present", flush=True) + else: + print(" ⬇️ InsightFace Buffalo_L: not found — will download", flush=True) + + hf_hub = Path(MODELS_DIR) / "hub" + if hf_hub.exists() and any(hf_hub.iterdir()): + print(" ✅ HuggingFace models: present", flush=True) + else: + print(" ⬇️ HuggingFace models: not found — will download", flush=True) + print("🚀 Starting if-curator...", flush=True) + + +NOW = time.time() +cron = croniter(SCHEDULE, NOW) next_run = cron.get_next(float) -print(f"▶ Scheduler active. Next run: {time.ctime(next_run)}") while True: - if time.time() >= next_run: - run_curator() + now = time.time() + if now >= next_run: + print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting if-curator...", flush=True) + check_models() + result = subprocess.run(["uv", "run", "if-curator"], env=RUN_ENV) + if result.returncode != 0: + logger.error(f"if-curator exited with code {result.returncode}") + print(f"❌ if-curator failed with exit code {result.returncode}", flush=True) + else: + print("✅ if-curator completed successfully", flush=True) next_run = cron.get_next(float) time.sleep(60)