refactor: rename logging.py, in-process scheduler, trim .gitignore
- winnow/logging.py → winnow/log_config.py: avoids shadowing the stdlib logging module; log file renamed from immich_export.log to winnow.log - scheduler.py: run main() in-process instead of subprocess.run so InsightFace and SigLIP models stay resident in memory across scheduled runs (hundreds of MB load, previously reloaded every run) - .gitignore: replaced 200-line boilerplate with ~30 project-relevant patterns; removed Django/Flask/Redis/RabbitMQ/Scrapy/etc. noise - .python-version: untracked (redundant with requires-python in pyproject.toml; kept in .gitignore for local pyenv users) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+19
-25
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -9,35 +8,31 @@ from pathlib import Path
|
||||
try:
|
||||
from croniter import croniter
|
||||
except ImportError:
|
||||
print("❌ croniter not installed. Run: uv add croniter")
|
||||
print("croniter not installed. Run: uv add croniter")
|
||||
sys.exit(1)
|
||||
|
||||
SCHEDULE = os.environ["CRON_SCHEDULE"]
|
||||
MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface")
|
||||
INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models")
|
||||
|
||||
RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_models():
|
||||
"""Log model status before each run."""
|
||||
print("📦 Checking models...", flush=True)
|
||||
def check_models() -> None:
|
||||
buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l"
|
||||
if buffalo.exists():
|
||||
print(" ✅ InsightFace Buffalo_L: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ InsightFace Buffalo_L: not found — will download", flush=True)
|
||||
|
||||
hf_hub = Path(MODELS_DIR) / "hub"
|
||||
if hf_hub.exists() and any(hf_hub.iterdir()):
|
||||
print(" ✅ HuggingFace models: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ HuggingFace models: not found — will download", flush=True)
|
||||
print("🚀 Starting winnow...", flush=True)
|
||||
buffalo_ok = buffalo.exists()
|
||||
hf_ok = hf_hub.exists() and any(hf_hub.iterdir())
|
||||
if not buffalo_ok:
|
||||
print(" InsightFace Buffalo_L not found — will download on first run", flush=True)
|
||||
if not hf_ok:
|
||||
print(" HuggingFace models not found — will download on first run", flush=True)
|
||||
|
||||
|
||||
# Import once — models loaded during the first run stay resident in memory
|
||||
# for all subsequent scheduled runs, avoiding repeated multi-GB load times.
|
||||
from winnow.cli import main # noqa: E402
|
||||
|
||||
NOW = time.time()
|
||||
cron = croniter(SCHEDULE, NOW)
|
||||
next_run = cron.get_next(float)
|
||||
@@ -45,14 +40,13 @@ next_run = cron.get_next(float)
|
||||
while True:
|
||||
now = time.time()
|
||||
if now >= next_run:
|
||||
print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow...", flush=True)
|
||||
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
|
||||
check_models()
|
||||
result = subprocess.run(["uv", "run", "winnow"], env=RUN_ENV)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"winnow exited with code {result.returncode}")
|
||||
print(f"❌ winnow failed with exit code {result.returncode}", flush=True)
|
||||
else:
|
||||
print("✅ winnow completed successfully", flush=True)
|
||||
try:
|
||||
main()
|
||||
print("winnow run complete", flush=True)
|
||||
except Exception as e:
|
||||
logger.error(f"winnow run failed: {e}", exc_info=True)
|
||||
print(f"winnow run failed: {e}", flush=True)
|
||||
next_run = cron.get_next(float)
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user