refactor: rename logging.py, in-process scheduler, trim .gitignore
- winnow/logging.py → winnow/log_config.py: avoids shadowing the stdlib logging module; log file renamed from immich_export.log to winnow.log - scheduler.py: run main() in-process instead of subprocess.run so InsightFace and SigLIP models stay resident in memory across scheduled runs (hundreds of MB load, previously reloaded every run) - .gitignore: replaced 200-line boilerplate with ~30 project-relevant patterns; removed Django/Flask/Redis/RabbitMQ/Scrapy/etc. noise - .python-version: untracked (redundant with requires-python in pyproject.toml; kept in .gitignore for local pyenv users) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
+20
-223
@@ -1,239 +1,36 @@
|
||||
# Custom
|
||||
# Output and runtime artefacts
|
||||
frigate_train/
|
||||
*.log
|
||||
runs/
|
||||
|
||||
# Model and cache files
|
||||
yolov9c.pt
|
||||
.insightface/
|
||||
.huggingface/
|
||||
.cache/huggingface
|
||||
.if_cache/
|
||||
|
||||
.immich_config.json
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
# Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
# poetry.lock
|
||||
# poetry.toml
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||
# pdm.lock
|
||||
# pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# pixi
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||
# pixi.lock
|
||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||
.pixi
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# Redis
|
||||
*.rdb
|
||||
*.aof
|
||||
*.pid
|
||||
|
||||
# RabbitMQ
|
||||
mnesia/
|
||||
rabbitmq/
|
||||
rabbitmq-data/
|
||||
|
||||
# ActiveMQ
|
||||
activemq-data/
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
# Secrets
|
||||
.env
|
||||
.envrc
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
*.so
|
||||
.Python
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
# Packaging
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
wheels/
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
# Virtual environments
|
||||
.venv/
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
# .idea/
|
||||
|
||||
# Abstra
|
||||
# Abstra is an AI-powered process automation framework.
|
||||
# Ignore directories containing user credentials, local state, and settings.
|
||||
# Learn more at https://abstra.io/docs
|
||||
.abstra/
|
||||
|
||||
# Visual Studio Code
|
||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||
# you could uncomment the following to ignore the entire vscode folder
|
||||
# .vscode/
|
||||
|
||||
# Ruff stuff:
|
||||
# Tools
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Marimo
|
||||
marimo/_static/
|
||||
marimo/_lsp/
|
||||
__marimo__/
|
||||
|
||||
# Streamlit
|
||||
.streamlit/secrets.toml
|
||||
compose.override.yml
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.python-version
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
3.13
|
||||
+19
-25
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -9,35 +8,31 @@ from pathlib import Path
|
||||
try:
|
||||
from croniter import croniter
|
||||
except ImportError:
|
||||
print("❌ croniter not installed. Run: uv add croniter")
|
||||
print("croniter not installed. Run: uv add croniter")
|
||||
sys.exit(1)
|
||||
|
||||
SCHEDULE = os.environ["CRON_SCHEDULE"]
|
||||
MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface")
|
||||
INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models")
|
||||
|
||||
RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_models():
|
||||
"""Log model status before each run."""
|
||||
print("📦 Checking models...", flush=True)
|
||||
def check_models() -> None:
|
||||
buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l"
|
||||
if buffalo.exists():
|
||||
print(" ✅ InsightFace Buffalo_L: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ InsightFace Buffalo_L: not found — will download", flush=True)
|
||||
|
||||
hf_hub = Path(MODELS_DIR) / "hub"
|
||||
if hf_hub.exists() and any(hf_hub.iterdir()):
|
||||
print(" ✅ HuggingFace models: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ HuggingFace models: not found — will download", flush=True)
|
||||
print("🚀 Starting winnow...", flush=True)
|
||||
buffalo_ok = buffalo.exists()
|
||||
hf_ok = hf_hub.exists() and any(hf_hub.iterdir())
|
||||
if not buffalo_ok:
|
||||
print(" InsightFace Buffalo_L not found — will download on first run", flush=True)
|
||||
if not hf_ok:
|
||||
print(" HuggingFace models not found — will download on first run", flush=True)
|
||||
|
||||
|
||||
# Import once — models loaded during the first run stay resident in memory
|
||||
# for all subsequent scheduled runs, avoiding repeated multi-GB load times.
|
||||
from winnow.cli import main # noqa: E402
|
||||
|
||||
NOW = time.time()
|
||||
cron = croniter(SCHEDULE, NOW)
|
||||
next_run = cron.get_next(float)
|
||||
@@ -45,14 +40,13 @@ next_run = cron.get_next(float)
|
||||
while True:
|
||||
now = time.time()
|
||||
if now >= next_run:
|
||||
print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow...", flush=True)
|
||||
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
|
||||
check_models()
|
||||
result = subprocess.run(["uv", "run", "winnow"], env=RUN_ENV)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"winnow exited with code {result.returncode}")
|
||||
print(f"❌ winnow failed with exit code {result.returncode}", flush=True)
|
||||
else:
|
||||
print("✅ winnow completed successfully", flush=True)
|
||||
try:
|
||||
main()
|
||||
print("winnow run complete", flush=True)
|
||||
except Exception as e:
|
||||
logger.error(f"winnow run failed: {e}", exc_info=True)
|
||||
print(f"winnow run failed: {e}", flush=True)
|
||||
next_run = cron.get_next(float)
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ from .config import Config, ConfigManager
|
||||
from .executor import execute_jobs, upload_to_frigate
|
||||
from .immich_api import get_people
|
||||
from .jobs import _show_preview, auto_configure, interactive_configure
|
||||
from .logging import console, setup_logging
|
||||
from .log_config import console, setup_logging
|
||||
from .upload_tracker import get_person_summary, reset_person
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn
|
||||
from .config import Config, get_headers
|
||||
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
||||
from .immich_api import fetch_face_data, fetch_full_image
|
||||
from .logging import console
|
||||
from .log_config import console
|
||||
from .upload_tracker import mark_rejected, mark_uploaded
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ from .diversity import select_diverse_assets
|
||||
from .embeddings import is_embedding_available, load_embedding_model
|
||||
from .frigate_api import get_frigate_face_counts
|
||||
from .immich_api import fetch_all_assets, filter_recent_assets
|
||||
from .logging import console
|
||||
from .log_config import console
|
||||
from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -37,7 +37,7 @@ def setup_logging(verbose: bool = False) -> logging.Logger:
|
||||
# File handler (always debug level) — log file respects OUTPUT_DIR if set
|
||||
log_dir = os.environ.get("OUTPUT_DIR", ".")
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
log_path = os.path.join(log_dir, "immich_export.log")
|
||||
log_path = os.path.join(log_dir, "winnow.log")
|
||||
file_handler = logging.FileHandler(log_path)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
|
||||
Reference in New Issue
Block a user