refactor: rename logging.py, in-process scheduler, trim .gitignore
- winnow/logging.py → winnow/log_config.py: avoids shadowing the stdlib logging module; log file renamed from immich_export.log to winnow.log - scheduler.py: run main() in-process instead of subprocess.run so InsightFace and SigLIP models stay resident in memory across scheduled runs (hundreds of MB load, previously reloaded every run) - .gitignore: replaced 200-line boilerplate with ~30 project-relevant patterns; removed Django/Flask/Redis/RabbitMQ/Scrapy/etc. noise - .python-version: untracked (redundant with requires-python in pyproject.toml; kept in .gitignore for local pyenv users) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+20
-223
@@ -1,239 +1,36 @@
|
|||||||
# Custom
|
# Output and runtime artefacts
|
||||||
frigate_train/
|
frigate_train/
|
||||||
*.log
|
*.log
|
||||||
runs/
|
runs/
|
||||||
|
|
||||||
|
# Model and cache files
|
||||||
yolov9c.pt
|
yolov9c.pt
|
||||||
.insightface/
|
.insightface/
|
||||||
.huggingface/
|
.huggingface/
|
||||||
.cache/huggingface
|
|
||||||
.if_cache/
|
.if_cache/
|
||||||
|
|
||||||
.immich_config.json
|
.immich_config.json
|
||||||
# Python-generated files
|
|
||||||
__pycache__/
|
|
||||||
*.py[oc]
|
|
||||||
build/
|
|
||||||
dist/
|
|
||||||
wheels/
|
|
||||||
*.egg-info
|
|
||||||
|
|
||||||
# Virtual environments
|
# Secrets
|
||||||
.venv
|
|
||||||
|
|
||||||
# Byte-compiled / optimized / DLL files
|
|
||||||
__pycache__/
|
|
||||||
*.py[codz]
|
|
||||||
*$py.class
|
|
||||||
|
|
||||||
# C extensions
|
|
||||||
*.so
|
|
||||||
|
|
||||||
# Distribution / packaging
|
|
||||||
.Python
|
|
||||||
build/
|
|
||||||
develop-eggs/
|
|
||||||
dist/
|
|
||||||
downloads/
|
|
||||||
eggs/
|
|
||||||
.eggs/
|
|
||||||
lib/
|
|
||||||
lib64/
|
|
||||||
parts/
|
|
||||||
sdist/
|
|
||||||
var/
|
|
||||||
wheels/
|
|
||||||
share/python-wheels/
|
|
||||||
*.egg-info/
|
|
||||||
.installed.cfg
|
|
||||||
*.egg
|
|
||||||
MANIFEST
|
|
||||||
|
|
||||||
# PyInstaller
|
|
||||||
# Usually these files are written by a python script from a template
|
|
||||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
||||||
*.manifest
|
|
||||||
*.spec
|
|
||||||
|
|
||||||
# Installer logs
|
|
||||||
pip-log.txt
|
|
||||||
pip-delete-this-directory.txt
|
|
||||||
|
|
||||||
# Unit test / coverage reports
|
|
||||||
htmlcov/
|
|
||||||
.tox/
|
|
||||||
.nox/
|
|
||||||
.coverage
|
|
||||||
.coverage.*
|
|
||||||
.cache
|
|
||||||
nosetests.xml
|
|
||||||
coverage.xml
|
|
||||||
*.cover
|
|
||||||
*.py.cover
|
|
||||||
.hypothesis/
|
|
||||||
.pytest_cache/
|
|
||||||
cover/
|
|
||||||
|
|
||||||
# Translations
|
|
||||||
*.mo
|
|
||||||
*.pot
|
|
||||||
|
|
||||||
# Django stuff:
|
|
||||||
*.log
|
|
||||||
local_settings.py
|
|
||||||
db.sqlite3
|
|
||||||
db.sqlite3-journal
|
|
||||||
|
|
||||||
# Flask stuff:
|
|
||||||
instance/
|
|
||||||
.webassets-cache
|
|
||||||
|
|
||||||
# Scrapy stuff:
|
|
||||||
.scrapy
|
|
||||||
|
|
||||||
# Sphinx documentation
|
|
||||||
docs/_build/
|
|
||||||
|
|
||||||
# PyBuilder
|
|
||||||
.pybuilder/
|
|
||||||
target/
|
|
||||||
|
|
||||||
# Jupyter Notebook
|
|
||||||
.ipynb_checkpoints
|
|
||||||
|
|
||||||
# IPython
|
|
||||||
profile_default/
|
|
||||||
ipython_config.py
|
|
||||||
|
|
||||||
# pyenv
|
|
||||||
# For a library or package, you might want to ignore these files since the code is
|
|
||||||
# intended to run in multiple environments; otherwise, check them in:
|
|
||||||
# .python-version
|
|
||||||
|
|
||||||
# pipenv
|
|
||||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
||||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
||||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
||||||
# install all needed dependencies.
|
|
||||||
# Pipfile.lock
|
|
||||||
|
|
||||||
# UV
|
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
||||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
||||||
# commonly ignored for libraries.
|
|
||||||
# uv.lock
|
|
||||||
|
|
||||||
# poetry
|
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
||||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
||||||
# commonly ignored for libraries.
|
|
||||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
||||||
# poetry.lock
|
|
||||||
# poetry.toml
|
|
||||||
|
|
||||||
# pdm
|
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
||||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
||||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
||||||
# pdm.lock
|
|
||||||
# pdm.toml
|
|
||||||
.pdm-python
|
|
||||||
.pdm-build/
|
|
||||||
|
|
||||||
# pixi
|
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
||||||
# pixi.lock
|
|
||||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
||||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
||||||
.pixi
|
|
||||||
|
|
||||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
||||||
__pypackages__/
|
|
||||||
|
|
||||||
# Celery stuff
|
|
||||||
celerybeat-schedule
|
|
||||||
celerybeat.pid
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
*.rdb
|
|
||||||
*.aof
|
|
||||||
*.pid
|
|
||||||
|
|
||||||
# RabbitMQ
|
|
||||||
mnesia/
|
|
||||||
rabbitmq/
|
|
||||||
rabbitmq-data/
|
|
||||||
|
|
||||||
# ActiveMQ
|
|
||||||
activemq-data/
|
|
||||||
|
|
||||||
# SageMath parsed files
|
|
||||||
*.sage.py
|
|
||||||
|
|
||||||
# Environments
|
|
||||||
.env
|
.env
|
||||||
.envrc
|
.envrc
|
||||||
.venv
|
|
||||||
env/
|
|
||||||
venv/
|
|
||||||
ENV/
|
|
||||||
env.bak/
|
|
||||||
venv.bak/
|
|
||||||
|
|
||||||
# Spyder project settings
|
# Python
|
||||||
.spyderproject
|
__pycache__/
|
||||||
.spyproject
|
*.py[oc]
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
|
||||||
# Rope project settings
|
# Packaging
|
||||||
.ropeproject
|
build/
|
||||||
|
dist/
|
||||||
|
*.egg-info/
|
||||||
|
wheels/
|
||||||
|
|
||||||
# mkdocs documentation
|
# Virtual environments
|
||||||
/site
|
.venv/
|
||||||
|
|
||||||
# mypy
|
# Tools
|
||||||
.mypy_cache/
|
|
||||||
.dmypy.json
|
|
||||||
dmypy.json
|
|
||||||
|
|
||||||
# Pyre type checker
|
|
||||||
.pyre/
|
|
||||||
|
|
||||||
# pytype static type analyzer
|
|
||||||
.pytype/
|
|
||||||
|
|
||||||
# Cython debug symbols
|
|
||||||
cython_debug/
|
|
||||||
|
|
||||||
# PyCharm
|
|
||||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
||||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
||||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
||||||
# .idea/
|
|
||||||
|
|
||||||
# Abstra
|
|
||||||
# Abstra is an AI-powered process automation framework.
|
|
||||||
# Ignore directories containing user credentials, local state, and settings.
|
|
||||||
# Learn more at https://abstra.io/docs
|
|
||||||
.abstra/
|
|
||||||
|
|
||||||
# Visual Studio Code
|
|
||||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
||||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
||||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
||||||
# you could uncomment the following to ignore the entire vscode folder
|
|
||||||
# .vscode/
|
|
||||||
|
|
||||||
# Ruff stuff:
|
|
||||||
.ruff_cache/
|
.ruff_cache/
|
||||||
|
.pytest_cache/
|
||||||
# PyPI configuration file
|
.mypy_cache/
|
||||||
.pypirc
|
.python-version
|
||||||
|
|
||||||
# Marimo
|
|
||||||
marimo/_static/
|
|
||||||
marimo/_lsp/
|
|
||||||
__marimo__/
|
|
||||||
|
|
||||||
# Streamlit
|
|
||||||
.streamlit/secrets.toml
|
|
||||||
compose.override.yml
|
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
3.13
|
|
||||||
+19
-25
@@ -1,7 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import subprocess
|
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -9,35 +8,31 @@ from pathlib import Path
|
|||||||
try:
|
try:
|
||||||
from croniter import croniter
|
from croniter import croniter
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print("❌ croniter not installed. Run: uv add croniter")
|
print("croniter not installed. Run: uv add croniter")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
SCHEDULE = os.environ["CRON_SCHEDULE"]
|
SCHEDULE = os.environ["CRON_SCHEDULE"]
|
||||||
MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface")
|
MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface")
|
||||||
INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models")
|
INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models")
|
||||||
|
|
||||||
RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"}
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def check_models():
|
def check_models() -> None:
|
||||||
"""Log model status before each run."""
|
|
||||||
print("📦 Checking models...", flush=True)
|
|
||||||
buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l"
|
buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l"
|
||||||
if buffalo.exists():
|
|
||||||
print(" ✅ InsightFace Buffalo_L: present", flush=True)
|
|
||||||
else:
|
|
||||||
print(" ⬇️ InsightFace Buffalo_L: not found — will download", flush=True)
|
|
||||||
|
|
||||||
hf_hub = Path(MODELS_DIR) / "hub"
|
hf_hub = Path(MODELS_DIR) / "hub"
|
||||||
if hf_hub.exists() and any(hf_hub.iterdir()):
|
buffalo_ok = buffalo.exists()
|
||||||
print(" ✅ HuggingFace models: present", flush=True)
|
hf_ok = hf_hub.exists() and any(hf_hub.iterdir())
|
||||||
else:
|
if not buffalo_ok:
|
||||||
print(" ⬇️ HuggingFace models: not found — will download", flush=True)
|
print(" InsightFace Buffalo_L not found — will download on first run", flush=True)
|
||||||
print("🚀 Starting winnow...", flush=True)
|
if not hf_ok:
|
||||||
|
print(" HuggingFace models not found — will download on first run", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Import once — models loaded during the first run stay resident in memory
|
||||||
|
# for all subsequent scheduled runs, avoiding repeated multi-GB load times.
|
||||||
|
from winnow.cli import main # noqa: E402
|
||||||
|
|
||||||
NOW = time.time()
|
NOW = time.time()
|
||||||
cron = croniter(SCHEDULE, NOW)
|
cron = croniter(SCHEDULE, NOW)
|
||||||
next_run = cron.get_next(float)
|
next_run = cron.get_next(float)
|
||||||
@@ -45,14 +40,13 @@ next_run = cron.get_next(float)
|
|||||||
while True:
|
while True:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if now >= next_run:
|
if now >= next_run:
|
||||||
print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow...", flush=True)
|
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
|
||||||
check_models()
|
check_models()
|
||||||
result = subprocess.run(["uv", "run", "winnow"], env=RUN_ENV)
|
try:
|
||||||
if result.returncode != 0:
|
main()
|
||||||
logger.error(f"winnow exited with code {result.returncode}")
|
print("winnow run complete", flush=True)
|
||||||
print(f"❌ winnow failed with exit code {result.returncode}", flush=True)
|
except Exception as e:
|
||||||
else:
|
logger.error(f"winnow run failed: {e}", exc_info=True)
|
||||||
print("✅ winnow completed successfully", flush=True)
|
print(f"winnow run failed: {e}", flush=True)
|
||||||
next_run = cron.get_next(float)
|
next_run = cron.get_next(float)
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -10,7 +10,7 @@ from .config import Config, ConfigManager
|
|||||||
from .executor import execute_jobs, upload_to_frigate
|
from .executor import execute_jobs, upload_to_frigate
|
||||||
from .immich_api import get_people
|
from .immich_api import get_people
|
||||||
from .jobs import _show_preview, auto_configure, interactive_configure
|
from .jobs import _show_preview, auto_configure, interactive_configure
|
||||||
from .logging import console, setup_logging
|
from .log_config import console, setup_logging
|
||||||
from .upload_tracker import get_person_summary, reset_person
|
from .upload_tracker import get_person_summary, reset_person
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn
|
|||||||
from .config import Config, get_headers
|
from .config import Config, get_headers
|
||||||
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
||||||
from .immich_api import fetch_face_data, fetch_full_image
|
from .immich_api import fetch_face_data, fetch_full_image
|
||||||
from .logging import console
|
from .log_config import console
|
||||||
from .upload_tracker import mark_rejected, mark_uploaded
|
from .upload_tracker import mark_rejected, mark_uploaded
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
+1
-1
@@ -13,7 +13,7 @@ from .diversity import select_diverse_assets
|
|||||||
from .embeddings import is_embedding_available, load_embedding_model
|
from .embeddings import is_embedding_available, load_embedding_model
|
||||||
from .frigate_api import get_frigate_face_counts
|
from .frigate_api import get_frigate_face_counts
|
||||||
from .immich_api import fetch_all_assets, filter_recent_assets
|
from .immich_api import fetch_all_assets, filter_recent_assets
|
||||||
from .logging import console
|
from .log_config import console
|
||||||
from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count
|
from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ def setup_logging(verbose: bool = False) -> logging.Logger:
|
|||||||
# File handler (always debug level) — log file respects OUTPUT_DIR if set
|
# File handler (always debug level) — log file respects OUTPUT_DIR if set
|
||||||
log_dir = os.environ.get("OUTPUT_DIR", ".")
|
log_dir = os.environ.get("OUTPUT_DIR", ".")
|
||||||
os.makedirs(log_dir, exist_ok=True)
|
os.makedirs(log_dir, exist_ok=True)
|
||||||
log_path = os.path.join(log_dir, "immich_export.log")
|
log_path = os.path.join(log_dir, "winnow.log")
|
||||||
file_handler = logging.FileHandler(log_path)
|
file_handler = logging.FileHandler(log_path)
|
||||||
file_handler.setLevel(logging.DEBUG)
|
file_handler.setLevel(logging.DEBUG)
|
||||||
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
|
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
|
||||||
Reference in New Issue
Block a user