From f50d3fe1abaab4888af65c674b66ab21c1db0b44 Mon Sep 17 00:00:00 2001 From: Holden Date: Fri, 12 Jun 2026 05:30:37 +0000 Subject: [PATCH] refactor: rename logging.py, in-process scheduler, trim .gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - winnow/logging.py ā†’ winnow/log_config.py: avoids shadowing the stdlib logging module; log file renamed from immich_export.log to winnow.log - scheduler.py: run main() in-process instead of subprocess.run so InsightFace and SigLIP models stay resident in memory across scheduled runs (hundreds of MB load, previously reloaded every run) - .gitignore: replaced 200-line boilerplate with ~30 project-relevant patterns; removed Django/Flask/Redis/RabbitMQ/Scrapy/etc. noise - .python-version: untracked (redundant with requires-python in pyproject.toml; kept in .gitignore for local pyenv users) Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 243 +++------------------------ .python-version | 1 - scheduler.py | 44 +++-- winnow/cli.py | 2 +- winnow/executor.py | 2 +- winnow/jobs.py | 2 +- winnow/{logging.py => log_config.py} | 2 +- 7 files changed, 43 insertions(+), 253 deletions(-) delete mode 100644 .python-version rename winnow/{logging.py => log_config.py} (96%) diff --git a/.gitignore b/.gitignore index fe18263..bf99b82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,239 +1,36 @@ -# Custom +# Output and runtime artefacts frigate_train/ *.log runs/ + +# Model and cache files yolov9c.pt .insightface/ .huggingface/ -.cache/huggingface .if_cache/ - .immich_config.json -# Python-generated files -__pycache__/ -*.py[oc] -build/ -dist/ -wheels/ -*.egg-info -# Virtual environments -.venv - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# Redis -*.rdb -*.aof -*.pid - -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ - -# ActiveMQ -activemq-data/ - -# SageMath parsed files -*.sage.py - -# Environments +# Secrets .env .envrc -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ -# Spyder project settings -.spyderproject -.spyproject +# Python +__pycache__/ +*.py[oc] +*.so +.Python -# Rope project settings -.ropeproject +# Packaging +build/ +dist/ +*.egg-info/ +wheels/ -# mkdocs documentation -/site +# Virtual environments +.venv/ -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ - -# Ruff stuff: +# Tools .ruff_cache/ - -# PyPI configuration file -.pypirc - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Streamlit -.streamlit/secrets.toml -compose.override.yml +.pytest_cache/ +.mypy_cache/ +.python-version diff --git a/.python-version b/.python-version deleted file mode 100644 index 24ee5b1..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 diff --git a/scheduler.py b/scheduler.py index 7498593..2481185 100644 --- a/scheduler.py +++ b/scheduler.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import logging import os -import subprocess import sys import time from pathlib import Path @@ -9,35 +8,31 @@ from pathlib import Path try: from croniter import croniter except ImportError: - print("āŒ croniter not installed. Run: uv add croniter") + print("croniter not installed. Run: uv add croniter") sys.exit(1) SCHEDULE = os.environ["CRON_SCHEDULE"] MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface") INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models") -RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"} - logger = logging.getLogger(__name__) -def check_models(): - """Log model status before each run.""" - print("šŸ“¦ Checking models...", flush=True) +def check_models() -> None: buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l" - if buffalo.exists(): - print(" āœ… InsightFace Buffalo_L: present", flush=True) - else: - print(" ā¬‡ļø InsightFace Buffalo_L: not found ā€” will download", flush=True) - hf_hub = Path(MODELS_DIR) / "hub" - if hf_hub.exists() and any(hf_hub.iterdir()): - print(" āœ… HuggingFace models: present", flush=True) - else: - print(" ā¬‡ļø HuggingFace models: not found ā€” will download", flush=True) - print("šŸš€ Starting winnow...", flush=True) + buffalo_ok = buffalo.exists() + hf_ok = hf_hub.exists() and any(hf_hub.iterdir()) + if not buffalo_ok: + print(" InsightFace Buffalo_L not found ā€” will download on first run", flush=True) + if not hf_ok: + print(" HuggingFace models not found ā€” will download on first run", flush=True) +# Import once ā€” models loaded during the first run stay resident in memory +# for all subsequent scheduled runs, avoiding repeated multi-GB load times. +from winnow.cli import main # noqa: E402 + NOW = time.time() cron = croniter(SCHEDULE, NOW) next_run = cron.get_next(float) @@ -45,14 +40,13 @@ next_run = cron.get_next(float) while True: now = time.time() if now >= next_run: - print(f"\nā–¶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow...", flush=True) + print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True) check_models() - result = subprocess.run(["uv", "run", "winnow"], env=RUN_ENV) - if result.returncode != 0: - logger.error(f"winnow exited with code {result.returncode}") - print(f"āŒ winnow failed with exit code {result.returncode}", flush=True) - else: - print("āœ… winnow completed successfully", flush=True) + try: + main() + print("winnow run complete", flush=True) + except Exception as e: + logger.error(f"winnow run failed: {e}", exc_info=True) + print(f"winnow run failed: {e}", flush=True) next_run = cron.get_next(float) time.sleep(60) - diff --git a/winnow/cli.py b/winnow/cli.py index d052595..d13beb0 100644 --- a/winnow/cli.py +++ b/winnow/cli.py @@ -10,7 +10,7 @@ from .config import Config, ConfigManager from .executor import execute_jobs, upload_to_frigate from .immich_api import get_people from .jobs import _show_preview, auto_configure, interactive_configure -from .logging import console, setup_logging +from .log_config import console, setup_logging from .upload_tracker import get_person_summary, reset_person logger = logging.getLogger(__name__) diff --git a/winnow/executor.py b/winnow/executor.py index 5f9e85e..4bc608f 100644 --- a/winnow/executor.py +++ b/winnow/executor.py @@ -14,7 +14,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn from .config import Config, get_headers from .image_processing import process_face_mode, process_full_mode, process_object_mode from .immich_api import fetch_face_data, fetch_full_image -from .logging import console +from .log_config import console from .upload_tracker import mark_rejected, mark_uploaded logger = logging.getLogger(__name__) diff --git a/winnow/jobs.py b/winnow/jobs.py index 74d6ddc..4075b21 100644 --- a/winnow/jobs.py +++ b/winnow/jobs.py @@ -13,7 +13,7 @@ from .diversity import select_diverse_assets from .embeddings import is_embedding_available, load_embedding_model from .frigate_api import get_frigate_face_counts from .immich_api import fetch_all_assets, filter_recent_assets -from .logging import console +from .log_config import console from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count logger = logging.getLogger(__name__) diff --git a/winnow/logging.py b/winnow/log_config.py similarity index 96% rename from winnow/logging.py rename to winnow/log_config.py index 3e2043a..e8378bc 100644 --- a/winnow/logging.py +++ b/winnow/log_config.py @@ -37,7 +37,7 @@ def setup_logging(verbose: bool = False) -> logging.Logger: # File handler (always debug level) ā€” log file respects OUTPUT_DIR if set log_dir = os.environ.get("OUTPUT_DIR", ".") os.makedirs(log_dir, exist_ok=True) - log_path = os.path.join(log_dir, "immich_export.log") + log_path = os.path.join(log_dir, "winnow.log") file_handler = logging.FileHandler(log_path) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))