Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8103801ec8 | ||
|
|
a281b4b896 | ||
|
|
5345798dc1 | ||
|
|
046004a5d0 | ||
|
|
86ee9a5ba2 | ||
|
|
248b7a6270 | ||
|
|
5b25b0df06 | ||
|
|
af9949c47f | ||
|
|
82d057b235 | ||
|
|
fa6dc01366 | ||
|
|
567e568c47 | ||
|
|
1ed8d7e25e | ||
|
|
32e4235384 | ||
|
|
f1df1c6bb3 | ||
|
|
85e499a677 | ||
|
|
ad4d212df0 | ||
|
|
31e7ca5af0 | ||
|
|
4f6967258f | ||
|
|
882af37d8e | ||
|
|
cad053e88f | ||
|
|
3db52c2141 | ||
|
|
af8bae1c45 | ||
|
|
ac9e9530f3 | ||
|
|
015db63d19 | ||
|
|
2804b21f9e | ||
|
|
981a86f28a | ||
|
|
ee585d4bae | ||
|
|
99185c2da9 | ||
|
|
196b0a5147 | ||
|
|
125ce54c7f | ||
|
|
f50d3fe1ab | ||
|
|
af92fe5fcc | ||
|
|
4cac3d28d2 | ||
|
|
21208f8331 | ||
|
|
209e077055 | ||
|
|
e7c1fc5d9b | ||
|
|
14a36f2aea | ||
|
|
8ec69b7f56 | ||
|
|
7bd66c4218 | ||
|
|
57915c67fc | ||
|
|
619a96b8c3 | ||
|
|
a8bdbcdd66 | ||
|
|
9e2a09cda9 | ||
|
|
a3405108c9 | ||
|
|
d563aa9e74 | ||
|
|
34d4780554 | ||
|
|
67b05502b3 | ||
|
|
3087d59558 | ||
|
|
c59acb5e31 | ||
|
|
c8fbbc0431 | ||
|
|
c1cc0dd9c2 | ||
|
|
57668dabf4 | ||
|
|
2172b61254 | ||
|
|
10eb7265c1 | ||
|
|
7b4722d3f2 | ||
|
|
444932c369 | ||
|
|
8f2a6d3163 | ||
|
|
b6659c2218 |
+13
-5
@@ -4,7 +4,10 @@ API_KEY=your-immich-api-key
|
||||
FRIGATE_URL=http://192.168.1.10:5000
|
||||
|
||||
# ── Mode & Strategy ───────────────────────────────────────────────────────────
|
||||
AUTO_MODE=true
|
||||
# Auto mode is active by default when no TTY is present (Docker/cron).
|
||||
# Set AUTO_MODE=true to force auto mode even in an interactive terminal.
|
||||
# AUTO_MODE=true
|
||||
# VERBOSE=true # Enable DEBUG-level console output (log file is always DEBUG)
|
||||
# TRAINING_MODE: face = upload to Frigate face recognition API
|
||||
# object = save crops to output dir for manual Frigate placement
|
||||
TRAINING_MODE=face
|
||||
@@ -29,8 +32,8 @@ STRATEGY=auto
|
||||
# MAX_AUTO_IMAGES=80 # Hard cap on auto-diversity selection (default: 80)
|
||||
|
||||
# ── Caching & Models ──────────────────────────────────────────────────────────
|
||||
FORCE_CPU=false
|
||||
ENABLE_CACHE=true
|
||||
# FORCE_CPU=true # Disable GPU, fall back to CPU
|
||||
# ENABLE_CACHE=false # Disable embedding cache (default: true)
|
||||
CACHE_DIR=/app/.if_cache
|
||||
HF_HOME=/models/huggingface
|
||||
INSIGHTFACE_HOME=/models/.insightface
|
||||
@@ -41,5 +44,10 @@ INSIGHTFACE_HOME=/models/.insightface
|
||||
# RESET_PERSON=John # Clear uploaded+rejected history for one person
|
||||
|
||||
# ── Scheduling ────────────────────────────────────────────────────────────────
|
||||
# Cron expression (unset = run once and exit)
|
||||
CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
||||
# CRON_SCHEDULE controls container lifetime:
|
||||
# unset — run once on startup, then exit
|
||||
# empty string — stay alive, run nothing (trigger manually: docker exec -it winnow winnow)
|
||||
# cron expression — run on startup, then on schedule
|
||||
# CRON_SCHEDULE= # Manual mode (keep alive, no auto-run)
|
||||
# CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
||||
# CRON_SCHEDULE=0 3 1 * * # First of every month
|
||||
|
||||
@@ -10,6 +10,10 @@ on:
|
||||
- ".github/workflows/lint.yml"
|
||||
- ".github/dependabot.yml"
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: sudolulo/winnow
|
||||
@@ -39,17 +43,17 @@ jobs:
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform == 'linux/arm64'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -98,10 +102,10 @@ jobs:
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -126,3 +130,72 @@ jobs:
|
||||
- name: Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
gh api -X PATCH /user/packages/container/winnow \
|
||||
-f visibility=public || true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-cpu:
|
||||
name: Build CPU-only (amd64)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Determine CPU image tag
|
||||
id: cpu-tag
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cpu" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push CPU image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
build-args: VARIANT=cpu
|
||||
cache-from: type=gha,scope=linux/amd64-cpu
|
||||
cache-to: type=gha,mode=max,scope=linux/amd64-cpu
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
push: true
|
||||
tags: ${{ steps.cpu-tag.outputs.tag }}
|
||||
|
||||
- name: Inspect CPU image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.cpu-tag.outputs.tag }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
gh api -X PATCH /user/packages/container/winnow \
|
||||
-f visibility=public || true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -9,6 +9,8 @@ on:
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
steps:
|
||||
@@ -21,7 +23,7 @@ jobs:
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Run Ruff
|
||||
uses: astral-sh/ruff-action@v3
|
||||
|
||||
@@ -30,12 +30,12 @@ jobs:
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Ensure uv.lock is current
|
||||
run: uv lock
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Resolve tag name
|
||||
id: tag
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
RELEASE_TAG: ${{ steps.tag.outputs.TAG }}
|
||||
RELEASE_NOTES: ${{ steps.changelog.outputs.NOTES }}
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
});
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -9,15 +9,17 @@ on:
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
run: uv python install 3.13
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --all-extras
|
||||
|
||||
@@ -22,13 +22,13 @@ jobs:
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
run: uv python install 3.13
|
||||
|
||||
- name: Regenerate lockfile
|
||||
run: uv lock
|
||||
|
||||
+20
-223
@@ -1,239 +1,36 @@
|
||||
# Custom
|
||||
# Output and runtime artefacts
|
||||
frigate_train/
|
||||
*.log
|
||||
runs/
|
||||
|
||||
# Model and cache files
|
||||
yolov9c.pt
|
||||
.insightface/
|
||||
.huggingface/
|
||||
.cache/huggingface
|
||||
.if_cache/
|
||||
|
||||
.immich_config.json
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
# Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
# poetry.lock
|
||||
# poetry.toml
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||
# pdm.lock
|
||||
# pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# pixi
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||
# pixi.lock
|
||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||
.pixi
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# Redis
|
||||
*.rdb
|
||||
*.aof
|
||||
*.pid
|
||||
|
||||
# RabbitMQ
|
||||
mnesia/
|
||||
rabbitmq/
|
||||
rabbitmq-data/
|
||||
|
||||
# ActiveMQ
|
||||
activemq-data/
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
# Secrets
|
||||
.env
|
||||
.envrc
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
*.so
|
||||
.Python
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
# Packaging
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
wheels/
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
# Virtual environments
|
||||
.venv/
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
# .idea/
|
||||
|
||||
# Abstra
|
||||
# Abstra is an AI-powered process automation framework.
|
||||
# Ignore directories containing user credentials, local state, and settings.
|
||||
# Learn more at https://abstra.io/docs
|
||||
.abstra/
|
||||
|
||||
# Visual Studio Code
|
||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||
# you could uncomment the following to ignore the entire vscode folder
|
||||
# .vscode/
|
||||
|
||||
# Ruff stuff:
|
||||
# Tools
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Marimo
|
||||
marimo/_static/
|
||||
marimo/_lsp/
|
||||
__marimo__/
|
||||
|
||||
# Streamlit
|
||||
.streamlit/secrets.toml
|
||||
compose.override.yml
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.python-version
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
3.12
|
||||
+107
@@ -7,6 +7,113 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.2.11] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **GPU broken on x86_64 Linux**: `insightface` 1.0.1 (pulled in by the 0.2.10 lock update) added a hard dependency on the CPU `onnxruntime` package. Combined with an incorrect `override-dependencies` entry introduced in 0.2.10, both `onnxruntime` (CPU) and `onnxruntime-gpu` were being installed into the same venv. The CPU package landed last and overwrote the GPU one, causing `CUDAExecutionProvider` to disappear from the provider list even when a GPU was present. Fixed by declaring `onnxruntime` and `onnxruntime-gpu` as conflicting packages in uv's resolver, ensuring only the correct one is installed per platform.
|
||||
- **OOM crash on large person libraries (CPU mode)**: All candidate thumbnails were downloaded into a single in-memory dict before any processing began. At ~5 MB per decoded preview image, a person with 472 candidates would accumulate ~2.4 GB of thumbnail data alone, exhausting a 4 GB container memory limit. Thumbnails are now downloaded and processed in batches of 32, with each image released immediately after embedding. Peak in-flight thumbnail memory is now bounded to ~256 MB regardless of candidate pool size. GPU users also benefit from lower host RAM pressure and faster time-to-first-result on large libraries.
|
||||
|
||||
## [0.2.10] - 2026-06-12
|
||||
|
||||
### Added
|
||||
|
||||
- **`VERBOSE` env var**: set `VERBOSE=true` to enable DEBUG-level console output. The log file always captures DEBUG; this flag controls what appears on the terminal. Useful when diagnosing issues without a full shell into the container.
|
||||
- **`:cpu` Docker image tag**: a separate CPU-only image (`ghcr.io/sudolulo/winnow:cpu`) is now built and pushed alongside `:latest`. Uses `onnxruntime` instead of `onnxruntime-gpu`; ~2 GB smaller. Suitable for systems without an NVIDIA GPU.
|
||||
- **Empty `CRON_SCHEDULE` keeps container alive**: setting `CRON_SCHEDULE=` (empty string) starts the container without running immediately and without exiting — useful for `docker exec` ad-hoc runs on a long-lived container. Previously, an empty value was treated the same as unset (run once, then exit).
|
||||
|
||||
### Changed
|
||||
|
||||
- **TTY auto-detection replaces `AUTO_MODE`**: winnow now detects whether a TTY is attached (`sys.stdin.isatty()`) and switches between interactive and auto mode automatically. `AUTO_MODE=true` becomes an explicit override for forcing auto mode in a terminal session. No config change needed for normal Docker deployments.
|
||||
- **Logging levels audited**: internal algorithmic detail (clustering steps, asset fetch progress, per-page pagination) demoted from INFO to DEBUG. INFO now reflects meaningful pipeline milestones only (model ready, selection complete, quality filtered). Reduces noise in production logs without losing information.
|
||||
- **Model load logging improved**: SigLIP and InsightFace loading now reports cache hit/miss, download size estimate, device used, and load time.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **GPU OOM crash loop (production)**: `CUDAExecutionProvider` was silently absent even with a GPU attached, causing InsightFace to run on CPU and exhaust RAM processing large person libraries. Root cause: CUDA/cuDNN libraries in nvidia pip packages were invisible to onnxruntime. Fixed by running `ldconfig` over all `nvidia-*/lib/` directories in the venv at image build time.
|
||||
- **ldconfig path now Python-version-agnostic**: the `find` command used to register nvidia pip libraries hardcoded `python3.13`; replaced with `python3.*` glob so the path survives a Python upgrade without silently producing an empty ldconfig config.
|
||||
- **`PYTHONPATH=/app` added to Dockerfile**: the entry point script sets `sys.path[0]` to the script directory, not `/app`. Since `uv sync` runs before `COPY winnow/`, the wheel has only dist-info in site-packages. `PYTHONPATH=/app` makes the `winnow` package importable without reverting to `python -m`.
|
||||
- **CPU fallback retrying broken GPU provider**: InsightFace CPU fallback omitted `providers=["CPUExecutionProvider"]`, causing onnxruntime to retry `CUDAExecutionProvider` on every inference call. Now explicitly sets the CPU provider and suppresses C-extension noise via fd-level redirect.
|
||||
- **`_suppress_output` stderr loss on fd exhaustion**: if the first `os.dup2` in the finally block raised `OSError`, the second call was skipped, permanently redirecting stderr to `/dev/null` for the process lifetime. Wrapped in nested `try/finally` so both restores are always attempted.
|
||||
- **Frigate `/api/faces` response parsing**: the response is `{person_name: [files], "train": [...]}` — `"train"` is a flat pending list, not a person. Previous code called `.items()` on the `"train"` value (a list), crashing with `AttributeError`. Now skips the `"train"` key explicitly.
|
||||
- **Immich 401 detection**: a stale or invalid API key now logs a clear error message (`Immich API key is invalid or expired (401 Unauthorized)`) instead of raising an unhandled exception.
|
||||
- **Falsy-zero detection confidence**: `face.get("score") or face.get("confidence")` treated a valid `score=0.0` as falsy, falling through to the `confidence` field (often `None`). Replaced with an explicit `None` check. Affected both quality filtering and hard-example weighting in diversity selection.
|
||||
- **Face crop using wrong person's image dimensions**: in multi-person assets, `_crop_face_from_thumbnail`'s scale-factor loop matched the first person with any face regardless of `person_id`, producing incorrectly scaled bounding box coordinates for the target person. Loop now applies the same `person_id` filter as `_get_face_bbox`.
|
||||
- **Adaptive stopping bypassed for partially-trained people**: in auto mode with `already_uploaded > 0`, `limit` was converted from `"auto"` to an integer, disabling the FPS adaptive threshold and early-stop check. Now keeps `limit="auto"` through selection and trims the result to the remaining capacity afterward.
|
||||
- **Embedding cache key mismatch**: HuggingFace cache path check hardcoded the model slug string; replaced with a derivation from `model_name` using `"models--" + model_name.replace("/", "--")` so the check stays correct if the model name changes.
|
||||
- **Scheduler: sleep until next run**: the loop slept a fixed 60 seconds regardless of schedule interval, causing runs to fire up to 59 seconds late and waking the process unnecessarily on long schedules (e.g. weekly). Now sleeps exactly until `next_run`.
|
||||
- **Scheduler swallowing `SystemExit`**: `except BaseException` in the run wrapper was replaced with `except Exception` (with `KeyboardInterrupt` re-raised above), so `sys.exit()` calls propagate correctly.
|
||||
- **Log handler leak**: `setup_logging` now closes and removes existing handlers before adding new ones, preventing file handle accumulation across repeated calls.
|
||||
- **`RETRY_REJECTED` silently applied in interactive mode**: the env var was applied unconditionally even in interactive sessions. Now used only as the default for the interactive prompt so users can override it per-run.
|
||||
- **`compose.yml` comment inverted**: a comment stated `-it` forces non-interactive mode; corrected to reflect that `-it` allocates a TTY (interactive mode).
|
||||
|
||||
### Security
|
||||
|
||||
- API key is no longer stored in any config file. All authentication uses environment variables or `.env` only.
|
||||
|
||||
## [0.2.9] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **arm64: `onnxruntime-gpu` has no arm64 wheels**: `onnxruntime-gpu` only publishes `manylinux_2_27_x86_64` and `manylinux_2_28_x86_64` wheels — `uv sync` on arm64 failed with exit code 2. Gated `onnxruntime-gpu` behind `sys_platform == 'linux' and platform_machine == 'x86_64'`; arm64 and non-Linux installs now get the CPU `onnxruntime` package instead.
|
||||
- **uv lockfile now covers arm64**: Added `required-environments` to `[tool.uv]` so the lockfile is solved for both `linux/x86_64` and `linux/aarch64`, preventing silent resolution gaps for the non-build platform.
|
||||
|
||||
## [0.2.8] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Dockerfile: use `add-apt-repository` with isolated GNUPGHOME**: the `curl | gpg --dearmor` approach was silently failing — gpg exits 0 on bad/empty input, leaving an invalid keyring and causing apt to skip the deadsnakes PPA entirely. Reverted to `add-apt-repository ppa:deadsnakes/ppa` with `GNUPGHOME=$(mktemp -d)` to prevent gpg from touching any pre-existing agent socket (the original QEMU crash cause).
|
||||
- **arm64 base: removed `--platform=$BUILDPLATFORM`**: the Ubuntu 24.04 base for arm64 is now pulled for the target platform, so the arm64 image contains real arm64 binaries rather than amd64 binaries in an arm64 manifest.
|
||||
|
||||
### Changed
|
||||
|
||||
- **License changed from MIT to AGPLv3+**: source and network use of winnow now require derivative works to be open-sourced under the same terms.
|
||||
|
||||
## [0.2.7] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **arm64 base reverted to Ubuntu 24.04**: Ubuntu 26.04 ships Python 3.14, not 3.13 — `python3.13` was not locatable in its repos. Reverted arm64 base to `ubuntu:24.04`.
|
||||
- **Deadsnakes PPA now uses curl+gpg instead of `add-apt-repository`**: `add-apt-repository` spawns a gpg-agent which crashes under QEMU (arm64 CI). The PPA is now added by fetching the key via `curl` and piping through `gpg --dearmor` — no agent, works on both architectures.
|
||||
- **PPA conditional removed**: both amd64 (Ubuntu 22.04 CUDA) and arm64 (Ubuntu 24.04) now go through the same deadsnakes install path, eliminating the per-arch branching and the `ARG TARGETARCH` dependency in RUN commands.
|
||||
|
||||
## [0.2.6] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Dockerfile: `ARG TARGETARCH` re-declared in each stage**: Docker's automatic platform ARGs are only in scope for `FROM` instructions, not `RUN` commands. The `$TARGETARCH` variable in the deadsnakes PPA conditional was silently empty, so the PPA was never added and `python3.13` could not be located on the Ubuntu 22.04 CUDA base. Adding `ARG TARGETARCH` at the top of both the `build` and `runtime` stage bodies fixes the amd64 build.
|
||||
|
||||
## [0.2.5] - 2026-06-12
|
||||
|
||||
### Changed
|
||||
|
||||
- **CUDA base upgraded to 13.3.0**: amd64 base image bumped from `nvidia/cuda:12.9.2-cudnn-runtime-ubuntu22.04` to `nvidia/cuda:13.3.0-cudnn-runtime-ubuntu22.04`. Python packages (torch, onnxruntime-gpu) still target CUDA 12.6 via pip-installed libraries; the base image upgrade requires a host GPU driver that supports CUDA 13+.
|
||||
- **`torchvision>=0.27.0` added as explicit dependency**: routed through the `pytorch-cu126` index for linux/x86_64, ensuring it resolves to `0.27.0+cu126` (paired with `torch 2.12.0+cu126`) rather than being pulled from PyPI where the older `0.21.0` wheel would downgrade torch to 2.6.0.
|
||||
- **`torch>=2.12.0`** floor raised from 2.6.0 to prevent silent downgrade.
|
||||
- **`nvidia-cudnn-cu12>=9.0.0`** floor kept intentionally loose — torch pins an exact cuDNN ABI version (`9.10.2.21`) and must own that constraint.
|
||||
|
||||
## [0.2.4] - 2026-06-12
|
||||
|
||||
### Changed
|
||||
|
||||
- **Python 3.13 across all platforms**: bumped from 3.12 to 3.13. amd64 installs Python 3.13 from the deadsnakes PPA on the Ubuntu 22.04 CUDA base; arm64 uses the Python 3.13 package available natively in Ubuntu 26.04. All confirmed dependencies (`insightface 1.0.1`, `onnxruntime-gpu 1.26.0`, `torch 2.12.0+cu126`) have Python 3.13 wheels.
|
||||
- **arm64 base: Ubuntu 26.04**: Python 3.12 was removed from Ubuntu 26.04's default repos; upgrading the base pulls Python 3.13 without a PPA.
|
||||
- **`requires-python = ">=3.13"`** and `target-version = "py313"` updated in `pyproject.toml`.
|
||||
- **CI updated to Python 3.13**: `test.yml` and `update-lockfile.yml` now install Python 3.13.
|
||||
- **`uv.lock` regenerated** for Python 3.13.5.
|
||||
|
||||
## [0.2.3] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Clear-text API key storage**: `config.py` no longer writes `API_KEY` to `.immich_config.json`. The key must come from an environment variable or `.env` file. Interactive mode now prints a tip directing users to `.env`. Resolves CodeQL `py/clear-text-storage-sensitive-data`.
|
||||
|
||||
### Changed
|
||||
|
||||
- **CI workflow permissions**: `test.yml` and `lint.yml` now declare `permissions: contents: read`, following least-privilege principle and resolving `actions/missing-workflow-permissions` scanner alerts.
|
||||
- **CI lockfile race condition**: removed the `verify-lockfile` pre-job from `docker-publish.yml` and `release.yml`. The `update-lockfile.yml` bot maintains the lockfile; the verify step raced against it on the same push event and caused false failures. `release.yml` now runs `uv lock` inline so tag-triggered builds are always self-consistent.
|
||||
- **Docs moved to wiki**: `docs/` folder removed from the repository. Setup, Troubleshooting, and FAQ pages are now at the [GitHub wiki](https://github.com/sudolulo/winnow/wiki).
|
||||
|
||||
## [0.2.2] - 2026-06-12
|
||||
|
||||
### Added
|
||||
|
||||
+48
-31
@@ -1,68 +1,82 @@
|
||||
# ── Platform-conditional base ─────────────────────────────────────────────
|
||||
# amd64: NVIDIA CUDA 12.6 (GPU acceleration when available, CPU fallback)
|
||||
# arm64: Plain Ubuntu (CPU-only, no CUDA on ARM)
|
||||
# ── Base images ───────────────────────────────────────────────────────────────
|
||||
# amd64 + gpu: NVIDIA CUDA 13.3 + cuDNN (GPU acceleration when available)
|
||||
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
||||
# arm64: Ubuntu 24.04 (CPU-only; no CUDA wheels on ARM)
|
||||
|
||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu22.04 AS base-amd64
|
||||
FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS base-arm64
|
||||
ARG VARIANT=gpu
|
||||
|
||||
# ── Build stage ───────────────────────────────────────────────────────────
|
||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:13.3.0-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
||||
FROM ubuntu:22.04 AS base-amd64-cpu
|
||||
FROM ubuntu:24.04 AS base-arm64-gpu
|
||||
FROM ubuntu:24.04 AS base-arm64-cpu
|
||||
|
||||
# ── Build stage ───────────────────────────────────────────────────────────────
|
||||
ARG TARGETARCH
|
||||
|
||||
FROM base-${TARGETARCH} AS build
|
||||
FROM base-${TARGETARCH}-${VARIANT} AS build
|
||||
|
||||
ARG VARIANT=gpu
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Both Ubuntu 22.04 and 24.04 get Python 3.13 from the deadsnakes PPA.
|
||||
# GNUPGHOME is isolated so gpg never contacts an agent socket under QEMU.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
software-properties-common \
|
||||
&& add-apt-repository ppa:deadsnakes/ppa -y \
|
||||
&& apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3.12 python3.12-venv python3.12-dev \
|
||||
ca-certificates curl gnupg software-properties-common \
|
||||
&& GNUPGHOME=$(mktemp -d) add-apt-repository ppa:deadsnakes/ppa -y \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3.13 python3.13-venv python3.13-dev \
|
||||
libgl1 libglib2.0-0 libxext6 g++ \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /usr/bin/python3.12 /usr/bin/python3
|
||||
&& ln -sf /usr/bin/python3.13 /usr/bin/python3
|
||||
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
|
||||
&& cp /root/.local/bin/uv /usr/local/bin/uv
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync --frozen --no-dev \
|
||||
# For cpu variant, swap in the CPU-only pyproject and lockfile before syncing.
|
||||
COPY pyproject.toml uv.lock pyproject-cpu.toml uv-cpu.lock ./
|
||||
RUN if [ "$VARIANT" = "cpu" ]; then \
|
||||
cp pyproject-cpu.toml pyproject.toml && \
|
||||
cp uv-cpu.lock uv.lock; \
|
||||
fi && \
|
||||
uv sync --frozen --no-dev \
|
||||
&& uv cache clean
|
||||
|
||||
COPY winnow/ winnow/
|
||||
COPY entrypoint.sh scheduler.py ./
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
# ── Runtime stage ─────────────────────────────────────────────────────────
|
||||
# ── Runtime stage ─────────────────────────────────────────────────────────────
|
||||
# Starts fresh from the base image — excludes build tools (g++,
|
||||
# python3.12-dev, curl, gnupg) that are not needed at runtime.
|
||||
# python3.13-dev, gnupg, software-properties-common) not needed at runtime.
|
||||
|
||||
FROM base-${TARGETARCH} AS runtime
|
||||
FROM base-${TARGETARCH}-${VARIANT} AS runtime
|
||||
|
||||
ARG VARIANT=gpu
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
software-properties-common \
|
||||
tini \
|
||||
&& add-apt-repository ppa:deadsnakes/ppa -y \
|
||||
&& apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3.12 python3.12-venv \
|
||||
ca-certificates curl gnupg software-properties-common tini \
|
||||
&& GNUPGHOME=$(mktemp -d) add-apt-repository ppa:deadsnakes/ppa -y \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3.13 python3.13-venv \
|
||||
libgl1 libglib2.0-0 libxext6 \
|
||||
&& apt-get purge -y --auto-remove curl gnupg software-properties-common \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /usr/bin/python3.12 /usr/bin/python3
|
||||
&& ln -sf /usr/bin/python3.13 /usr/bin/python3
|
||||
|
||||
# Copy app (with .venv) and uv from build stage
|
||||
COPY --from=build /app /app
|
||||
COPY --from=build /usr/local/bin/uv /usr/local/bin/uv
|
||||
|
||||
# Expose CUDA/cuDNN libraries from pip packages so onnxruntime-gpu
|
||||
# can find libcublasLt.so.12 and libcudnn.so.9 at runtime
|
||||
ENV LD_LIBRARY_PATH="/app/.venv/lib/python3.12/site-packages/nvidia/cudnn/lib:/app/.venv/lib/python3.12/site-packages/nvidia/cuda_runtime/lib:${LD_LIBRARY_PATH}"
|
||||
# Register every nvidia pip-package lib/ directory with ldconfig so that
|
||||
# onnxruntime-gpu and torch can find libcudnn, libcublas, libcufft, etc.
|
||||
# without a hand-maintained LD_LIBRARY_PATH. Skipped silently on cpu builds.
|
||||
RUN find /app/.venv/lib/python3.*/site-packages/nvidia -type d -name "lib" \
|
||||
2>/dev/null > /etc/ld.so.conf.d/nvidia-pip.conf && ldconfig || true
|
||||
|
||||
RUN groupadd -g 568 apps && useradd -u 568 -g apps -m -s /bin/bash appuser \
|
||||
&& mkdir -p /models/.insightface /models/huggingface \
|
||||
@@ -70,7 +84,10 @@ RUN groupadd -g 568 apps && useradd -u 568 -g apps -m -s /bin/bash appuser \
|
||||
|
||||
WORKDIR /app
|
||||
USER appuser
|
||||
ENV HF_HOME=/models/huggingface INSIGHTFACE_HOME=/models
|
||||
# PYTHONPATH=/app makes the winnow package importable from the entry point script.
|
||||
# uv sync builds the wheel before winnow/ is COPY'd, so site-packages has only
|
||||
# the dist-info. Explicitly adding /app lets Python find winnow/__init__.py there.
|
||||
ENV HF_HOME=/models/huggingface INSIGHTFACE_HOME=/models/.insightface PYTHONPATH=/app
|
||||
|
||||
HEALTHCHECK CMD test -f /app/entrypoint.sh || exit 1
|
||||
ENTRYPOINT ["tini", "--", "/app/entrypoint.sh"]
|
||||
|
||||
@@ -1,22 +1,661 @@
|
||||
MIT License
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (c) 2026 Holden Salomon
|
||||
Copyright (c) 2026 Sebastian
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
Preamble
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
[](https://github.com/sudolulo/winnow/actions/workflows/docker-publish.yml) [](https://github.com/sudolulo/winnow/actions/workflows/release.yml) [](https://github.com/sudolulo/winnow/actions/workflows/lint.yml) [](https://github.com/sudolulo/winnow/actions/workflows/test.yml)
|
||||
[](https://immich.app) [](https://frigate.video)
|
||||
|
||||
**Docs:** [Setup Guide](docs/setup.md) · [Troubleshooting](docs/troubleshooting.md) · [FAQ](docs/faq.md)
|
||||
|
||||
# winnow
|
||||
|
||||
[](https://github.com/sudolulo/winnow/actions/workflows/docker-publish.yml) [](https://github.com/sudolulo/winnow/actions/workflows/test.yml) [](https://immich.app) [](https://frigate.video)
|
||||
|
||||
**Docs:** [Setup](https://github.com/sudolulo/winnow/wiki/Setup) · [Troubleshooting](https://github.com/sudolulo/winnow/wiki/Troubleshooting) · [FAQ](https://github.com/sudolulo/winnow/wiki/FAQ)
|
||||
|
||||
`winnow` pulls photos of people and objects from your [Immich](https://immich.app) library, selects the most diverse and highest-quality subset using AI embeddings, and delivers them as training data for [Frigate](https://frigate.video)'s face recognition and object classification models.
|
||||
|
||||
It runs fully headless in Docker, is configured entirely through environment variables, and can run on a schedule — no interactive prompts, no manual steps.
|
||||
@@ -112,17 +111,23 @@ If the embedding model is unavailable, the tool falls back to **time spread**: e
|
||||
|
||||
## Running in Docker
|
||||
|
||||
### Image Tags
|
||||
|
||||
| Tag | Arch | GPU | Notes |
|
||||
| :-- | :-- | :-- | :-- |
|
||||
| `:latest` | amd64 + arm64 | CUDA 13.3 (amd64) | Requires NVIDIA Container Toolkit on amd64 |
|
||||
| `:cpu` | amd64 | None | ~2 GB smaller; use if you have no NVIDIA GPU |
|
||||
|
||||
### Quick Start
|
||||
|
||||
```yaml
|
||||
services:
|
||||
winnow:
|
||||
image: ghcr.io/sudolulo/winnow:latest
|
||||
image: ghcr.io/sudolulo/winnow:latest # or :cpu for CPU-only amd64
|
||||
environment:
|
||||
- IMMICH_URL=http://192.168.1.10:2283
|
||||
- API_KEY=your-immich-api-key
|
||||
- FRIGATE_URL=http://192.168.1.10:5000
|
||||
- AUTO_MODE=true
|
||||
- CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
||||
volumes:
|
||||
- /path/to/models:/models
|
||||
@@ -137,11 +142,21 @@ services:
|
||||
capabilities: [gpu]
|
||||
```
|
||||
|
||||
> **CPU users (`:cpu` tag):** remove the `deploy.resources` block — no NVIDIA runtime needed. Add `mem_limit: 2g` to the service to prevent an OOM restart loop on large libraries.
|
||||
|
||||
See [compose.yml](compose.yml) for the full annotated example.
|
||||
|
||||
### Scheduling Behaviour
|
||||
|
||||
On startup the container always runs once immediately. If `CRON_SCHEDULE` is set, it then starts a scheduler that fires on the defined interval, keeping the process (and loaded models) alive between runs. Without `CRON_SCHEDULE` the container exits after the first run.
|
||||
`CRON_SCHEDULE` controls container lifetime:
|
||||
|
||||
| `CRON_SCHEDULE` value | Behaviour |
|
||||
| :-- | :-- |
|
||||
| *(unset)* | Run once on startup, then exit |
|
||||
| *(empty string)* | Stay alive, run nothing — trigger manually with `docker exec -it winnow winnow` |
|
||||
| Cron expression | Run on startup, then repeat on schedule |
|
||||
|
||||
In scheduled mode the process (and loaded models) stays resident between runs. In manual mode the container idles indefinitely with `sleep infinity` — useful when you want to trigger runs interactively on demand without pulling a new container each time.
|
||||
|
||||
The first run after a fresh install downloads the embedding models (~1-2 GB). Subsequent runs use the cached models from the mounted volume and start immediately.
|
||||
|
||||
@@ -153,7 +168,8 @@ The first run after a fresh install downloads the embedding models (~1-2 GB). Su
|
||||
|
||||
| Variable | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `AUTO_MODE` | `false` | Run without interactive prompts — required for Docker/cron use |
|
||||
| `AUTO_MODE` | *(auto)* | Force non-interactive mode even in a terminal; auto-detected otherwise (no TTY = auto) |
|
||||
| `VERBOSE` | `false` | Set to `true` to enable DEBUG-level console output (the log file is always DEBUG) |
|
||||
| `TRAINING_MODE` | `face` | `face` — upload crops to Frigate API; `object` — save crops to disk |
|
||||
| `STRATEGY` | `auto` | `auto` (adaptive), `standard` (30 images), `broad` (100 images) |
|
||||
| `LIMIT` | *(unset)* | Exact image count — overrides `STRATEGY` |
|
||||
@@ -193,7 +209,7 @@ The first run after a fresh install downloads the embedding models (~1-2 GB). Su
|
||||
| Variable | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `FORCE_CPU` | `false` | Disable GPU — fall back to CPU for embedding computation |
|
||||
| `ENABLE_CACHE` | `false` | Cache computed embeddings to disk (speeds up re-runs on the same library) |
|
||||
| `ENABLE_CACHE` | `true` | Cache computed embeddings to disk (speeds up re-runs on the same library) |
|
||||
| `CACHE_DIR` | `.if_cache` | Path for embedding cache and upload tracker files |
|
||||
| `HF_HOME` | *(system)* | HuggingFace model cache location (SigLIP) |
|
||||
| `INSIGHTFACE_HOME` | *(system)* | InsightFace model cache location (Buffalo_L) |
|
||||
@@ -210,7 +226,7 @@ The first run after a fresh install downloads the embedding models (~1-2 GB). Su
|
||||
|
||||
| Variable | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `CRON_SCHEDULE` | *(unset)* | Cron expression for recurring runs — unset exits after first run |
|
||||
| `CRON_SCHEDULE` | *(unset)* | Unset = run once and exit; empty = stay alive for manual `docker exec`; cron expression = scheduled |
|
||||
|
||||
---
|
||||
|
||||
@@ -223,7 +239,7 @@ uv sync
|
||||
uv run winnow
|
||||
```
|
||||
|
||||
Requires Python 3.12+ and [uv](https://astral.sh/uv/). An NVIDIA GPU is strongly recommended — CPU mode works but embedding computation is significantly slower.
|
||||
Requires Python 3.13+ and [uv](https://astral.sh/uv/). An NVIDIA GPU is strongly recommended — CPU mode works but embedding computation is slower (typically tens of seconds per person vs under a second on GPU).
|
||||
|
||||
---
|
||||
|
||||
@@ -232,7 +248,7 @@ Requires Python 3.12+ and [uv](https://astral.sh/uv/). An NVIDIA GPU is strongly
|
||||
- **Immich** v1.106+
|
||||
- **Frigate** v0.16+ (face mode only — object mode has no Frigate API dependency)
|
||||
- **NVIDIA GPU** recommended (CUDA 12.x)
|
||||
- **Python 3.12+**
|
||||
- **Python 3.13+**
|
||||
|
||||
---
|
||||
|
||||
|
||||
+14
-10
@@ -9,7 +9,10 @@ services:
|
||||
- FRIGATE_URL=${FRIGATE_URL}
|
||||
|
||||
# ── Mode & Strategy ───────────────────────────────────────────────────
|
||||
- AUTO_MODE=true
|
||||
# Auto mode is active by default when no TTY is present (Docker/cron).
|
||||
# Set AUTO_MODE=true to force auto mode in an interactive terminal.
|
||||
# To run interactively: docker exec -it winnow winnow
|
||||
# - VERBOSE=true # Enable DEBUG-level console output
|
||||
# TRAINING_MODE: face = upload to Frigate face recognition API
|
||||
# object = save crops to output dir for manual Frigate placement
|
||||
- TRAINING_MODE=face
|
||||
@@ -34,8 +37,8 @@ services:
|
||||
# - MAX_AUTO_IMAGES=80 # Hard cap on auto-diversity selection (default: 80)
|
||||
|
||||
# ── Caching & Models ──────────────────────────────────────────────────
|
||||
- FORCE_CPU=false
|
||||
- ENABLE_CACHE=true
|
||||
# - FORCE_CPU=true # Disable GPU, fall back to CPU
|
||||
# - ENABLE_CACHE=false # Disable embedding cache (default: true)
|
||||
- CACHE_DIR=/app/.if_cache
|
||||
- HF_HOME=/models/huggingface
|
||||
- INSIGHTFACE_HOME=/models/.insightface
|
||||
@@ -46,18 +49,19 @@ services:
|
||||
# - RESET_PERSON=John # Clear uploaded+rejected history for one person
|
||||
|
||||
# ── Scheduling ────────────────────────────────────────────────────────
|
||||
# Cron expression (unset = run once and exit)
|
||||
# Every Sunday at 3 AM:
|
||||
- CRON_SCHEDULE=0 3 * * 0
|
||||
# - CRON_SCHEDULE=0 3 1 * *
|
||||
# - CRON_SCHEDULE=*/30 * * * *
|
||||
# CRON_SCHEDULE controls container lifetime:
|
||||
# unset — run once on startup, then exit
|
||||
# empty string — stay alive, run nothing; trigger manually with:
|
||||
# docker exec -it winnow winnow
|
||||
# cron expression — run on startup, then on schedule
|
||||
# - CRON_SCHEDULE= # Manual mode (keep alive, no auto-run)
|
||||
# - CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
||||
# - CRON_SCHEDULE=0 3 1 * * # First of every month
|
||||
volumes:
|
||||
# Replace with absolute paths on your host, e.g. /opt/winnow/models
|
||||
- /path/to/winnow/models:/models
|
||||
- /path/to/winnow/cache:/app/.if_cache
|
||||
- /path/to/winnow/output:/app/frigate_train
|
||||
stdin_open: true
|
||||
tty: true
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
# FAQ
|
||||
|
||||
## Does winnow modify my Immich library?
|
||||
|
||||
No. winnow only reads from Immich (assets, people, face bounding boxes). It never writes back to Immich or deletes anything.
|
||||
|
||||
---
|
||||
|
||||
## How many images should I upload to Frigate?
|
||||
|
||||
The `auto` strategy decides this for you — it keeps selecting until adding more images would be redundant. In practice this is usually 20–60 per person. You can cap it with `MAX_AUTO_IMAGES` (default 80).
|
||||
|
||||
Quality and diversity matter far more than volume. 30 well-spread images outperform 200 from the same week.
|
||||
|
||||
---
|
||||
|
||||
## What's the difference between face mode and object mode?
|
||||
|
||||
- **Face mode**: Extracts and aligns face crops, uploads them directly to Frigate's face training API. This is for teaching Frigate to recognize specific people.
|
||||
- **Object mode**: Runs YOLO detection on full images and saves crops of a target class (dog, cat, car, etc.) to disk. Frigate has no API for object training data, so you place them manually.
|
||||
|
||||
---
|
||||
|
||||
## Can I run it without Frigate?
|
||||
|
||||
Yes — in object mode, `FRIGATE_URL` is not used and crops are saved to the output volume. In face mode you need Frigate to receive the uploads, but you can use `DRY_RUN=true` to preview selection without uploading.
|
||||
|
||||
---
|
||||
|
||||
## How does auto-diversity mode work?
|
||||
|
||||
winnow computes a vector embedding for each candidate image (what the face/object actually looks like — angle, lighting, expression). It then clusters those embeddings and picks representatives that are maximally spread across the embedding space. It stops when the next-most-different image is already close to something already selected. See the README for the full pipeline.
|
||||
|
||||
---
|
||||
|
||||
## Does it support multiple people in one run?
|
||||
|
||||
Yes. By default it processes every named person in your Immich library. Use `ONLY_PEOPLE` to whitelist specific names or `SKIP_PEOPLE` to exclude them.
|
||||
|
||||
---
|
||||
|
||||
## What GPU is needed?
|
||||
|
||||
Any NVIDIA GPU with CUDA 12.x support. The models (InsightFace Buffalo_L + SigLIP) fit comfortably in 4 GB VRAM. CPU mode works but is significantly slower.
|
||||
|
||||
ARM builds (linux/arm64) use CPU-only — CUDA is not available on ARM.
|
||||
|
||||
---
|
||||
|
||||
## Does it work on Unraid / Proxmox / bare Docker?
|
||||
|
||||
Yes — the `compose.yml` uses standard Docker volume mounts. Replace the example paths with whatever absolute paths suit your setup.
|
||||
|
||||
---
|
||||
|
||||
## How do I update winnow?
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
The `latest` tag on GHCR tracks the `main` branch. Pinning to a version tag (e.g. `ghcr.io/sudolulo/winnow:v0.2.0`) is recommended for stability.
|
||||
@@ -1,95 +0,0 @@
|
||||
# Setup Guide
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Immich](https://immich.app) v1.106+ with face recognition enabled and people tagged
|
||||
- [Frigate](https://frigate.video) v0.16+ (face mode only)
|
||||
- Docker with the NVIDIA container toolkit (optional but strongly recommended)
|
||||
|
||||
---
|
||||
|
||||
## 1. Get your Immich API key
|
||||
|
||||
1. Open Immich → **Account Settings** → **API Keys**
|
||||
2. Click **New API Key**, give it a name (e.g. `winnow`), copy the key
|
||||
|
||||
---
|
||||
|
||||
## 2. Get your Frigate URL
|
||||
|
||||
This is the base URL of your Frigate instance, e.g. `http://192.168.1.10:5000`. Only needed for face mode — omit it entirely if you're using object mode.
|
||||
|
||||
---
|
||||
|
||||
## 3. Deploy with Docker Compose
|
||||
|
||||
Copy [`compose.yml`](../compose.yml) and [`.env.example`](../.env.example) to a directory on your host:
|
||||
|
||||
```bash
|
||||
mkdir winnow && cd winnow
|
||||
curl -O https://raw.githubusercontent.com/sudolulo/winnow/main/compose.yml
|
||||
curl -O https://raw.githubusercontent.com/sudolulo/winnow/main/.env.example
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` with your values:
|
||||
|
||||
```bash
|
||||
IMMICH_URL=http://192.168.1.10:2283
|
||||
API_KEY=your-immich-api-key
|
||||
FRIGATE_URL=http://192.168.1.10:5000
|
||||
```
|
||||
|
||||
Edit the volume paths in `compose.yml` to point to directories on your host where models, cache, and output crops should be stored:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /your/path/to/models:/models
|
||||
- /your/path/to/cache:/app/.if_cache
|
||||
- /your/path/to/output:/app/frigate_train
|
||||
```
|
||||
|
||||
These directories will be created automatically by Docker if they don't exist.
|
||||
|
||||
Start it:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Logs:
|
||||
|
||||
```bash
|
||||
docker compose logs -f winnow
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. First run
|
||||
|
||||
On the first run, winnow downloads the embedding models (~1–2 GB) from HuggingFace and InsightFace. This happens once — subsequent runs use the cached models from your mounted volume and start immediately.
|
||||
|
||||
---
|
||||
|
||||
## 5. Scheduling
|
||||
|
||||
Set `CRON_SCHEDULE` in your `.env` to keep winnow running on a schedule:
|
||||
|
||||
```
|
||||
CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
||||
```
|
||||
|
||||
Without `CRON_SCHEDULE`, the container runs once and exits.
|
||||
|
||||
---
|
||||
|
||||
## GPU passthrough
|
||||
|
||||
To enable GPU acceleration, include the `deploy` block in `compose.yml` (already present in the example) and ensure the NVIDIA container toolkit is installed on your host:
|
||||
|
||||
```bash
|
||||
# Verify GPU is accessible to Docker
|
||||
docker run --rm --gpus all nvidia/cuda:12.9.2-base-ubuntu22.04 nvidia-smi
|
||||
```
|
||||
|
||||
CPU mode works without any GPU setup — set `FORCE_CPU=true` to disable GPU explicitly.
|
||||
@@ -1,77 +0,0 @@
|
||||
# Troubleshooting
|
||||
|
||||
## Container exits immediately
|
||||
|
||||
Check logs:
|
||||
```bash
|
||||
docker compose logs winnow
|
||||
```
|
||||
|
||||
Common causes:
|
||||
- **Missing required env var** — `IMMICH_URL` or `API_KEY` not set
|
||||
- **Cannot reach Immich** — check the URL and that Immich is running; use `http://` not `https://` unless you have TLS set up
|
||||
|
||||
---
|
||||
|
||||
## "No people found" / nothing processed
|
||||
|
||||
- Make sure Immich has completed face recognition and you have named people in your library
|
||||
- `YEARS_FILTER` defaults to 10 years — increase it if your tagged photos are older
|
||||
- `MIN_FACE_COUNT` skips people with few photos — lower or remove it
|
||||
|
||||
---
|
||||
|
||||
## Frigate upload fails
|
||||
|
||||
- Confirm `FRIGATE_URL` is reachable from inside the container: `docker exec winnow curl $FRIGATE_URL/api/stats`
|
||||
- Check Frigate v0.16+ — older versions don't have the face training API
|
||||
- Set `DRY_RUN=true` to verify selection without uploading
|
||||
|
||||
---
|
||||
|
||||
## Models fail to download
|
||||
|
||||
winnow downloads InsightFace and HuggingFace (SigLIP) models on first run.
|
||||
|
||||
- Ensure the container has internet access
|
||||
- Confirm the model volume is mounted and writable
|
||||
- If behind a proxy, set `HTTP_PROXY` / `HTTPS_PROXY` env vars
|
||||
|
||||
---
|
||||
|
||||
## Running on CPU (no GPU)
|
||||
|
||||
Set `FORCE_CPU=true`. Everything works but embedding computation is slower — expect several minutes per person instead of seconds.
|
||||
|
||||
If you have a GPU but it's not being used:
|
||||
- Confirm the NVIDIA container toolkit is installed: `docker run --rm --gpus all nvidia/cuda:12.9.2-base-ubuntu22.04 nvidia-smi`
|
||||
- Confirm the `deploy.resources.reservations.devices` block is present in `compose.yml`
|
||||
|
||||
---
|
||||
|
||||
## Same images uploaded every run
|
||||
|
||||
The upload tracker is stored in `CACHE_DIR` (`/app/.if_cache` by default). If this volume isn't persisted between runs, the tracker resets and images are re-uploaded.
|
||||
|
||||
Make sure `/app/.if_cache` is mounted to a persistent host path.
|
||||
|
||||
---
|
||||
|
||||
## Re-uploading a specific person
|
||||
|
||||
To clear the upload history for one person and start fresh:
|
||||
|
||||
```env
|
||||
RESET_PERSON=John
|
||||
```
|
||||
|
||||
Remove this after one run — it clears the history and then processes normally.
|
||||
|
||||
---
|
||||
|
||||
## Image quality issues
|
||||
|
||||
- **Too blurry**: Lower `BLUR_THRESHOLD` (default 100) — e.g. `50` accepts more blur
|
||||
- **Face too small**: Lower `MIN_FACE_WIDTH` (default 50px)
|
||||
- **Low confidence detections included**: Raise `MIN_CONFIDENCE` (default 0.7)
|
||||
- **Rejected images being re-tried**: Set `RETRY_REJECTED=true` for one run
|
||||
+11
-4
@@ -2,11 +2,19 @@
|
||||
set -e
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
# 1. Run the job immediately on startup
|
||||
# CRON_SCHEDULE controls container lifetime:
|
||||
# unset — run once and exit
|
||||
# empty string — stay alive, run nothing (use: docker exec -it winnow winnow)
|
||||
# cron expression — run immediately, then on schedule
|
||||
|
||||
if [ "${CRON_SCHEDULE+isset}" = "isset" ] && [ -z "$CRON_SCHEDULE" ]; then
|
||||
echo "▶ CRON_SCHEDULE is empty — manual mode. Use 'docker exec -it winnow winnow' to run."
|
||||
exec sleep infinity
|
||||
fi
|
||||
|
||||
echo "▶ Running on startup..."
|
||||
/app/.venv/bin/python -m winnow.cli
|
||||
/app/.venv/bin/winnow
|
||||
|
||||
# 2. If a schedule exists, start the scheduler
|
||||
if [ -n "${CRON_SCHEDULE:-}" ]; then
|
||||
echo "▶ CRON_SCHEDULE set to: $CRON_SCHEDULE"
|
||||
echo "▶ Switching to scheduled mode..."
|
||||
@@ -14,4 +22,3 @@ if [ -n "${CRON_SCHEDULE:-}" ]; then
|
||||
else
|
||||
echo "▶ No schedule set, exiting."
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
[project]
|
||||
name = "winnow"
|
||||
version = "0.2.11"
|
||||
description = "Immich to Frigate training sets"
|
||||
license = "AGPL-3.0-or-later"
|
||||
requires-python = ">=3.13"
|
||||
authors = [{ name = "Holden Salomon", email = "holden@arch.fyi" }]
|
||||
keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"]
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Scientific/Engineering :: Image Recognition",
|
||||
]
|
||||
dependencies = [
|
||||
"croniter>=5.0.2",
|
||||
"insightface>=0.7.3",
|
||||
"numpy>=2.2.6",
|
||||
"onnxruntime>=1.23.2",
|
||||
"opencv-python-headless>=4.12.0.88",
|
||||
"pillow>=12.1.0",
|
||||
"python-dotenv>=1.2.1",
|
||||
"requests>=2.32.5",
|
||||
"rich>=14.2.0",
|
||||
"torch>=2.12.0",
|
||||
"torchvision>=0.27.0",
|
||||
"transformers>=4.57.6",
|
||||
"ultralytics>=8.4.66",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
winnow = "winnow.cli:main"
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/sudolulo/winnow"
|
||||
|
||||
[tool.uv]
|
||||
required-environments = [
|
||||
"sys_platform == 'linux' and platform_machine == 'x86_64'",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
|
||||
]
|
||||
torchvision = [
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"ruff>=0.15.17",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["winnow"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = "py313"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I"]
|
||||
|
||||
[tool.deptry]
|
||||
pep621_dev_dependency_groups = ["dev"]
|
||||
|
||||
[tool.deptry.package_module_name_map]
|
||||
pillow = "PIL"
|
||||
opencv-python-headless = "cv2"
|
||||
python-dotenv = "dotenv"
|
||||
insightface = "insightface"
|
||||
numpy = "numpy"
|
||||
onnxruntime = "onnxruntime"
|
||||
requests = "requests"
|
||||
rich = "rich"
|
||||
torch = "torch"
|
||||
transformers = "transformers"
|
||||
ultralytics = "ultralytics"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
+28
-11
@@ -1,16 +1,16 @@
|
||||
[project]
|
||||
name = "winnow"
|
||||
version = "0.2.2"
|
||||
version = "0.2.11"
|
||||
description = "Immich to Frigate training sets"
|
||||
license = "MIT"
|
||||
requires-python = ">=3.12"
|
||||
license = "AGPL-3.0-or-later"
|
||||
requires-python = ">=3.13"
|
||||
authors = [{ name = "Holden Salomon", email = "holden@arch.fyi" }]
|
||||
keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"]
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Scientific/Engineering :: Image Recognition",
|
||||
]
|
||||
dependencies = [
|
||||
@@ -18,15 +18,18 @@ dependencies = [
|
||||
"insightface>=0.7.3",
|
||||
"nvidia-cudnn-cu12>=9.0.0",
|
||||
"numpy>=2.2.6",
|
||||
"onnxruntime-gpu>=1.23.2",
|
||||
"onnxruntime-gpu>=1.23.2; sys_platform == 'linux' and platform_machine == 'x86_64'",
|
||||
"onnxruntime>=1.23.2; sys_platform == 'linux' and platform_machine != 'x86_64'",
|
||||
"onnxruntime>=1.23.2; sys_platform != 'linux'",
|
||||
"opencv-python-headless>=4.12.0.88",
|
||||
"pillow>=12.1.0",
|
||||
"python-dotenv>=1.2.1",
|
||||
"requests>=2.32.5",
|
||||
"rich>=14.2.0",
|
||||
"torch>=2.6.0",
|
||||
"torch>=2.12.0",
|
||||
"torchvision>=0.27.0",
|
||||
"transformers>=4.57.6",
|
||||
"ultralytics>=8.3.252",
|
||||
"ultralytics>=8.4.66",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
@@ -36,7 +39,16 @@ winnow = "winnow.cli:main"
|
||||
Repository = "https://github.com/sudolulo/winnow"
|
||||
|
||||
[tool.uv]
|
||||
override-dependencies = ["onnxruntime-gpu>=1.23.2"]
|
||||
conflicts = [
|
||||
[
|
||||
{ package = "onnxruntime" },
|
||||
{ package = "onnxruntime-gpu" },
|
||||
],
|
||||
]
|
||||
required-environments = [
|
||||
"sys_platform == 'linux' and platform_machine == 'x86_64'",
|
||||
"sys_platform == 'linux' and platform_machine == 'aarch64'",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [
|
||||
@@ -44,6 +56,11 @@ torch = [
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" },
|
||||
{ index = "pytorch-cpu", marker = "sys_platform != 'linux'" },
|
||||
]
|
||||
torchvision = [
|
||||
{ index = "pytorch-cu126", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" },
|
||||
{ index = "pytorch-cpu", marker = "sys_platform != 'linux'" },
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu126"
|
||||
@@ -58,7 +75,7 @@ explicit = true
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"ruff>=0.9.2",
|
||||
"ruff>=0.15.17",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
@@ -66,7 +83,7 @@ packages = ["winnow"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = "py312"
|
||||
target-version = "py313"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I"]
|
||||
|
||||
+22
-28
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -9,33 +8,27 @@ from pathlib import Path
|
||||
try:
|
||||
from croniter import croniter
|
||||
except ImportError:
|
||||
print("❌ croniter not installed. Run: uv add croniter")
|
||||
print("croniter not installed. Run: uv add croniter")
|
||||
sys.exit(1)
|
||||
|
||||
# Imported at module level so models loaded during the first run stay
|
||||
# resident in memory across all subsequent scheduled runs.
|
||||
from winnow.cli import main
|
||||
|
||||
SCHEDULE = os.environ["CRON_SCHEDULE"]
|
||||
MODELS_DIR = os.environ.get("HF_HOME", "/models/huggingface")
|
||||
INSIGHTFACE_BASE = os.environ.get("INSIGHTFACE_HOME", "/models")
|
||||
|
||||
RUN_ENV = {**os.environ, "PYTHONUNBUFFERED": "1"}
|
||||
INSIGHTFACE_HOME = os.environ.get("INSIGHTFACE_HOME", "/models/.insightface")
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_models():
|
||||
"""Log model status before each run."""
|
||||
print("📦 Checking models...", flush=True)
|
||||
buffalo = Path(INSIGHTFACE_BASE) / ".insightface" / "models" / "buffalo_l"
|
||||
if buffalo.exists():
|
||||
print(" ✅ InsightFace Buffalo_L: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ InsightFace Buffalo_L: not found — will download", flush=True)
|
||||
|
||||
def check_models() -> None:
|
||||
buffalo = Path(INSIGHTFACE_HOME) / "models" / "buffalo_l"
|
||||
hf_hub = Path(MODELS_DIR) / "hub"
|
||||
if hf_hub.exists() and any(hf_hub.iterdir()):
|
||||
print(" ✅ HuggingFace models: present", flush=True)
|
||||
else:
|
||||
print(" ⬇️ HuggingFace models: not found — will download", flush=True)
|
||||
print("🚀 Starting winnow...", flush=True)
|
||||
if not buffalo.exists():
|
||||
print(" InsightFace Buffalo_L not found — will download on first run", flush=True)
|
||||
if not (hf_hub.exists() and any(hf_hub.iterdir())):
|
||||
print(" HuggingFace models not found — will download on first run", flush=True)
|
||||
|
||||
|
||||
NOW = time.time()
|
||||
@@ -45,14 +38,15 @@ next_run = cron.get_next(float)
|
||||
while True:
|
||||
now = time.time()
|
||||
if now >= next_run:
|
||||
print(f"\n▶ [{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow...", flush=True)
|
||||
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
|
||||
check_models()
|
||||
result = subprocess.run(["uv", "run", "winnow"], env=RUN_ENV)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"winnow exited with code {result.returncode}")
|
||||
print(f"❌ winnow failed with exit code {result.returncode}", flush=True)
|
||||
else:
|
||||
print("✅ winnow completed successfully", flush=True)
|
||||
try:
|
||||
main()
|
||||
print("winnow run complete", flush=True)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"winnow run failed: {e}", exc_info=True)
|
||||
print(f"winnow run failed: {e}", flush=True)
|
||||
next_run = cron.get_next(float)
|
||||
time.sleep(60)
|
||||
|
||||
time.sleep(max(1, next_run - time.time()))
|
||||
|
||||
@@ -25,7 +25,7 @@ def test_config_loads_defaults(monkeypatch):
|
||||
assert cfg.FACE_MARGIN == 0.15
|
||||
assert cfg.USE_FULL_RESOLUTION is True
|
||||
assert cfg.ENABLE_FACE_ALIGNMENT is True
|
||||
assert cfg.ENABLE_CACHE is False
|
||||
assert cfg.ENABLE_CACHE is True
|
||||
|
||||
_Config.reset()
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Tests for image quality filtering functions."""
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def _rgb_image(r, g, b, size=(100, 100)) -> Image.Image:
|
||||
arr = np.full((*size, 3), [r, g, b], dtype=np.uint8)
|
||||
return Image.fromarray(arr, "RGB")
|
||||
|
||||
|
||||
def _noisy_color_image(size=(100, 100)) -> Image.Image:
|
||||
"""Noisy image with a strong red channel so grayscale check passes."""
|
||||
rng = np.random.default_rng(0)
|
||||
arr = rng.integers(0, 256, (*size, 3), dtype=np.uint8)
|
||||
arr[:, :, 0] = np.clip(arr[:, :, 0].astype(int) + 80, 0, 255).astype(np.uint8)
|
||||
arr[:, :, 2] = np.clip(arr[:, :, 2].astype(int) - 80, 0, 255).astype(np.uint8)
|
||||
return Image.fromarray(arr, "RGB")
|
||||
|
||||
|
||||
# ── check_blur ────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_blur_rejects_flat_image():
|
||||
from winnow.quality import check_blur
|
||||
flat = np.full((100, 100, 3), 128, dtype=np.uint8)
|
||||
passed, reason = check_blur(flat, threshold=100.0)
|
||||
assert not passed
|
||||
assert "Blurry" in reason
|
||||
|
||||
|
||||
def test_blur_passes_noisy_color_image():
|
||||
from winnow.quality import check_blur
|
||||
img = _noisy_color_image()
|
||||
passed, _ = check_blur(np.asarray(img), threshold=100.0)
|
||||
assert passed
|
||||
|
||||
|
||||
# ── check_grayscale ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_grayscale_rejects_ir_image():
|
||||
from winnow.quality import check_grayscale
|
||||
gray = np.full((100, 100, 3), 128, dtype=np.uint8)
|
||||
passed, reason = check_grayscale(gray)
|
||||
assert not passed
|
||||
assert "Grayscale" in reason
|
||||
|
||||
|
||||
def test_grayscale_passes_color_image():
|
||||
from winnow.quality import check_grayscale
|
||||
color = np.zeros((100, 100, 3), dtype=np.uint8)
|
||||
color[:, :, 0] = 200 # strong red channel
|
||||
passed, _ = check_grayscale(color)
|
||||
assert passed
|
||||
|
||||
|
||||
def test_grayscale_rejects_single_channel():
|
||||
from winnow.quality import check_grayscale
|
||||
single = np.full((100, 100, 1), 128, dtype=np.uint8)
|
||||
passed, reason = check_grayscale(single)
|
||||
assert not passed
|
||||
|
||||
|
||||
# ── check_exposure ────────────────────────────────────────────────────────────
|
||||
|
||||
def test_exposure_rejects_black_image():
|
||||
from winnow.quality import check_exposure
|
||||
black = np.zeros((100, 100, 3), dtype=np.uint8)
|
||||
passed, reason = check_exposure(black)
|
||||
assert not passed
|
||||
assert "Underexposed" in reason
|
||||
|
||||
|
||||
def test_exposure_rejects_white_image():
|
||||
from winnow.quality import check_exposure
|
||||
white = np.full((100, 100, 3), 255, dtype=np.uint8)
|
||||
passed, reason = check_exposure(white)
|
||||
assert not passed
|
||||
assert "Overexposed" in reason
|
||||
|
||||
|
||||
def test_exposure_passes_normal_image():
|
||||
from winnow.quality import check_exposure
|
||||
mid = np.full((100, 100, 3), 128, dtype=np.uint8)
|
||||
passed, _ = check_exposure(mid)
|
||||
assert passed
|
||||
|
||||
|
||||
# ── check_face_size ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_face_size_rejects_small_face():
|
||||
from winnow.quality import check_face_size
|
||||
passed, reason = check_face_size(30, 30, min_px=50)
|
||||
assert not passed
|
||||
assert "small" in reason
|
||||
|
||||
|
||||
def test_face_size_passes_adequate_face():
|
||||
from winnow.quality import check_face_size
|
||||
passed, _ = check_face_size(100, 100, min_px=50)
|
||||
assert passed
|
||||
|
||||
|
||||
def test_face_size_rejects_if_either_dimension_small():
|
||||
from winnow.quality import check_face_size
|
||||
passed, _ = check_face_size(100, 30, min_px=50)
|
||||
assert not passed
|
||||
|
||||
|
||||
# ── check_confidence ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_confidence_rejects_low_score():
|
||||
from winnow.quality import check_confidence
|
||||
passed, reason = check_confidence(0.5, min_conf=0.7)
|
||||
assert not passed
|
||||
assert "confidence" in reason.lower()
|
||||
|
||||
|
||||
def test_confidence_passes_high_score():
|
||||
from winnow.quality import check_confidence
|
||||
passed, _ = check_confidence(0.95, min_conf=0.7)
|
||||
assert passed
|
||||
|
||||
|
||||
def test_confidence_passes_none_score():
|
||||
from winnow.quality import check_confidence
|
||||
passed, _ = check_confidence(None, min_conf=0.7)
|
||||
assert passed
|
||||
|
||||
|
||||
# ── assess_quality (integration) ─────────────────────────────────────────────
|
||||
|
||||
def test_assess_quality_passes_good_image():
|
||||
from winnow.quality import assess_quality
|
||||
img = _noisy_color_image()
|
||||
result = assess_quality(img, face_bbox=(10, 10, 110, 110), confidence=0.9)
|
||||
assert result.passed
|
||||
|
||||
|
||||
def test_assess_quality_collects_multiple_failures():
|
||||
from winnow.quality import assess_quality
|
||||
black = _rgb_image(0, 0, 0)
|
||||
result = assess_quality(black, face_bbox=(0, 0, 10, 10), confidence=0.3)
|
||||
assert not result.passed
|
||||
assert len(result.reasons) >= 2
|
||||
|
||||
|
||||
def test_assess_quality_skips_face_size_without_bbox():
|
||||
from winnow.quality import assess_quality
|
||||
img = _noisy_color_image()
|
||||
result = assess_quality(img, face_bbox=None, confidence=0.9)
|
||||
assert result.passed
|
||||
+1884
File diff suppressed because it is too large
Load Diff
+6
-1
@@ -5,4 +5,9 @@ Immich library for Frigate's Face Recognition (ArcFace) and Object/State
|
||||
Classification models.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
try:
|
||||
__version__ = version("winnow")
|
||||
except PackageNotFoundError:
|
||||
__version__ = "unknown"
|
||||
|
||||
+8
-5
@@ -2,6 +2,7 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from rich import print as rprint
|
||||
from rich.prompt import Confirm
|
||||
@@ -10,7 +11,7 @@ from .config import Config, ConfigManager
|
||||
from .executor import execute_jobs, upload_to_frigate
|
||||
from .immich_api import get_people
|
||||
from .jobs import _show_preview, auto_configure, interactive_configure
|
||||
from .logging import console, setup_logging
|
||||
from .log_config import console, setup_logging
|
||||
from .upload_tracker import get_person_summary, reset_person
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -19,7 +20,8 @@ logger = logging.getLogger(__name__)
|
||||
def main() -> None:
|
||||
"""Entry point for winnow CLI."""
|
||||
try:
|
||||
setup_logging(verbose=False)
|
||||
verbose = os.environ.get("VERBOSE", "").lower() in ("true", "1", "yes")
|
||||
setup_logging(verbose=verbose)
|
||||
|
||||
console.print(r"""
|
||||
[bold blue]winnow[/bold blue]
|
||||
@@ -63,17 +65,18 @@ def main() -> None:
|
||||
rprint("[bold red]Could not fetch people from Immich. Check URL/Key.[/bold red]")
|
||||
return
|
||||
|
||||
# Check for non-interactive mode
|
||||
auto_mode = os.environ.get("AUTO_MODE", "false").lower() == "true"
|
||||
# Auto mode when no TTY (Docker, cron, pipes) — the primary use case.
|
||||
# A TTY means local interactive use; AUTO_MODE=true overrides that for scripting.
|
||||
auto_mode = not sys.stdin.isatty() or os.environ.get("AUTO_MODE", "").lower() in ("true", "1", "yes")
|
||||
dry_run = os.environ.get("DRY_RUN", "false").lower() in ("true", "1", "yes")
|
||||
|
||||
if dry_run:
|
||||
rprint("[bold yellow]DRY RUN — no images will be downloaded or uploaded[/bold yellow]")
|
||||
|
||||
if auto_mode:
|
||||
rprint("[bold cyan]Running in AUTO mode (non-interactive)[/bold cyan]")
|
||||
jobs = auto_configure(people)
|
||||
else:
|
||||
rprint("[bold cyan]Interactive mode — set AUTO_MODE=true to skip prompts[/bold cyan]")
|
||||
jobs = interactive_configure(people)
|
||||
|
||||
if jobs:
|
||||
|
||||
+15
-9
@@ -39,8 +39,7 @@ class _Config:
|
||||
USE_FULL_RESOLUTION: bool = True
|
||||
ENABLE_FACE_ALIGNMENT: bool = True
|
||||
|
||||
# Caching (opt-in to avoid unexpected files)
|
||||
ENABLE_CACHE: bool = False
|
||||
ENABLE_CACHE: bool = True
|
||||
CACHE_DIR: str = ".if_cache"
|
||||
|
||||
def __new__(cls) -> "_Config":
|
||||
@@ -64,15 +63,14 @@ class _Config:
|
||||
self.FACE_MARGIN = float(os.getenv("FACE_MARGIN", "0.15"))
|
||||
self.USE_FULL_RESOLUTION = os.getenv("USE_FULL_RESOLUTION", "true").lower() in ("true", "1", "yes")
|
||||
self.ENABLE_FACE_ALIGNMENT = os.getenv("ENABLE_FACE_ALIGNMENT", "true").lower() in ("true", "1", "yes")
|
||||
self.ENABLE_CACHE = os.getenv("ENABLE_CACHE", "false").lower() in ("true", "1", "yes")
|
||||
self.ENABLE_CACHE = os.getenv("ENABLE_CACHE", "true").lower() in ("true", "1", "yes")
|
||||
self.CACHE_DIR = os.getenv("CACHE_DIR", ".if_cache")
|
||||
|
||||
# Fall back to config file for missing values
|
||||
# Fall back to config file for non-sensitive values (API_KEY not stored here)
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text())
|
||||
self.IMMICH_URL = self.IMMICH_URL or data.get("IMMICH_URL")
|
||||
self.API_KEY = self.API_KEY or data.get("API_KEY")
|
||||
if not os.getenv("OUTPUT_DIR"):
|
||||
self.OUTPUT_DIR = data.get("OUTPUT_DIR", self.OUTPUT_DIR)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
@@ -84,13 +82,16 @@ class _Config:
|
||||
cls._instance = None
|
||||
|
||||
def save(self) -> None:
|
||||
"""Persist configuration to file."""
|
||||
"""Persist non-sensitive configuration to file.
|
||||
|
||||
API_KEY is intentionally excluded — store it in .env or as an
|
||||
environment variable instead of a plain-text config file.
|
||||
"""
|
||||
try:
|
||||
CONFIG_FILE.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"IMMICH_URL": self.IMMICH_URL,
|
||||
"API_KEY": self.API_KEY,
|
||||
"OUTPUT_DIR": self.OUTPUT_DIR,
|
||||
},
|
||||
indent=2,
|
||||
@@ -113,8 +114,8 @@ class _Config:
|
||||
|
||||
if not self.API_KEY:
|
||||
console.print("[yellow]Immich API Key not found.[/yellow]")
|
||||
console.print("[dim]Tip: set API_KEY in your .env file to avoid re-entering it.[/dim]")
|
||||
self.API_KEY = Prompt.ask("Enter Immich API Key", password=True)
|
||||
self.save()
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Raise ValueError if required config is missing."""
|
||||
@@ -159,7 +160,12 @@ class _ConfigAccessor:
|
||||
|
||||
|
||||
Config = _ConfigAccessor()
|
||||
ConfigManager = type("ConfigManager", (), {"get": staticmethod(lambda: _Config())})
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
@staticmethod
|
||||
def get() -> _Config:
|
||||
return _Config()
|
||||
|
||||
|
||||
def get_headers() -> dict[str, str]:
|
||||
|
||||
+79
-60
@@ -29,6 +29,7 @@ def select_diverse_assets(
|
||||
entity_name: str,
|
||||
selection_mode: str = "smart",
|
||||
entity_type: str = "face",
|
||||
person_id: str | None = None,
|
||||
progress_callback=None,
|
||||
) -> list:
|
||||
"""
|
||||
@@ -59,7 +60,7 @@ def select_diverse_assets(
|
||||
return _select_time_spread(assets, limit)
|
||||
|
||||
try:
|
||||
return _select_by_embedding(assets, limit, entity_type, progress_callback)
|
||||
return _select_by_embedding(assets, limit, entity_type, person_id, progress_callback)
|
||||
except Exception as e:
|
||||
logger.error(f"Smart Diversity failed: {e}. Falling back to time spread.")
|
||||
return _select_time_spread(assets, limit)
|
||||
@@ -80,9 +81,11 @@ def _fetch_thumbnail(asset_id: str, timeout: int = 10) -> Image.Image | None:
|
||||
return None
|
||||
|
||||
|
||||
def _get_face_bbox(asset: dict) -> tuple[float, float, float, float] | None:
|
||||
"""Extract face bounding box from asset metadata if available."""
|
||||
def _get_face_bbox(asset: dict, person_id: str | None = None) -> tuple[float, float, float, float] | None:
|
||||
"""Extract face bounding box from asset metadata for the given person."""
|
||||
for person in asset.get("people", []):
|
||||
if person_id and person.get("id") != person_id:
|
||||
continue
|
||||
faces = person.get("faces", [])
|
||||
if faces:
|
||||
f = faces[0]
|
||||
@@ -95,12 +98,16 @@ def _get_face_bbox(asset: dict) -> tuple[float, float, float, float] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _get_face_confidence(asset: dict) -> float | None:
|
||||
"""Extract face detection confidence from asset metadata if available."""
|
||||
def _get_face_confidence(asset: dict, person_id: str | None = None) -> float | None:
|
||||
"""Extract face detection confidence from asset metadata for the given person."""
|
||||
for person in asset.get("people", []):
|
||||
if person_id and person.get("id") != person_id:
|
||||
continue
|
||||
faces = person.get("faces", [])
|
||||
if faces:
|
||||
return faces[0].get("score") or faces[0].get("confidence")
|
||||
f = faces[0]
|
||||
score = f.get("score")
|
||||
return score if score is not None else f.get("confidence")
|
||||
return None
|
||||
|
||||
|
||||
@@ -108,6 +115,7 @@ def _crop_face_from_thumbnail(
|
||||
img: Image.Image,
|
||||
asset: dict,
|
||||
margin: float = 0.25,
|
||||
person_id: str | None = None,
|
||||
) -> Image.Image | None:
|
||||
"""Crop the face region from a thumbnail using Immich bbox metadata.
|
||||
|
||||
@@ -118,19 +126,22 @@ def _crop_face_from_thumbnail(
|
||||
img: Full preview thumbnail
|
||||
asset: Asset dict with people/faces metadata
|
||||
margin: Extra margin around the bbox (fraction, default 25%)
|
||||
person_id: If provided, only crop from this person's face data.
|
||||
|
||||
Returns:
|
||||
Cropped face PIL image, or None if no face metadata available
|
||||
"""
|
||||
bbox = _get_face_bbox(asset)
|
||||
bbox = _get_face_bbox(asset, person_id=person_id)
|
||||
if bbox is None:
|
||||
return None
|
||||
|
||||
x1, y1, x2, y2 = bbox
|
||||
img_w, img_h = img.size
|
||||
|
||||
# Get metadata dimensions to scale bbox
|
||||
# Get metadata dimensions to scale bbox — must match the same person as _get_face_bbox
|
||||
for person in asset.get("people", []):
|
||||
if person_id and person.get("id") != person_id:
|
||||
continue
|
||||
faces = person.get("faces", [])
|
||||
if faces:
|
||||
meta_w = faces[0].get("imageWidth") or img_w
|
||||
@@ -169,6 +180,7 @@ def _select_by_embedding(
|
||||
assets: list,
|
||||
limit: int | str,
|
||||
entity_type: str,
|
||||
person_id: str | None = None,
|
||||
progress_callback=None,
|
||||
) -> list:
|
||||
"""Select assets using embedding-based cluster-aware FPS.
|
||||
@@ -191,64 +203,71 @@ def _select_by_embedding(
|
||||
else:
|
||||
candidates = assets
|
||||
|
||||
# --- Phase 1: Concurrent thumbnail download ---
|
||||
# --- Phases 1-4: Batched download → quality filter → crop → embed ---
|
||||
# Process in bounded batches so at most _BATCH decoded images live in RAM
|
||||
# at once. With 472 candidates each thumbnail is ~3-8 MB decoded; loading
|
||||
# all at once easily exhausts a 4 GB container limit on CPU.
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
thumbnail_map: dict[str, Image.Image] = {}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
futures = {pool.submit(_fetch_thumbnail, a["id"]): a for a in candidates}
|
||||
for i, future in enumerate(as_completed(futures)):
|
||||
if progress_callback:
|
||||
progress_callback(i, len(candidates))
|
||||
asset = futures[future]
|
||||
try:
|
||||
img = future.result()
|
||||
if img is not None:
|
||||
thumbnail_map[asset["id"]] = img
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# --- Phase 2-4: Quality filter → Crop → Embed ---
|
||||
_BATCH = 32
|
||||
embeddings, valid_candidates, confidence_scores = [], [], []
|
||||
quality_filtered = 0
|
||||
processed = 0
|
||||
|
||||
for asset in candidates:
|
||||
img = thumbnail_map.get(asset["id"])
|
||||
if img is None:
|
||||
continue
|
||||
for batch_start in range(0, len(candidates), _BATCH):
|
||||
batch = candidates[batch_start : batch_start + _BATCH]
|
||||
|
||||
confidence = _get_face_confidence(asset)
|
||||
# Download this batch concurrently
|
||||
batch_images: dict[str, Image.Image] = {}
|
||||
with ThreadPoolExecutor(max_workers=min(8, len(batch))) as pool:
|
||||
futures = {pool.submit(_fetch_thumbnail, a["id"]): a for a in batch}
|
||||
for future in as_completed(futures):
|
||||
asset = futures[future]
|
||||
try:
|
||||
img = future.result()
|
||||
if img is not None:
|
||||
batch_images[asset["id"]] = img
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to fetch thumbnail for {asset['id']}: {e}")
|
||||
continue
|
||||
|
||||
# Quality gate: filter before expensive embedding computation
|
||||
if entity_type == "face":
|
||||
face_bbox = _get_face_bbox(asset)
|
||||
quality = assess_quality(
|
||||
img,
|
||||
face_bbox=face_bbox,
|
||||
confidence=confidence,
|
||||
blur_threshold=Config.BLUR_THRESHOLD,
|
||||
min_face_px=Config.MIN_FACE_WIDTH,
|
||||
min_confidence=Config.MIN_CONFIDENCE,
|
||||
)
|
||||
if not quality.passed:
|
||||
quality_filtered += 1
|
||||
logger.debug(f"Quality filtered {asset['id']}: {quality.reason}")
|
||||
# Process each image; batch_images goes out of scope after this loop,
|
||||
# bounding peak thumbnail memory to _BATCH images per iteration.
|
||||
for asset in batch:
|
||||
img = batch_images.get(asset["id"])
|
||||
processed += 1
|
||||
if progress_callback:
|
||||
progress_callback(processed, len(candidates))
|
||||
if img is None:
|
||||
continue
|
||||
|
||||
# Crop the target person's face before embedding
|
||||
face_crop = _crop_face_from_thumbnail(img, asset)
|
||||
embed_img = face_crop if face_crop is not None else img
|
||||
else:
|
||||
embed_img = img
|
||||
confidence = _get_face_confidence(asset, person_id=person_id)
|
||||
|
||||
emb = get_embedding(embed_img, entity_type, asset_id=asset["id"])
|
||||
if emb is not None:
|
||||
embeddings.append(emb)
|
||||
valid_candidates.append(asset)
|
||||
confidence_scores.append(confidence)
|
||||
if entity_type == "face":
|
||||
face_bbox = _get_face_bbox(asset, person_id=person_id)
|
||||
quality = assess_quality(
|
||||
img,
|
||||
face_bbox=face_bbox,
|
||||
confidence=confidence,
|
||||
blur_threshold=Config.BLUR_THRESHOLD,
|
||||
min_face_px=Config.MIN_FACE_WIDTH,
|
||||
min_confidence=Config.MIN_CONFIDENCE,
|
||||
)
|
||||
if not quality.passed:
|
||||
quality_filtered += 1
|
||||
logger.debug(f"Quality filtered {asset['id']}: {quality.reason}")
|
||||
continue
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(len(candidates), len(candidates))
|
||||
face_crop = _crop_face_from_thumbnail(img, asset, person_id=person_id)
|
||||
embed_img = face_crop if face_crop is not None else img
|
||||
else:
|
||||
embed_img = img
|
||||
|
||||
emb = get_embedding(embed_img, entity_type, asset_id=asset["id"])
|
||||
if emb is not None:
|
||||
embeddings.append(emb)
|
||||
valid_candidates.append(asset)
|
||||
confidence_scores.append(confidence)
|
||||
|
||||
if quality_filtered > 0:
|
||||
logger.info(f"Quality filtering removed {quality_filtered} images.")
|
||||
@@ -360,7 +379,7 @@ def _compute_adaptive_threshold(emb_normed: np.ndarray, entity_type: str) -> flo
|
||||
fraction = 0.20 if entity_type == "face" else 0.10
|
||||
threshold = max(0.05, median_dist * fraction)
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"Adaptive threshold: {threshold:.4f} "
|
||||
f"(median_dist={median_dist:.4f}, fraction={fraction}, type={entity_type})"
|
||||
)
|
||||
@@ -402,7 +421,7 @@ def _cluster_aware_selection(
|
||||
|
||||
# --- Stage 1: K-Medoids clustering ---
|
||||
k = min(max(5, target // 4), n // 3, n) # e.g., 5-20 clusters
|
||||
logger.info(f"Clustering {n} embeddings into {k} groups (K-Medoids)...")
|
||||
logger.debug(f"Clustering {n} embeddings into {k} groups (K-Medoids)...")
|
||||
|
||||
# Compute full cosine distance matrix
|
||||
dist_matrix = 1 - emb_normed @ emb_normed.T
|
||||
@@ -411,7 +430,7 @@ def _cluster_aware_selection(
|
||||
selected = list(medoid_indices)
|
||||
selected_set = set(selected)
|
||||
|
||||
logger.info(f"Selected {len(selected)} cluster medoids as initial picks.")
|
||||
logger.debug(f"Selected {len(selected)} cluster medoids as initial picks.")
|
||||
|
||||
# --- Stage 2: FPS with hard example weighting ---
|
||||
min_dists = np.full(n, np.inf)
|
||||
@@ -436,8 +455,8 @@ def _cluster_aware_selection(
|
||||
break # All points selected
|
||||
|
||||
if limit == "auto" and best_dist < auto_threshold:
|
||||
logger.info(
|
||||
f"Auto-stop: Next best image {best_dist:.3f} away " f"(adaptive threshold {auto_threshold:.4f})."
|
||||
logger.debug(
|
||||
f"Auto-stop: next best image {best_dist:.3f} away (adaptive threshold {auto_threshold:.4f})."
|
||||
)
|
||||
break
|
||||
|
||||
|
||||
+85
-32
@@ -6,11 +6,13 @@ Unified embedding interface for faces and objects.
|
||||
- Caching: Disk-based cache avoids recomputation on reruns
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
@@ -20,6 +22,26 @@ from .cache import get_cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _suppress_output():
|
||||
"""Suppress stdout/stderr at the file-descriptor level, silencing C extension noise."""
|
||||
devnull_fd = os.open(os.devnull, os.O_WRONLY)
|
||||
saved_out, saved_err = os.dup(1), os.dup(2)
|
||||
try:
|
||||
os.dup2(devnull_fd, 1)
|
||||
os.dup2(devnull_fd, 2)
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
os.dup2(saved_out, 1)
|
||||
finally:
|
||||
os.dup2(saved_err, 2)
|
||||
os.close(devnull_fd)
|
||||
os.close(saved_out)
|
||||
os.close(saved_err)
|
||||
|
||||
|
||||
# Lazy-loaded singletons
|
||||
_insightface_app = None
|
||||
_insightface_loaded = False
|
||||
@@ -37,18 +59,14 @@ def _preload_cuda_libs() -> None:
|
||||
"""Preload CUDA/cuDNN DLLs so onnxruntime-gpu registers CUDAExecutionProvider.
|
||||
|
||||
Starting with onnxruntime-gpu 1.19+, CUDA/cuDNN libraries are no longer
|
||||
bundled inside the ORT package. They must be loaded from the nvidia-*
|
||||
pip packages (nvidia-cuda-runtime-cu12, nvidia-cudnn-cu12) before any
|
||||
InferenceSession is created.
|
||||
|
||||
Calling preload_dlls() with directory="" searches NVIDIA site-packages
|
||||
directories automatically.
|
||||
bundled inside the ORT package — they come from the nvidia-* pip packages.
|
||||
preload_dlls() locates them automatically via site-packages discovery.
|
||||
"""
|
||||
try:
|
||||
import onnxruntime
|
||||
if hasattr(onnxruntime, "preload_dlls"):
|
||||
onnxruntime.preload_dlls(cuda=True, cudnn=True, directory="")
|
||||
logger.info("Preloaded CUDA/cuDNN DLLs for onnxruntime-gpu")
|
||||
onnxruntime.preload_dlls(cuda=True, cudnn=True)
|
||||
logger.debug("Preloaded CUDA/cuDNN DLLs for onnxruntime-gpu")
|
||||
else:
|
||||
logger.debug("onnxruntime.preload_dlls() not available (ORT < 1.21)")
|
||||
except Exception as e:
|
||||
@@ -70,32 +88,48 @@ def get_insightface_app():
|
||||
# Preload CUDA/cuDNN DLLs BEFORE any ORT InferenceSession is created
|
||||
_preload_cuda_libs()
|
||||
|
||||
ctx_id = -1
|
||||
insightface_home = os.environ.get("INSIGHTFACE_HOME", os.path.expanduser("~/.insightface"))
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
# Disk cache check — lets the user know whether a download is coming
|
||||
buffalo_path = Path(insightface_home) / "models" / "buffalo_l"
|
||||
if buffalo_path.exists() and any(buffalo_path.iterdir()):
|
||||
logger.info("InsightFace Buffalo_L: found in model cache")
|
||||
else:
|
||||
logger.info("InsightFace Buffalo_L: not cached — downloading now (~300 MB)")
|
||||
|
||||
# Get providers, excluding TensorRT to avoid noisy errors
|
||||
providers = [p for p in ort.get_available_providers() if p != "TensorrtExecutionProvider"]
|
||||
logger.info(f"Available ONNX providers: {providers}")
|
||||
logger.debug(f"ONNX providers available: {providers}")
|
||||
|
||||
# Determine device: 0 for GPU, -1 for CPU
|
||||
gpu_providers = {
|
||||
"CUDAExecutionProvider",
|
||||
"ROCmExecutionProvider",
|
||||
"MPSExecutionProvider",
|
||||
"CoreMLExecutionProvider",
|
||||
}
|
||||
ctx_id = -1 if _is_force_cpu() else (0 if gpu_providers & set(providers) else -1)
|
||||
has_gpu_provider = bool(gpu_providers & set(providers))
|
||||
ctx_id = -1 if _is_force_cpu() else (0 if has_gpu_provider else -1)
|
||||
|
||||
if not has_gpu_provider and not _is_force_cpu():
|
||||
logger.warning(
|
||||
"No GPU execution provider found — running InsightFace on CPU. "
|
||||
"If you have an NVIDIA GPU, ensure the NVIDIA Container Toolkit is "
|
||||
"installed and the container has GPU access (deploy.resources in compose)."
|
||||
)
|
||||
|
||||
device_str = "GPU" if ctx_id >= 0 else "CPU"
|
||||
logger.info(f"Loading InsightFace Buffalo_L on {device_str} (ctx_id={ctx_id})...")
|
||||
logger.info(f"InsightFace Buffalo_L: loading into memory on {device_str}...")
|
||||
|
||||
# Suppress C-level output during model loading
|
||||
with open(os.devnull, "w") as devnull, contextlib.redirect_stdout(devnull), contextlib.redirect_stderr(devnull):
|
||||
insightface_home = os.environ.get("INSIGHTFACE_HOME", os.path.expanduser("~/.insightface"))
|
||||
t0 = time.time()
|
||||
with _suppress_output():
|
||||
_insightface_app = FaceAnalysis(name="buffalo_l", root=insightface_home, providers=providers)
|
||||
_insightface_app.prepare(ctx_id=ctx_id, det_size=(640, 640))
|
||||
|
||||
logger.info(f"InsightFace Buffalo_L: ready on {device_str} ({time.time() - t0:.1f}s)")
|
||||
return _insightface_app
|
||||
|
||||
except ImportError:
|
||||
@@ -103,17 +137,23 @@ def get_insightface_app():
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load InsightFace: {e}")
|
||||
# Retry on CPU if GPU failed
|
||||
if ctx_id == 0:
|
||||
logger.warning("Retrying InsightFace on CPU...")
|
||||
logger.warning("InsightFace GPU load failed — retrying on CPU...")
|
||||
try:
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
_insightface_app = FaceAnalysis(name="buffalo_l", root=insightface_home)
|
||||
_insightface_app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
t0 = time.time()
|
||||
with _suppress_output():
|
||||
_insightface_app = FaceAnalysis(
|
||||
name="buffalo_l",
|
||||
root=insightface_home,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
_insightface_app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
logger.info(f"InsightFace Buffalo_L: ready on CPU (fallback, {time.time() - t0:.1f}s)")
|
||||
return _insightface_app
|
||||
except Exception as ex:
|
||||
logger.error(f"CPU fallback failed: {ex}")
|
||||
logger.error(f"InsightFace CPU fallback failed: {ex}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -162,7 +202,18 @@ def get_siglip_model():
|
||||
from transformers import AutoImageProcessor, SiglipVisionModel
|
||||
|
||||
model_name = "google/siglip-base-patch16-224"
|
||||
logger.info(f"Loading SigLIP model ({model_name})...")
|
||||
|
||||
# Disk cache check — path derived from model_name using HuggingFace's slug convention
|
||||
hf_home = os.environ.get("HF_HOME", os.path.join(os.path.expanduser("~"), ".cache", "huggingface"))
|
||||
cache_slug = "models--" + model_name.replace("/", "--")
|
||||
model_cache = Path(hf_home) / "hub" / cache_slug
|
||||
if model_cache.exists() and any(model_cache.iterdir()):
|
||||
logger.info(f"SigLIP {model_name}: found in model cache")
|
||||
else:
|
||||
logger.info(f"SigLIP {model_name}: not cached — downloading now (~380 MB)")
|
||||
|
||||
logger.info(f"SigLIP {model_name}: loading into memory...")
|
||||
t0 = time.time()
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=FutureWarning)
|
||||
@@ -176,15 +227,16 @@ def get_siglip_model():
|
||||
if not _is_force_cpu():
|
||||
if torch.cuda.is_available():
|
||||
_siglip_model = _siglip_model.cuda()
|
||||
logger.info("SigLIP running on CUDA GPU")
|
||||
device_name = "CUDA GPU"
|
||||
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
||||
_siglip_model = _siglip_model.to("mps")
|
||||
logger.info("SigLIP running on Apple MPS")
|
||||
device_name = "Apple MPS"
|
||||
else:
|
||||
logger.info("SigLIP running on CPU")
|
||||
device_name = "CPU"
|
||||
else:
|
||||
logger.info("FORCE_CPU set. SigLIP running on CPU")
|
||||
device_name = "CPU (FORCE_CPU)"
|
||||
|
||||
logger.info(f"SigLIP {model_name}: ready on {device_name} ({time.time() - t0:.1f}s)")
|
||||
return _siglip_model, _siglip_processor
|
||||
|
||||
except ImportError as e:
|
||||
@@ -267,30 +319,31 @@ def get_embedding(
|
||||
|
||||
use_cache = Config.ENABLE_CACHE and asset_id is not None
|
||||
cache = get_cache(Config.CACHE_DIR) if use_cache else None
|
||||
model_key = "immich" if entity_type == "face" else "siglip"
|
||||
# Use a single consistent cache key per model so lookups and stores always match.
|
||||
# "immich" was previously used as the face key on the lookup path but "insightface"
|
||||
# on the store path — meaning the cache was never hit for locally-computed embeddings.
|
||||
cache_key = "insightface" if entity_type == "face" else "siglip"
|
||||
|
||||
# 1. Use Immich embedding if provided
|
||||
if immich_embedding is not None:
|
||||
if cache:
|
||||
cache.put(asset_id, immich_embedding, model_key)
|
||||
cache.put(asset_id, immich_embedding, cache_key)
|
||||
return immich_embedding
|
||||
|
||||
# 2. Check disk cache
|
||||
if cache:
|
||||
cached = cache.get(asset_id, model_key)
|
||||
cached = cache.get(asset_id, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# 3. Compute locally
|
||||
if entity_type == "face":
|
||||
emb = get_face_embedding(img_pil)
|
||||
model_key = "insightface"
|
||||
else:
|
||||
emb = get_object_embedding(img_pil)
|
||||
|
||||
# Cache the result
|
||||
if emb is not None and cache:
|
||||
cache.put(asset_id, emb, model_key)
|
||||
cache.put(asset_id, emb, cache_key)
|
||||
|
||||
return emb
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn
|
||||
from .config import Config, get_headers
|
||||
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
||||
from .immich_api import fetch_face_data, fetch_full_image
|
||||
from .logging import console
|
||||
from .log_config import console
|
||||
from .upload_tracker import mark_rejected, mark_uploaded
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,8 +21,13 @@ def get_frigate_face_counts() -> dict[str, int] | None:
|
||||
resp = requests.get(f"{frigate_url}/api/faces", timeout=10)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
train = data.get("train", {})
|
||||
return {name: len(files) for name, files in train.items() if isinstance(files, list)}
|
||||
# Response: {person_name: [file, ...], "train": [...], ...}
|
||||
# "train" is a flat pending list, not a person — skip it.
|
||||
return {
|
||||
name: len(files)
|
||||
for name, files in data.items()
|
||||
if name != "train" and isinstance(files, list)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not query Frigate face counts: {e}")
|
||||
return None
|
||||
|
||||
@@ -35,10 +35,13 @@ def get_people() -> list[dict]:
|
||||
headers=get_headers(),
|
||||
timeout=10,
|
||||
)
|
||||
if resp.status_code == 401:
|
||||
logger.error("Immich API key is invalid or expired (401 Unauthorized). Update API_KEY.")
|
||||
return []
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("people", [])
|
||||
except (requests.RequestException, ValueError) as e:
|
||||
logger.error(f"Failed to fetch people: {e}")
|
||||
logger.error(f"Failed to fetch people from Immich: {e}")
|
||||
return []
|
||||
|
||||
|
||||
@@ -49,7 +52,7 @@ def fetch_all_assets(person: dict) -> list[dict]:
|
||||
url = f"{Config.IMMICH_URL}/api/search/metadata"
|
||||
page_size = 1000
|
||||
|
||||
logger.info(f"Fetching assets for {name}...")
|
||||
logger.debug(f"Fetching assets for {name}...")
|
||||
|
||||
assets = []
|
||||
for page in range(1, MAX_PAGES + 1):
|
||||
@@ -137,10 +140,11 @@ def fetch_face_data(asset_id: str, person_id: str | None = None) -> FaceData | N
|
||||
face.get("boundingBoxY2", 0),
|
||||
)
|
||||
|
||||
score = face.get("score")
|
||||
return FaceData(
|
||||
embedding=embedding,
|
||||
bbox=bbox,
|
||||
confidence=face.get("score") or face.get("confidence"),
|
||||
confidence=score if score is not None else face.get("confidence"),
|
||||
image_width=face.get("imageWidth", 0),
|
||||
image_height=face.get("imageHeight", 0),
|
||||
)
|
||||
@@ -212,6 +216,6 @@ def filter_recent_assets(assets: list[dict], years: int | None = None) -> list[d
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
logger.info(f"Retained {len(recent)} assets (filtered {skipped} old assets).")
|
||||
logger.debug(f"Retained {len(recent)} assets (filtered {skipped} old assets).")
|
||||
return recent
|
||||
|
||||
|
||||
+28
-13
@@ -13,7 +13,7 @@ from .diversity import select_diverse_assets
|
||||
from .embeddings import is_embedding_available, load_embedding_model
|
||||
from .frigate_api import get_frigate_face_counts
|
||||
from .immich_api import fetch_all_assets, filter_recent_assets
|
||||
from .logging import console
|
||||
from .log_config import console
|
||||
from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -81,7 +81,9 @@ def _resolve_strategy(strategy: str, has_embedding: bool) -> tuple[int | str, st
|
||||
return strategy_map.get(strategy, ("auto", "smart"))
|
||||
|
||||
|
||||
def _perform_selection(assets: list, limit: int | str, name: str, selection_mode: str, entity_type: str) -> list:
|
||||
def _perform_selection(
|
||||
assets: list, limit: int | str, name: str, selection_mode: str, entity_type: str, person_id: str | None = None
|
||||
) -> list:
|
||||
"""Run diversity selection with progress display."""
|
||||
if selection_mode == "smart":
|
||||
model_display = "InsightFace (face embeddings)" if entity_type == "face" else "SigLIP (visual embeddings)"
|
||||
@@ -104,6 +106,7 @@ def _perform_selection(assets: list, limit: int | str, name: str, selection_mode
|
||||
name,
|
||||
selection_mode=selection_mode,
|
||||
entity_type=entity_type,
|
||||
person_id=person_id,
|
||||
progress_callback=lambda c, t: progress.update(task, completed=c, total=t),
|
||||
)
|
||||
|
||||
@@ -113,7 +116,9 @@ def _perform_selection(assets: list, limit: int | str, name: str, selection_mode
|
||||
|
||||
rprint(f"\n[cyan]Using time-spread selection for {limit} images...[/cyan]")
|
||||
with console.status(f"[bold]Selecting {limit} images evenly distributed over time...[/bold]"):
|
||||
selected = select_diverse_assets(assets, limit, name, selection_mode="time", entity_type=entity_type)
|
||||
selected = select_diverse_assets(
|
||||
assets, limit, name, selection_mode="time", entity_type=entity_type, person_id=person_id
|
||||
)
|
||||
rprint(f" [green]Selected {len(selected)} images using time spread.[/green]")
|
||||
return selected
|
||||
|
||||
@@ -145,8 +150,11 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None:
|
||||
|
||||
rprint(f" Found [bold]{len(all_assets)}[/bold] total, [bold]{len(recent_assets)}[/bold] in range ({years} years).")
|
||||
|
||||
# Filter out assets already uploaded to Frigate
|
||||
retry_rejected = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
|
||||
# Filter out assets already uploaded to Frigate.
|
||||
# In interactive mode, ask — use the env var only as the default so it can
|
||||
# still be pre-set (e.g. RETRY_REJECTED=true) without forcing the answer.
|
||||
retry_env = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
|
||||
retry_rejected = Confirm.ask("Include previously rejected images?", default=retry_env)
|
||||
before_dedup = len(recent_assets)
|
||||
new_asset_ids = set(filter_already_uploaded([a["id"] for a in recent_assets], retry_rejected=retry_rejected))
|
||||
recent_assets = [a for a in recent_assets if a["id"] in new_asset_ids]
|
||||
@@ -167,7 +175,9 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None:
|
||||
return None
|
||||
|
||||
# Perform selection
|
||||
selected_assets = _perform_selection(recent_assets, limit, name, selection_mode, entity_type)
|
||||
selected_assets = _perform_selection(
|
||||
recent_assets, limit, name, selection_mode, entity_type, person_id=person["id"]
|
||||
)
|
||||
|
||||
rprint(f" [green]Queued {len(selected_assets)} images for {name}.[/green]")
|
||||
return {"person": person, "assets": selected_assets, "limit": len(selected_assets), "config": config}
|
||||
@@ -276,10 +286,8 @@ def auto_configure(people: list[dict]) -> list[dict]:
|
||||
if frigate_counts is not None:
|
||||
already_uploaded = frigate_counts.get(name, 0)
|
||||
else:
|
||||
already_uploaded = (
|
||||
person_summary.get("frigate_count")
|
||||
or person_summary.get("uploaded", 0)
|
||||
)
|
||||
fc = person_summary.get("frigate_count")
|
||||
already_uploaded = fc if fc is not None else person_summary.get("uploaded", 0)
|
||||
capacity = Config.MAX_AUTO_IMAGES - already_uploaded
|
||||
if capacity <= 0:
|
||||
rprint(
|
||||
@@ -291,17 +299,24 @@ def auto_configure(people: list[dict]) -> list[dict]:
|
||||
has_embedding = is_embedding_available(entity_type)
|
||||
limit, selection_mode = _resolve_strategy(strategy, has_embedding)
|
||||
|
||||
# Cap selection to remaining capacity
|
||||
# Cap selection to remaining capacity.
|
||||
# For auto mode with partial training, keep "auto" so adaptive stopping
|
||||
# still runs — just trim the result to the remaining capacity afterward.
|
||||
auto_cap = None
|
||||
if limit == "auto":
|
||||
if already_uploaded > 0:
|
||||
limit = capacity # partially filled — select exactly what remains
|
||||
auto_cap = capacity
|
||||
else:
|
||||
limit = min(limit, capacity)
|
||||
|
||||
if selection_mode == "skip":
|
||||
continue
|
||||
|
||||
selected_assets = _perform_selection(recent_assets, limit, name, selection_mode, entity_type)
|
||||
selected_assets = _perform_selection(
|
||||
recent_assets, limit, name, selection_mode, entity_type, person_id=person["id"]
|
||||
)
|
||||
if auto_cap is not None:
|
||||
selected_assets = selected_assets[:auto_cap]
|
||||
|
||||
if selected_assets:
|
||||
rprint(f" [green]Queued {len(selected_assets)} images for {name}.[/green]")
|
||||
|
||||
@@ -26,10 +26,12 @@ def setup_logging(verbose: bool = False) -> logging.Logger:
|
||||
"""Configure logging with Rich console and file output."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
|
||||
# Configure root logger
|
||||
# Configure root logger; close existing handlers before replacing them
|
||||
root = logging.getLogger()
|
||||
root.setLevel(level)
|
||||
root.handlers.clear()
|
||||
for h in root.handlers[:]:
|
||||
h.close()
|
||||
root.removeHandler(h)
|
||||
|
||||
# Rich console handler - uses shared console to avoid breaking progress bars
|
||||
root.addHandler(RichHandler(rich_tracebacks=True, markup=True, console=console))
|
||||
@@ -37,7 +39,7 @@ def setup_logging(verbose: bool = False) -> logging.Logger:
|
||||
# File handler (always debug level) — log file respects OUTPUT_DIR if set
|
||||
log_dir = os.environ.get("OUTPUT_DIR", ".")
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
log_path = os.path.join(log_dir, "immich_export.log")
|
||||
log_path = os.path.join(log_dir, "winnow.log")
|
||||
file_handler = logging.FileHandler(log_path)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
|
||||
Reference in New Issue
Block a user