Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7b4722d3f2 | ||
|
|
444932c369 | ||
|
|
8f2a6d3163 | ||
|
|
b98fc94179 | ||
|
|
762ee160c3 | ||
|
|
759579fc30 |
@@ -15,22 +15,8 @@ env:
|
|||||||
IMAGE_NAME: sudolulo/winnow
|
IMAGE_NAME: sudolulo/winnow
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
verify-lockfile:
|
|
||||||
name: Verify uv.lock is current
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Check lockfile is up to date
|
|
||||||
run: uv lock --check
|
|
||||||
|
|
||||||
build:
|
build:
|
||||||
name: Build (${{ matrix.platform }})
|
name: Build (${{ matrix.platform }})
|
||||||
needs: verify-lockfile
|
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -140,4 +126,3 @@ jobs:
|
|||||||
- name: Inspect image
|
- name: Inspect image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
lint:
|
lint:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
env:
|
env:
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@@ -14,22 +14,8 @@ concurrency:
|
|||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
verify-lockfile:
|
|
||||||
name: Verify uv.lock is current
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Check lockfile is up to date
|
|
||||||
run: uv lock --check
|
|
||||||
|
|
||||||
release:
|
release:
|
||||||
name: Create GitHub Release & Build Image
|
name: Create GitHub Release & Build Image
|
||||||
needs: verify-lockfile
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
@@ -48,6 +34,12 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v4
|
||||||
|
|
||||||
|
- name: Ensure uv.lock is current
|
||||||
|
run: uv lock
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@v3
|
||||||
|
|
||||||
@@ -138,4 +130,3 @@ jobs:
|
|||||||
tags: |
|
tags: |
|
||||||
ghcr.io/sudolulo/winnow:latest
|
ghcr.io/sudolulo/winnow:latest
|
||||||
ghcr.io/sudolulo/winnow:${{ steps.tag.outputs.TAG }}
|
ghcr.io/sudolulo/winnow:${{ steps.tag.outputs.TAG }}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|||||||
@@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.2.3] - 2026-06-12
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **Clear-text API key storage**: `config.py` no longer writes `API_KEY` to `.immich_config.json`. The key must come from an environment variable or `.env` file. Interactive mode now prints a tip directing users to `.env`. Resolves CodeQL `py/clear-text-storage-sensitive-data`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **CI workflow permissions**: `test.yml` and `lint.yml` now declare `permissions: contents: read`, following least-privilege principle and resolving `actions/missing-workflow-permissions` scanner alerts.
|
||||||
|
- **CI lockfile race condition**: removed the `verify-lockfile` pre-job from `docker-publish.yml` and `release.yml`. The `update-lockfile.yml` bot maintains the lockfile; the verify step raced against it on the same push event and caused false failures. `release.yml` now runs `uv lock` inline so tag-triggered builds are always self-consistent.
|
||||||
|
- **Docs moved to wiki**: `docs/` folder removed from the repository. Setup, Troubleshooting, and FAQ pages are now at the [GitHub wiki](https://github.com/sudolulo/winnow/wiki).
|
||||||
|
|
||||||
|
## [0.2.2] - 2026-06-12
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Confidence scores in upload tracker**: Immich face confidence scores are now stored per asset in `frigate_uploaded_ids.json` under `by_person[name].scores`. Lays the groundwork for future replacement logic (remove low-confidence uploads when better images are found).
|
||||||
|
- **Frigate-authoritative capacity tracking**: At startup, `GET /api/faces` is queried on the Frigate host to retrieve the actual number of trained images per person from the `train` directory (pending/unclassified queue is excluded). This count is stored as `frigate_count` in the tracker JSON so it survives Frigate downtime.
|
||||||
|
- **Lifetime cap uses Frigate count**: `MAX_AUTO_IMAGES` is now enforced against Frigate's live training image count rather than the local uploaded-asset tally. Fallback priority: live Frigate API → last cached `frigate_count` in JSON → local uploaded count.
|
||||||
|
- **Startup summary shows Frigate count**: Tracker summary at startup now includes the last known Frigate training count per person (e.g. `78 uploaded, 2 rejected, 42 in Frigate`).
|
||||||
|
- **`winnow/frigate_api.py`**: new module encapsulating Frigate API helpers; currently exposes `get_frigate_face_counts()`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `upload_tracker.py`: `by_person` entries migrated from flat list to `{asset_ids, scores, frigate_count}` dict. Old list format is read and migrated transparently on first write.
|
||||||
|
- `mark_uploaded()` now accepts an optional `score` keyword argument.
|
||||||
|
- `get_person_summary()` now returns `frigate_count` and `scores` fields alongside `uploaded` and `rejected`.
|
||||||
|
|
||||||
## [0.2.1] - 2026-06-12
|
## [0.2.1] - 2026-06-12
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
-63
@@ -1,63 +0,0 @@
|
|||||||
# FAQ
|
|
||||||
|
|
||||||
## Does winnow modify my Immich library?
|
|
||||||
|
|
||||||
No. winnow only reads from Immich (assets, people, face bounding boxes). It never writes back to Immich or deletes anything.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## How many images should I upload to Frigate?
|
|
||||||
|
|
||||||
The `auto` strategy decides this for you — it keeps selecting until adding more images would be redundant. In practice this is usually 20–60 per person. You can cap it with `MAX_AUTO_IMAGES` (default 80).
|
|
||||||
|
|
||||||
Quality and diversity matter far more than volume. 30 well-spread images outperform 200 from the same week.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## What's the difference between face mode and object mode?
|
|
||||||
|
|
||||||
- **Face mode**: Extracts and aligns face crops, uploads them directly to Frigate's face training API. This is for teaching Frigate to recognize specific people.
|
|
||||||
- **Object mode**: Runs YOLO detection on full images and saves crops of a target class (dog, cat, car, etc.) to disk. Frigate has no API for object training data, so you place them manually.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Can I run it without Frigate?
|
|
||||||
|
|
||||||
Yes — in object mode, `FRIGATE_URL` is not used and crops are saved to the output volume. In face mode you need Frigate to receive the uploads, but you can use `DRY_RUN=true` to preview selection without uploading.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## How does auto-diversity mode work?
|
|
||||||
|
|
||||||
winnow computes a vector embedding for each candidate image (what the face/object actually looks like — angle, lighting, expression). It then clusters those embeddings and picks representatives that are maximally spread across the embedding space. It stops when the next-most-different image is already close to something already selected. See the README for the full pipeline.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Does it support multiple people in one run?
|
|
||||||
|
|
||||||
Yes. By default it processes every named person in your Immich library. Use `ONLY_PEOPLE` to whitelist specific names or `SKIP_PEOPLE` to exclude them.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## What GPU is needed?
|
|
||||||
|
|
||||||
Any NVIDIA GPU with CUDA 12.x support. The models (InsightFace Buffalo_L + SigLIP) fit comfortably in 4 GB VRAM. CPU mode works but is significantly slower.
|
|
||||||
|
|
||||||
ARM builds (linux/arm64) use CPU-only — CUDA is not available on ARM.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Does it work on Unraid / Proxmox / bare Docker?
|
|
||||||
|
|
||||||
Yes — the `compose.yml` uses standard Docker volume mounts. Replace the example paths with whatever absolute paths suit your setup.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## How do I update winnow?
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose pull
|
|
||||||
docker compose up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
The `latest` tag on GHCR tracks the `main` branch. Pinning to a version tag (e.g. `ghcr.io/sudolulo/winnow:v0.2.0`) is recommended for stability.
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
# Setup Guide
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- [Immich](https://immich.app) v1.106+ with face recognition enabled and people tagged
|
|
||||||
- [Frigate](https://frigate.video) v0.16+ (face mode only)
|
|
||||||
- Docker with the NVIDIA container toolkit (optional but strongly recommended)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. Get your Immich API key
|
|
||||||
|
|
||||||
1. Open Immich → **Account Settings** → **API Keys**
|
|
||||||
2. Click **New API Key**, give it a name (e.g. `winnow`), copy the key
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. Get your Frigate URL
|
|
||||||
|
|
||||||
This is the base URL of your Frigate instance, e.g. `http://192.168.1.10:5000`. Only needed for face mode — omit it entirely if you're using object mode.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. Deploy with Docker Compose
|
|
||||||
|
|
||||||
Copy [`compose.yml`](../compose.yml) and [`.env.example`](../.env.example) to a directory on your host:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mkdir winnow && cd winnow
|
|
||||||
curl -O https://raw.githubusercontent.com/sudolulo/winnow/main/compose.yml
|
|
||||||
curl -O https://raw.githubusercontent.com/sudolulo/winnow/main/.env.example
|
|
||||||
cp .env.example .env
|
|
||||||
```
|
|
||||||
|
|
||||||
Edit `.env` with your values:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
IMMICH_URL=http://192.168.1.10:2283
|
|
||||||
API_KEY=your-immich-api-key
|
|
||||||
FRIGATE_URL=http://192.168.1.10:5000
|
|
||||||
```
|
|
||||||
|
|
||||||
Edit the volume paths in `compose.yml` to point to directories on your host where models, cache, and output crops should be stored:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
volumes:
|
|
||||||
- /your/path/to/models:/models
|
|
||||||
- /your/path/to/cache:/app/.if_cache
|
|
||||||
- /your/path/to/output:/app/frigate_train
|
|
||||||
```
|
|
||||||
|
|
||||||
These directories will be created automatically by Docker if they don't exist.
|
|
||||||
|
|
||||||
Start it:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
Logs:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose logs -f winnow
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. First run
|
|
||||||
|
|
||||||
On the first run, winnow downloads the embedding models (~1–2 GB) from HuggingFace and InsightFace. This happens once — subsequent runs use the cached models from your mounted volume and start immediately.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. Scheduling
|
|
||||||
|
|
||||||
Set `CRON_SCHEDULE` in your `.env` to keep winnow running on a schedule:
|
|
||||||
|
|
||||||
```
|
|
||||||
CRON_SCHEDULE=0 3 * * 0 # Every Sunday at 3 AM
|
|
||||||
```
|
|
||||||
|
|
||||||
Without `CRON_SCHEDULE`, the container runs once and exits.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## GPU passthrough
|
|
||||||
|
|
||||||
To enable GPU acceleration, include the `deploy` block in `compose.yml` (already present in the example) and ensure the NVIDIA container toolkit is installed on your host:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Verify GPU is accessible to Docker
|
|
||||||
docker run --rm --gpus all nvidia/cuda:12.9.2-base-ubuntu22.04 nvidia-smi
|
|
||||||
```
|
|
||||||
|
|
||||||
CPU mode works without any GPU setup — set `FORCE_CPU=true` to disable GPU explicitly.
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
# Troubleshooting
|
|
||||||
|
|
||||||
## Container exits immediately
|
|
||||||
|
|
||||||
Check logs:
|
|
||||||
```bash
|
|
||||||
docker compose logs winnow
|
|
||||||
```
|
|
||||||
|
|
||||||
Common causes:
|
|
||||||
- **Missing required env var** — `IMMICH_URL` or `API_KEY` not set
|
|
||||||
- **Cannot reach Immich** — check the URL and that Immich is running; use `http://` not `https://` unless you have TLS set up
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## "No people found" / nothing processed
|
|
||||||
|
|
||||||
- Make sure Immich has completed face recognition and you have named people in your library
|
|
||||||
- `YEARS_FILTER` defaults to 10 years — increase it if your tagged photos are older
|
|
||||||
- `MIN_FACE_COUNT` skips people with few photos — lower or remove it
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Frigate upload fails
|
|
||||||
|
|
||||||
- Confirm `FRIGATE_URL` is reachable from inside the container: `docker exec winnow curl $FRIGATE_URL/api/stats`
|
|
||||||
- Check Frigate v0.16+ — older versions don't have the face training API
|
|
||||||
- Set `DRY_RUN=true` to verify selection without uploading
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Models fail to download
|
|
||||||
|
|
||||||
winnow downloads InsightFace and HuggingFace (SigLIP) models on first run.
|
|
||||||
|
|
||||||
- Ensure the container has internet access
|
|
||||||
- Confirm the model volume is mounted and writable
|
|
||||||
- If behind a proxy, set `HTTP_PROXY` / `HTTPS_PROXY` env vars
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Running on CPU (no GPU)
|
|
||||||
|
|
||||||
Set `FORCE_CPU=true`. Everything works but embedding computation is slower — expect several minutes per person instead of seconds.
|
|
||||||
|
|
||||||
If you have a GPU but it's not being used:
|
|
||||||
- Confirm the NVIDIA container toolkit is installed: `docker run --rm --gpus all nvidia/cuda:12.9.2-base-ubuntu22.04 nvidia-smi`
|
|
||||||
- Confirm the `deploy.resources.reservations.devices` block is present in `compose.yml`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Same images uploaded every run
|
|
||||||
|
|
||||||
The upload tracker is stored in `CACHE_DIR` (`/app/.if_cache` by default). If this volume isn't persisted between runs, the tracker resets and images are re-uploaded.
|
|
||||||
|
|
||||||
Make sure `/app/.if_cache` is mounted to a persistent host path.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Re-uploading a specific person
|
|
||||||
|
|
||||||
To clear the upload history for one person and start fresh:
|
|
||||||
|
|
||||||
```env
|
|
||||||
RESET_PERSON=John
|
|
||||||
```
|
|
||||||
|
|
||||||
Remove this after one run — it clears the history and then processes normally.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Image quality issues
|
|
||||||
|
|
||||||
- **Too blurry**: Lower `BLUR_THRESHOLD` (default 100) — e.g. `50` accepts more blur
|
|
||||||
- **Face too small**: Lower `MIN_FACE_WIDTH` (default 50px)
|
|
||||||
- **Low confidence detections included**: Raise `MIN_CONFIDENCE` (default 0.7)
|
|
||||||
- **Rejected images being re-tried**: Set `RETRY_REJECTED=true` for one run
|
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.2.1"
|
version = "0.2.3"
|
||||||
description = "Immich to Frigate training sets"
|
description = "Immich to Frigate training sets"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
|
|||||||
@@ -2289,7 +2289,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.2.1"
|
version = "0.2.3"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "croniter" },
|
{ name = "croniter" },
|
||||||
|
|||||||
+9
-1
@@ -48,7 +48,15 @@ def main() -> None:
|
|||||||
if summary:
|
if summary:
|
||||||
rprint("\n[dim]Tracker summary:[/dim]")
|
rprint("\n[dim]Tracker summary:[/dim]")
|
||||||
for person_name, counts in summary.items():
|
for person_name, counts in summary.items():
|
||||||
rprint(f" [dim]{person_name}: {counts['uploaded']} uploaded, {counts['rejected']} rejected[/dim]")
|
frigate_part = (
|
||||||
|
f", {counts['frigate_count']} in Frigate"
|
||||||
|
if counts.get("frigate_count") is not None
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
rprint(
|
||||||
|
f" [dim]{person_name}: {counts['uploaded']} uploaded,"
|
||||||
|
f" {counts['rejected']} rejected{frigate_part}[/dim]"
|
||||||
|
)
|
||||||
|
|
||||||
people = get_people()
|
people = get_people()
|
||||||
if not people:
|
if not people:
|
||||||
|
|||||||
+7
-5
@@ -67,12 +67,11 @@ class _Config:
|
|||||||
self.ENABLE_CACHE = os.getenv("ENABLE_CACHE", "false").lower() in ("true", "1", "yes")
|
self.ENABLE_CACHE = os.getenv("ENABLE_CACHE", "false").lower() in ("true", "1", "yes")
|
||||||
self.CACHE_DIR = os.getenv("CACHE_DIR", ".if_cache")
|
self.CACHE_DIR = os.getenv("CACHE_DIR", ".if_cache")
|
||||||
|
|
||||||
# Fall back to config file for missing values
|
# Fall back to config file for non-sensitive values (API_KEY not stored here)
|
||||||
if CONFIG_FILE.exists():
|
if CONFIG_FILE.exists():
|
||||||
try:
|
try:
|
||||||
data = json.loads(CONFIG_FILE.read_text())
|
data = json.loads(CONFIG_FILE.read_text())
|
||||||
self.IMMICH_URL = self.IMMICH_URL or data.get("IMMICH_URL")
|
self.IMMICH_URL = self.IMMICH_URL or data.get("IMMICH_URL")
|
||||||
self.API_KEY = self.API_KEY or data.get("API_KEY")
|
|
||||||
if not os.getenv("OUTPUT_DIR"):
|
if not os.getenv("OUTPUT_DIR"):
|
||||||
self.OUTPUT_DIR = data.get("OUTPUT_DIR", self.OUTPUT_DIR)
|
self.OUTPUT_DIR = data.get("OUTPUT_DIR", self.OUTPUT_DIR)
|
||||||
except (json.JSONDecodeError, OSError) as e:
|
except (json.JSONDecodeError, OSError) as e:
|
||||||
@@ -84,13 +83,16 @@ class _Config:
|
|||||||
cls._instance = None
|
cls._instance = None
|
||||||
|
|
||||||
def save(self) -> None:
|
def save(self) -> None:
|
||||||
"""Persist configuration to file."""
|
"""Persist non-sensitive configuration to file.
|
||||||
|
|
||||||
|
API_KEY is intentionally excluded — store it in .env or as an
|
||||||
|
environment variable instead of a plain-text config file.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
CONFIG_FILE.write_text(
|
CONFIG_FILE.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
"IMMICH_URL": self.IMMICH_URL,
|
"IMMICH_URL": self.IMMICH_URL,
|
||||||
"API_KEY": self.API_KEY,
|
|
||||||
"OUTPUT_DIR": self.OUTPUT_DIR,
|
"OUTPUT_DIR": self.OUTPUT_DIR,
|
||||||
},
|
},
|
||||||
indent=2,
|
indent=2,
|
||||||
@@ -113,8 +115,8 @@ class _Config:
|
|||||||
|
|
||||||
if not self.API_KEY:
|
if not self.API_KEY:
|
||||||
console.print("[yellow]Immich API Key not found.[/yellow]")
|
console.print("[yellow]Immich API Key not found.[/yellow]")
|
||||||
|
console.print("[dim]Tip: set API_KEY in your .env file to avoid re-entering it.[/dim]")
|
||||||
self.API_KEY = Prompt.ask("Enter Immich API Key", password=True)
|
self.API_KEY = Prompt.ask("Enter Immich API Key", password=True)
|
||||||
self.save()
|
|
||||||
|
|
||||||
def validate(self) -> None:
|
def validate(self) -> None:
|
||||||
"""Raise ValueError if required config is missing."""
|
"""Raise ValueError if required config is missing."""
|
||||||
|
|||||||
+9
-3
@@ -58,6 +58,7 @@ def _enrich_asset_with_face_data(asset: dict, person: dict) -> dict:
|
|||||||
|
|
||||||
# Inject into asset so process_face_mode can find it via asset["people"]
|
# Inject into asset so process_face_mode can find it via asset["people"]
|
||||||
asset["people"] = [{"id": person_id, "faces": [face_info]}]
|
asset["people"] = [{"id": person_id, "faces": [face_info]}]
|
||||||
|
asset["face_confidence"] = face_data.confidence
|
||||||
return asset
|
return asset
|
||||||
|
|
||||||
|
|
||||||
@@ -96,8 +97,9 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
shutil.rmtree(person_dir)
|
shutil.rmtree(person_dir)
|
||||||
os.makedirs(person_dir, exist_ok=True)
|
os.makedirs(person_dir, exist_ok=True)
|
||||||
|
|
||||||
# Track filename → asset_id mapping for upload dedup
|
# Track filename → asset_id and filename → confidence score
|
||||||
asset_map: dict[str, str] = {}
|
asset_map: dict[str, str] = {}
|
||||||
|
score_map: dict[str, float | None] = {}
|
||||||
|
|
||||||
count = 0
|
count = 0
|
||||||
for asset in assets:
|
for asset in assets:
|
||||||
@@ -132,11 +134,13 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
# Record which asset produced which output file
|
# Record which asset produced which output file
|
||||||
filename = f"{count}.jpg"
|
filename = f"{count}.jpg"
|
||||||
asset_map[filename] = asset["id"]
|
asset_map[filename] = asset["id"]
|
||||||
|
score_map[filename] = asset.get("face_confidence")
|
||||||
# Also record object-mode variant filenames
|
# Also record object-mode variant filenames
|
||||||
if mode == "object":
|
if mode == "object":
|
||||||
for f in sorted(os.listdir(person_dir)):
|
for f in sorted(os.listdir(person_dir)):
|
||||||
if f.startswith(f"{count}_") and f not in asset_map:
|
if f.startswith(f"{count}_") and f not in asset_map:
|
||||||
asset_map[f] = asset["id"]
|
asset_map[f] = asset["id"]
|
||||||
|
score_map[f] = asset.get("face_confidence")
|
||||||
|
|
||||||
count += 1
|
count += 1
|
||||||
else:
|
else:
|
||||||
@@ -149,8 +153,9 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
progress.advance(job_task)
|
progress.advance(job_task)
|
||||||
progress.advance(overall_task)
|
progress.advance(overall_task)
|
||||||
|
|
||||||
# Store asset_map on the job so upload_to_frigate can use it
|
# Store maps on the job so upload_to_frigate can use them
|
||||||
job["asset_map"] = asset_map
|
job["asset_map"] = asset_map
|
||||||
|
job["score_map"] = score_map
|
||||||
|
|
||||||
progress.remove_task(job_task)
|
progress.remove_task(job_task)
|
||||||
|
|
||||||
@@ -230,6 +235,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
asset_map = filename_to_asset_id.get(name, {})
|
asset_map = filename_to_asset_id.get(name, {})
|
||||||
|
score_map = job.get("score_map", {})
|
||||||
person_files = sorted(asset_map.keys())
|
person_files = sorted(asset_map.keys())
|
||||||
|
|
||||||
if not person_files:
|
if not person_files:
|
||||||
@@ -257,7 +263,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
# Mark this asset as uploaded so it's skipped on future runs
|
# Mark this asset as uploaded so it's skipped on future runs
|
||||||
asset_id = asset_map.get(fname)
|
asset_id = asset_map.get(fname)
|
||||||
if asset_id:
|
if asset_id:
|
||||||
mark_uploaded(asset_id, person_name=name)
|
mark_uploaded(asset_id, person_name=name, score=score_map.get(fname))
|
||||||
|
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""Frigate API helpers for querying face training state."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def get_frigate_face_counts() -> dict[str, int] | None:
|
||||||
|
"""Return {person_name: training_image_count} from Frigate's train directory.
|
||||||
|
|
||||||
|
Returns None if FRIGATE_URL is not set or the API is unreachable, so callers
|
||||||
|
can distinguish "API unavailable" from "person has 0 images."
|
||||||
|
"""
|
||||||
|
frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
|
||||||
|
if not frigate_url:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
resp = requests.get(f"{frigate_url}/api/faces", timeout=10)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
train = data.get("train", {})
|
||||||
|
return {name: len(files) for name, files in train.items() if isinstance(files, list)}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Could not query Frigate face counts: {e}")
|
||||||
|
return None
|
||||||
+33
-1
@@ -11,9 +11,10 @@ from rich.table import Table
|
|||||||
from .config import Config
|
from .config import Config
|
||||||
from .diversity import select_diverse_assets
|
from .diversity import select_diverse_assets
|
||||||
from .embeddings import is_embedding_available, load_embedding_model
|
from .embeddings import is_embedding_available, load_embedding_model
|
||||||
|
from .frigate_api import get_frigate_face_counts
|
||||||
from .immich_api import fetch_all_assets, filter_recent_assets
|
from .immich_api import fetch_all_assets, filter_recent_assets
|
||||||
from .logging import console
|
from .logging import console
|
||||||
from .upload_tracker import filter_already_uploaded
|
from .upload_tracker import filter_already_uploaded, get_person_summary, update_frigate_count
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -236,6 +237,12 @@ def auto_configure(people: list[dict]) -> list[dict]:
|
|||||||
f" ≥{min_face_count} assets (MIN_FACE_COUNT={min_face_count})"
|
f" ≥{min_face_count} assets (MIN_FACE_COUNT={min_face_count})"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
frigate_counts = get_frigate_face_counts()
|
||||||
|
# Persist each count to tracker so the last known value survives Frigate downtime
|
||||||
|
if frigate_counts is not None:
|
||||||
|
for pname, count in frigate_counts.items():
|
||||||
|
update_frigate_count(pname, count)
|
||||||
|
upload_summary = get_person_summary()
|
||||||
jobs = []
|
jobs = []
|
||||||
for person in valid_people:
|
for person in valid_people:
|
||||||
name = person["name"]
|
name = person["name"]
|
||||||
@@ -263,9 +270,34 @@ def auto_configure(people: list[dict]) -> list[dict]:
|
|||||||
rprint(f" [dim]Skipping {name} (0 new images after dedup).[/dim]")
|
rprint(f" [dim]Skipping {name} (0 new images after dedup).[/dim]")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Enforce MAX_AUTO_IMAGES as a lifetime cap per person.
|
||||||
|
# Priority: live Frigate count → last cached Frigate count → local uploaded count.
|
||||||
|
person_summary = upload_summary.get(name, {})
|
||||||
|
if frigate_counts is not None:
|
||||||
|
already_uploaded = frigate_counts.get(name, 0)
|
||||||
|
else:
|
||||||
|
already_uploaded = (
|
||||||
|
person_summary.get("frigate_count")
|
||||||
|
or person_summary.get("uploaded", 0)
|
||||||
|
)
|
||||||
|
capacity = Config.MAX_AUTO_IMAGES - already_uploaded
|
||||||
|
if capacity <= 0:
|
||||||
|
rprint(
|
||||||
|
f" [dim]Skipping {name} (at lifetime cap:"
|
||||||
|
f" {already_uploaded}/{Config.MAX_AUTO_IMAGES} trained).[/dim]"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
has_embedding = is_embedding_available(entity_type)
|
has_embedding = is_embedding_available(entity_type)
|
||||||
limit, selection_mode = _resolve_strategy(strategy, has_embedding)
|
limit, selection_mode = _resolve_strategy(strategy, has_embedding)
|
||||||
|
|
||||||
|
# Cap selection to remaining capacity
|
||||||
|
if limit == "auto":
|
||||||
|
if already_uploaded > 0:
|
||||||
|
limit = capacity # partially filled — select exactly what remains
|
||||||
|
else:
|
||||||
|
limit = min(limit, capacity)
|
||||||
|
|
||||||
if selection_mode == "skip":
|
if selection_mode == "skip":
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
+61
-19
@@ -8,6 +8,13 @@ Both are excluded from future candidate pools. To reset:
|
|||||||
- All: delete both files
|
- All: delete both files
|
||||||
- One person: call reset_person("Name") or set RESET_PERSON=Name
|
- One person: call reset_person("Name") or set RESET_PERSON=Name
|
||||||
- Rejects only: delete frigate_rejected_ids.json, or set RETRY_REJECTED=true
|
- Rejects only: delete frigate_rejected_ids.json, or set RETRY_REJECTED=true
|
||||||
|
|
||||||
|
by_person schema (frigate_uploaded_ids.json):
|
||||||
|
{
|
||||||
|
"asset_ids": ["immich-id-1", ...], # all assets we attempted to upload
|
||||||
|
"scores": {"immich-id-1": 0.953}, # Immich face confidence at upload time
|
||||||
|
"frigate_count": 42 # last known Frigate training image count
|
||||||
|
}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
@@ -55,7 +62,23 @@ def _load_flat(filename: str) -> set[str]:
|
|||||||
return set(_load(filename).get(_flat_key(filename), []))
|
return set(_load(filename).get(_flat_key(filename), []))
|
||||||
|
|
||||||
|
|
||||||
def _mark(filename: str, asset_id: str, person_name: str | None) -> None:
|
def _get_ids(entry: list | dict) -> list[str]:
|
||||||
|
"""Extract asset_ids from either the old list format or the new dict format."""
|
||||||
|
if isinstance(entry, list):
|
||||||
|
return entry
|
||||||
|
return entry.get("asset_ids", [])
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_entry(entry: list | dict) -> dict:
|
||||||
|
"""Ensure by_person entry is in the current dict format."""
|
||||||
|
if isinstance(entry, list):
|
||||||
|
return {"asset_ids": sorted(entry), "scores": {}}
|
||||||
|
entry.setdefault("asset_ids", [])
|
||||||
|
entry.setdefault("scores", {})
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
def _mark(filename: str, asset_id: str, person_name: str | None, score: float | None = None) -> None:
|
||||||
data = _load(filename)
|
data = _load(filename)
|
||||||
flat_key = _flat_key(filename)
|
flat_key = _flat_key(filename)
|
||||||
flat = set(data.get(flat_key, []))
|
flat = set(data.get(flat_key, []))
|
||||||
@@ -63,9 +86,13 @@ def _mark(filename: str, asset_id: str, person_name: str | None) -> None:
|
|||||||
data[flat_key] = sorted(flat)
|
data[flat_key] = sorted(flat)
|
||||||
if person_name:
|
if person_name:
|
||||||
by_person = data.setdefault("by_person", {})
|
by_person = data.setdefault("by_person", {})
|
||||||
person_ids = set(by_person.get(person_name, []))
|
entry = _migrate_entry(by_person.get(person_name, {}))
|
||||||
person_ids.add(asset_id)
|
ids = set(entry["asset_ids"])
|
||||||
by_person[person_name] = sorted(person_ids)
|
ids.add(asset_id)
|
||||||
|
entry["asset_ids"] = sorted(ids)
|
||||||
|
if score is not None:
|
||||||
|
entry["scores"][asset_id] = round(score, 4)
|
||||||
|
by_person[person_name] = entry
|
||||||
_save(filename, data)
|
_save(filename, data)
|
||||||
|
|
||||||
|
|
||||||
@@ -79,8 +106,8 @@ def load_rejected_ids() -> set[str]:
|
|||||||
return _load_flat(REJECT_TRACKER_FILE)
|
return _load_flat(REJECT_TRACKER_FILE)
|
||||||
|
|
||||||
|
|
||||||
def mark_uploaded(asset_id: str, person_name: str | None = None) -> None:
|
def mark_uploaded(asset_id: str, person_name: str | None = None, score: float | None = None) -> None:
|
||||||
_mark(UPLOAD_TRACKER_FILE, asset_id, person_name)
|
_mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score)
|
||||||
logger.debug(f"Marked {asset_id} as uploaded ({person_name})")
|
logger.debug(f"Marked {asset_id} as uploaded ({person_name})")
|
||||||
|
|
||||||
|
|
||||||
@@ -89,14 +116,25 @@ def mark_rejected(asset_id: str, person_name: str | None = None) -> None:
|
|||||||
logger.debug(f"Marked {asset_id} as rejected ({person_name})")
|
logger.debug(f"Marked {asset_id} as rejected ({person_name})")
|
||||||
|
|
||||||
|
|
||||||
|
def update_frigate_count(person_name: str, count: int) -> None:
|
||||||
|
"""Record Frigate's authoritative training image count for a person."""
|
||||||
|
data = _load(UPLOAD_TRACKER_FILE)
|
||||||
|
by_person = data.setdefault("by_person", {})
|
||||||
|
entry = _migrate_entry(by_person.get(person_name, {}))
|
||||||
|
entry["frigate_count"] = count
|
||||||
|
by_person[person_name] = entry
|
||||||
|
_save(UPLOAD_TRACKER_FILE, data)
|
||||||
|
|
||||||
|
|
||||||
def reset_person(person_name: str) -> None:
|
def reset_person(person_name: str) -> None:
|
||||||
"""Remove all uploaded and rejected records for a given person."""
|
"""Remove all uploaded and rejected records for a given person."""
|
||||||
for filename in (UPLOAD_TRACKER_FILE, REJECT_TRACKER_FILE):
|
for filename in (UPLOAD_TRACKER_FILE, REJECT_TRACKER_FILE):
|
||||||
data = _load(filename)
|
data = _load(filename)
|
||||||
flat_key = _flat_key(filename)
|
flat_key = _flat_key(filename)
|
||||||
by_person = data.get("by_person", {})
|
by_person = data.get("by_person", {})
|
||||||
person_ids = set(by_person.pop(person_name, []))
|
entry = by_person.pop(person_name, None)
|
||||||
if person_ids:
|
if entry is not None:
|
||||||
|
person_ids = set(_get_ids(entry))
|
||||||
flat = set(data.get(flat_key, [])) - person_ids
|
flat = set(data.get(flat_key, [])) - person_ids
|
||||||
data[flat_key] = sorted(flat)
|
data[flat_key] = sorted(flat)
|
||||||
data["by_person"] = by_person
|
data["by_person"] = by_person
|
||||||
@@ -104,18 +142,22 @@ def reset_person(person_name: str) -> None:
|
|||||||
logger.info(f"Reset tracking data for {person_name}")
|
logger.info(f"Reset tracking data for {person_name}")
|
||||||
|
|
||||||
|
|
||||||
def get_person_summary() -> dict[str, dict[str, int]]:
|
def get_person_summary() -> dict[str, dict]:
|
||||||
"""Return {person_name: {uploaded: N, rejected: N}} for display."""
|
"""Return {person_name: {uploaded, rejected, frigate_count, scores}} for display/capacity."""
|
||||||
uploaded_by = _load(UPLOAD_TRACKER_FILE).get("by_person", {})
|
uploaded_data = _load(UPLOAD_TRACKER_FILE).get("by_person", {})
|
||||||
rejected_by = _load(REJECT_TRACKER_FILE).get("by_person", {})
|
rejected_data = _load(REJECT_TRACKER_FILE).get("by_person", {})
|
||||||
names = set(uploaded_by) | set(rejected_by)
|
names = set(uploaded_data) | set(rejected_data)
|
||||||
return {
|
result = {}
|
||||||
name: {
|
for name in sorted(names):
|
||||||
"uploaded": len(uploaded_by.get(name, [])),
|
u_entry = uploaded_data.get(name, {})
|
||||||
"rejected": len(rejected_by.get(name, [])),
|
r_entry = rejected_data.get(name, {})
|
||||||
|
result[name] = {
|
||||||
|
"uploaded": len(_get_ids(u_entry)),
|
||||||
|
"rejected": len(_get_ids(r_entry)),
|
||||||
|
"frigate_count": u_entry.get("frigate_count") if isinstance(u_entry, dict) else None,
|
||||||
|
"scores": u_entry.get("scores", {}) if isinstance(u_entry, dict) else {},
|
||||||
}
|
}
|
||||||
for name in sorted(names)
|
return result
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def filter_already_uploaded(
|
def filter_already_uploaded(
|
||||||
|
|||||||
Reference in New Issue
Block a user