Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f86c1054a | ||
|
|
634688fc93 | ||
|
|
9f0a78522f | ||
|
|
8d1f5da05a | ||
|
|
a7257cf031 | ||
|
|
326fdbdf38 | ||
|
|
405413490b | ||
|
|
91e0858aa6 | ||
|
|
e67f2d9638 | ||
|
|
71df0e81de | ||
|
|
9bb0727807 | ||
|
|
bbbac18207 | ||
|
|
6fcea587ff | ||
|
|
ed045f07dd | ||
|
|
110a45f467 | ||
|
|
785c9d4a22 | ||
|
|
e0a5d98df6 | ||
|
|
ef5934d1af | ||
|
|
903d7f1054 | ||
|
|
f322eba380 | ||
|
|
26b598db98 | ||
|
|
03be6ce2cb | ||
|
|
ab641847b2 | ||
|
|
2b1d9e8b8a | ||
|
|
4856a6d36f | ||
|
|
7c306a4423 | ||
|
|
c22857b912 | ||
|
|
c53172f5ff | ||
|
|
c0c2d88941 |
+5
-2
@@ -23,13 +23,16 @@ STRATEGY=auto
|
||||
# YEARS_FILTER=10 # Only include images from the last N years (default: 10)
|
||||
|
||||
# ── Image Quality ─────────────────────────────────────────────────────────────
|
||||
# MIN_FACE_WIDTH=50 # Minimum face width in pixels (default: 50)
|
||||
# MIN_FACE_WIDTH=90 # Minimum face width in pixels (default: 90, guarantees ≥8,100px crop)
|
||||
# FACE_MARGIN=0.15 # Padding around face crop as fraction (default: 0.15)
|
||||
# ENABLE_FACE_ALIGNMENT=true # Align face before cropping (default: true)
|
||||
# USE_FULL_RESOLUTION=true # Use full-res images vs thumbnails (default: true)
|
||||
# MIN_CONFIDENCE=0.7 # Minimum face detection confidence (default: 0.7)
|
||||
# BLUR_THRESHOLD=100.0 # Laplacian blur threshold; lower = accept more blur (default: 100.0)
|
||||
# BLUR_THRESHOLD=120.0 # Laplacian blur threshold; lower = accept more blur (default: 120.0)
|
||||
# MAX_AUTO_IMAGES=80 # Hard cap on auto-diversity selection (default: 80)
|
||||
# QUALITY_REPLACEMENT=true # At cap, replace a weaker tracked image with a better candidate (default: true)
|
||||
# FRIGATE_SCORE_CEILING=0.0 # Skip uploads already well-covered (pre-upload score > ceiling = redundant; 0 = disabled; requires at least one prior run)
|
||||
# ENABLE_FRIGATE_SCORES=true # Call Frigate's recognize endpoint pre-upload to store diversity scores (default: true; adds ~200ms per upload)
|
||||
|
||||
# ── Caching & Models ──────────────────────────────────────────────────────────
|
||||
# FORCE_CPU=true # Disable GPU, fall back to CPU
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Publish Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "dev"]
|
||||
branches: ["dev"]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -15,6 +15,16 @@ on:
|
||||
- "uv-cpu.lock"
|
||||
- "uv-rocm.lock"
|
||||
- "uv-intel.lock"
|
||||
workflow_call:
|
||||
inputs:
|
||||
tag:
|
||||
type: string
|
||||
required: false
|
||||
description: "Release tag, e.g. v0.4.1 — triggers :latest + versioned image tags"
|
||||
version:
|
||||
type: string
|
||||
required: false
|
||||
description: "Version string without v prefix, e.g. 0.4.1"
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.ref }}
|
||||
@@ -50,6 +60,8 @@ jobs:
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform == 'linux/arm64'
|
||||
@@ -65,6 +77,15 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute build version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.version }}" ]; then
|
||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
@@ -72,6 +93,7 @@ jobs:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: VERSION=${{ steps.version.outputs.value }}
|
||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -120,22 +142,26 @@ jobs:
|
||||
- name: Determine image tags
|
||||
id: tags
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev" >> "$GITHUB_OUTPUT"
|
||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||
INPUT_TAG="${{ inputs.tag }}"
|
||||
if [ -n "$INPUT_TAG" ]; then
|
||||
echo "tag_args=-t ${IMAGE}:latest -t ${IMAGE}:${INPUT_TAG}" >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "tag_args=-t ${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create and push multi-arch manifest
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ steps.tags.outputs.tags }} \
|
||||
${{ steps.tags.outputs.tag_args }} \
|
||||
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
||||
docker buildx imagetools inspect ${{ steps.tags.outputs.inspect_tag }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
@@ -162,6 +188,8 @@ jobs:
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
@@ -176,13 +204,30 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Determine CPU image tag
|
||||
id: cpu-tag
|
||||
- name: Determine CPU image tags
|
||||
id: cpu-tags
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||
INPUT_TAG="${{ inputs.tag }}"
|
||||
if [ -n "$INPUT_TAG" ]; then
|
||||
{
|
||||
echo "tags<<EOF"
|
||||
printf '%s\n' "${IMAGE}:cpu" "${IMAGE}:${INPUT_TAG}-cpu"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:cpu" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cpu" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compute build version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.version }}" ]; then
|
||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push CPU image
|
||||
@@ -191,16 +236,18 @@ jobs:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: VARIANT=cpu
|
||||
build-args: |
|
||||
VARIANT=cpu
|
||||
VERSION=${{ steps.version.outputs.value }}
|
||||
cache-from: type=gha,scope=cpu
|
||||
cache-to: type=gha,mode=max,scope=cpu
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
push: true
|
||||
tags: ${{ steps.cpu-tag.outputs.tag }}
|
||||
tags: ${{ steps.cpu-tags.outputs.tags }}
|
||||
|
||||
- name: Inspect CPU image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.cpu-tag.outputs.tag }}
|
||||
docker buildx imagetools inspect ${{ steps.cpu-tags.outputs.inspect_tag }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
@@ -227,6 +274,8 @@ jobs:
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
@@ -238,13 +287,30 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Determine ROCm image tag
|
||||
id: rocm-tag
|
||||
- name: Determine ROCm image tags
|
||||
id: rocm-tags
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||
INPUT_TAG="${{ inputs.tag }}"
|
||||
if [ -n "$INPUT_TAG" ]; then
|
||||
{
|
||||
echo "tags<<EOF"
|
||||
printf '%s\n' "${IMAGE}:rocm" "${IMAGE}:${INPUT_TAG}-rocm"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:rocm" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:rocm" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compute build version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.version }}" ]; then
|
||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push ROCm image
|
||||
@@ -253,16 +319,18 @@ jobs:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
build-args: VARIANT=rocm
|
||||
build-args: |
|
||||
VARIANT=rocm
|
||||
VERSION=${{ steps.version.outputs.value }}
|
||||
cache-from: type=gha,scope=linux/amd64-rocm
|
||||
cache-to: type=gha,mode=max,scope=linux/amd64-rocm
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
push: true
|
||||
tags: ${{ steps.rocm-tag.outputs.tag }}
|
||||
tags: ${{ steps.rocm-tags.outputs.tags }}
|
||||
|
||||
- name: Inspect ROCm image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.rocm-tag.outputs.tag }}
|
||||
docker buildx imagetools inspect ${{ steps.rocm-tags.outputs.inspect_tag }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
@@ -289,6 +357,8 @@ jobs:
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
@@ -300,13 +370,30 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Determine Intel image tag
|
||||
id: intel-tag
|
||||
- name: Determine Intel image tags
|
||||
id: intel-tags
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||
INPUT_TAG="${{ inputs.tag }}"
|
||||
if [ -n "$INPUT_TAG" ]; then
|
||||
{
|
||||
echo "tags<<EOF"
|
||||
printf '%s\n' "${IMAGE}:intel" "${IMAGE}:${INPUT_TAG}-intel"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:intel" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:intel" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||
echo "inspect_tag=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compute build version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.version }}" ]; then
|
||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push Intel image
|
||||
@@ -315,16 +402,18 @@ jobs:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
build-args: VARIANT=intel
|
||||
build-args: |
|
||||
VARIANT=intel
|
||||
VERSION=${{ steps.version.outputs.value }}
|
||||
cache-from: type=gha,scope=linux/amd64-intel
|
||||
cache-to: type=gha,mode=max,scope=linux/amd64-intel
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
push: true
|
||||
tags: ${{ steps.intel-tag.outputs.tag }}
|
||||
tags: ${{ steps.intel-tags.outputs.tags }}
|
||||
|
||||
- name: Inspect Intel image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.intel-tag.outputs.tag }}
|
||||
docker buildx imagetools inspect ${{ steps.intel-tags.outputs.inspect_tag }}
|
||||
|
||||
- name: Ensure package is public
|
||||
run: |
|
||||
|
||||
@@ -127,188 +127,14 @@ jobs:
|
||||
prerelease: false,
|
||||
});
|
||||
|
||||
build-gpu:
|
||||
name: Build GPU image
|
||||
build-images:
|
||||
name: Build and push Docker images
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
uses: ./.github/workflows/docker-publish.yml
|
||||
with:
|
||||
tag: ${{ needs.release.outputs.tag }}
|
||||
version: ${{ needs.release.outputs.version }}
|
||||
secrets: inherit
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push GPU image (latest)
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: VERSION=${{ needs.release.outputs.version }}
|
||||
cache-from: type=gha,scope=release-gpu
|
||||
cache-to: type=gha,mode=max,scope=release-gpu
|
||||
tags: |
|
||||
ghcr.io/sudolulo/winnow:latest
|
||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}
|
||||
|
||||
build-cpu:
|
||||
name: Build CPU image
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push CPU image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
VARIANT=cpu
|
||||
VERSION=${{ needs.release.outputs.version }}
|
||||
cache-from: type=gha,scope=release-cpu
|
||||
cache-to: type=gha,mode=max,scope=release-cpu
|
||||
tags: |
|
||||
ghcr.io/sudolulo/winnow:cpu
|
||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-cpu
|
||||
|
||||
build-rocm:
|
||||
name: Build ROCm image
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push ROCm image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
VARIANT=rocm
|
||||
VERSION=${{ needs.release.outputs.version }}
|
||||
cache-from: type=gha,scope=release-rocm
|
||||
cache-to: type=gha,mode=max,scope=release-rocm
|
||||
tags: |
|
||||
ghcr.io/sudolulo/winnow:rocm
|
||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-rocm
|
||||
|
||||
build-intel:
|
||||
name: Build Intel image
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
echo "Disk space freed."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push Intel image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
VARIANT=intel
|
||||
VERSION=${{ needs.release.outputs.version }}
|
||||
cache-from: type=gha,scope=release-intel
|
||||
cache-to: type=gha,mode=max,scope=release-intel
|
||||
tags: |
|
||||
ghcr.io/sudolulo/winnow:intel
|
||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-intel
|
||||
contents: read
|
||||
|
||||
@@ -7,6 +7,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.4.2] - 2026-06-13
|
||||
|
||||
### Changed
|
||||
|
||||
- **GPU image now uses CUDA 12.8.1** (was 13.3): CUDA 13.3 requires driver ≥ 575; driver 570 (the current stable release) was incorrectly rejected with "CUDA driver version is insufficient" at startup. The `:latest` image now works with any NVIDIA driver ≥ 570.
|
||||
|
||||
### Added
|
||||
|
||||
- **`scripts/benchmark.py`**: measures InsightFace and SigLIP inference latency and throughput across GPU and CPU modes. Run inside the container with `python /app/scripts/benchmark.py`. RTX 2070 SUPER results: InsightFace 12.8 ms / 78 img/s (8× CPU), SigLIP batch 32 at 5.4 ms/img / 187 img/s (33× CPU).
|
||||
|
||||
## [0.4.1] - 2026-06-13
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`RESET_PERSON` no longer creates duplicate Frigate files**: previously, resetting a person only wiped the local tracker — existing Frigate training files were left as unmanaged orphans, causing the next run to upload a full new batch on top of them. `reset_person` now deletes all winnow-managed files for that person from Frigate before clearing the tracker. Manually-added Frigate files are unaffected.
|
||||
- **No spurious warning when `FRIGATE_URL` is unset and `RESET_PERSON` is used**: the deletion step is now skipped silently at info level rather than logging a misleading "could not delete" warning.
|
||||
|
||||
## [0.4.0] - 2026-06-13
|
||||
|
||||
### Added
|
||||
|
||||
- **Pre-upload Frigate recognition scores**: `recognize_face` is now called before each upload to measure how novel the candidate is relative to the existing training set. The score is stored in the tracker (`frigate_scores` field) and drives quality replacement in subsequent runs. Adds ~200 ms per upload.
|
||||
- **`ENABLE_FRIGATE_SCORES`** (default `true`): controls all pre-upload Frigate recognize calls. Set `false` to use blur-score replacement only and skip the Frigate round-trip entirely.
|
||||
- **`FRIGATE_SCORE_CEILING`** (default `0.0`): skip uploads whose pre-upload recognize score already exceeds this value — those face conditions are already well-covered by the training set. `0` disables (no ceiling); requires at least one prior run to have stored scores.
|
||||
- **`get_most_redundant_mapped_file()`**: new upload-tracker function that returns the mapped file with the highest Frigate pre-upload score. High score = the training set already covers that face condition well = the best deletion target for quality replacement.
|
||||
- **Cold-start notice**: first run (no existing Frigate model) now logs a clear message explaining why Frigate scores are unavailable and that they will populate on subsequent runs.
|
||||
- **4 new tests** for `get_most_redundant_mapped_file` covering score ordering, ties, excludes, and no-score cases.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Quality replacement now uses Frigate scores**: when Frigate scores are available, at-cap replacement targets the _most redundant_ mapped file (highest pre-upload score) and replaces it only when the candidate is _more novel_ (lower score). Falls back to blur-score comparison when no Frigate scores have been stored yet.
|
||||
- **`recognize_face` returns `(face_name, score) | None`** instead of `float | None`: the caller now validates that the recognized person matches the expected person before using the score. Wrong-person scores no longer drive ceiling skips or replacement decisions.
|
||||
- **Bootstrap fix**: recognize was previously called below-cap only when `FRIGATE_SCORE_CEILING > 0`, so `frigate_scores` was never populated with default settings and the Frigate replacement path never activated. Recognize is now called for all below-cap uploads when `ENABLE_FRIGATE_SCORES=true`, seeding scores for future at-cap runs regardless of ceiling setting.
|
||||
- **Batch GET `/api/faces`**: Frigate file-count lookups are now batched to reduce round-trip overhead on runs with many people.
|
||||
- **Skip candidate download on low Frigate confidence**: candidates where the Immich detection confidence is below threshold are now filtered before the full-resolution download, saving bandwidth.
|
||||
|
||||
### Removed
|
||||
|
||||
- **Post-upload quality gate (`FRIGATE_SCORE_THRESHOLD`)**: enforcement of a Frigate score threshold after upload has been removed. Post-upload scores are taken after the image is already in the training set, so the model has already retrained on it — deleting it at that point is wasteful and disrupts the model for the next Frigate run. Pre-upload scoring (`FRIGATE_SCORE_CEILING`) provides a cleaner signal at the right moment.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Frigate replacement path never activated with default settings**: with `FRIGATE_SCORE_CEILING=0.0` (default), the bootstrap call to `recognize_face` was gated behind `CEILING > 0`, so `frigate_scores` stayed empty, `has_frigate_scores` was always False, and the Frigate replacement branch was permanently unreachable. Removing the ceiling guard from the below-cap recognize call breaks the circular dependency.
|
||||
- **Schema comment contradiction**: `upload_tracker.py` line-16 comment described `frigate_scores` as "post-upload" while the block comment on lines 22–24 said "pre-upload". Corrected to "pre-upload" throughout.
|
||||
- **README default values**: `MIN_FACE_WIDTH` was documented as `50` (actual default: `90`); `BLUR_THRESHOLD` was documented as `100.0` (actual default: `120.0`). Both corrected.
|
||||
- **README missing env vars**: `FRIGATE_SCORE_CEILING` and `ENABLE_FRIGATE_SCORES` were present in `config.py` and `.env.example` but absent from the README env var table. Both added.
|
||||
- **README quality-replacement description**: Step 8 and the `QUALITY_REPLACEMENT` row now document the dual-mode behaviour (Frigate-score path and blur-score fallback) instead of describing only the original blur-score path.
|
||||
|
||||
## [0.3.3] - 2026-06-13
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`MIN_FACE_WIDTH` default raised from 50 → 90px**: 50px crops produce 2,500–4,225 total pixels, well below Frigate's own camera capture range of 16k–50k px. 90px guarantees ≥8,100 total pixels even when face margins are fully clipped by image edges, keeping winnow training crops above the floor Frigate considers useful.
|
||||
|
||||
## [0.3.2] - 2026-06-13
|
||||
|
||||
### Added
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
# ── Base images ───────────────────────────────────────────────────────────────
|
||||
# amd64 + gpu: NVIDIA CUDA 13.3 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
||||
# amd64 + gpu: NVIDIA CUDA 12.8 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
||||
# amd64 + rocm: Ubuntu 22.04 (AMD GPU via ROCm — pass /dev/kfd and /dev/dri)
|
||||
# amd64 + intel: Ubuntu 22.04 (Intel Arc / iGPU via OpenVINO — pass /dev/dri)
|
||||
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
ARG VARIANT=gpu
|
||||
|
||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:13.3.0-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
||||
FROM ubuntu:22.04 AS base-amd64-rocm
|
||||
FROM ubuntu:22.04 AS base-amd64-intel
|
||||
FROM ubuntu:22.04 AS base-amd64-cpu
|
||||
|
||||
@@ -2,12 +2,17 @@
|
||||
|
||||
[](https://github.com/sudolulo/winnow/actions/workflows/docker-publish.yml) [](https://github.com/sudolulo/winnow/actions/workflows/test.yml) [](https://github.com/sudolulo/winnow/releases/latest) [](LICENSE) [](https://immich.app) [](https://frigate.video)
|
||||
|
||||
> **Early Development — Use With Caution**
|
||||
> winnow is functional but still maturing. Features that modify your Frigate training data — quality replacement, stale mapping cleanup — can remove images from your dataset and are not yet battle-tested at scale. Review the logs after each run and keep backups of your Frigate face training directory until you are confident in the results.
|
||||
|
||||
**Docs:** [Setup](https://github.com/sudolulo/winnow/wiki/Setup) · [Troubleshooting](https://github.com/sudolulo/winnow/wiki/Troubleshooting) · [FAQ](https://github.com/sudolulo/winnow/wiki/FAQ)
|
||||
|
||||
`winnow` pulls photos from your [Immich](https://immich.app) library, selects the most diverse and highest-quality subset using AI embeddings, and delivers them as training data for [Frigate](https://frigate.video)'s face recognition and object classification models.
|
||||
|
||||
Frigate's face recognition is only as good as its training data — and the key quality metric is **diversity**, not volume. A hundred photos from the same week teach the model one lighting condition. What you need is a spread: different years, different angles, different lighting, different contexts. Your photo library already has that data. winnow finds and delivers the right subset automatically.
|
||||
|
||||
> **winnow only touches files it uploaded.** Faces added to Frigate manually through its UI are never deleted, replaced, or modified — not by quality replacement, not by `RESET_PERSON`, not by stale cleanup. If you have a curated training set you want to keep, it is safe.
|
||||
|
||||
---
|
||||
|
||||
## How It Works
|
||||
@@ -56,9 +61,11 @@ Immich library
|
||||
8. Deliver
|
||||
• Face mode: upload crops to Frigate's face registration API
|
||||
↳ below MAX_AUTO_IMAGES — upload freely
|
||||
↳ at cap + QUALITY_REPLACEMENT=true — swap the lowest-scoring tracked
|
||||
image if the new candidate scores higher; manually added files are
|
||||
never touched
|
||||
↳ at cap + QUALITY_REPLACEMENT=true — with Frigate scoring active,
|
||||
swap the most redundant tracked image (highest pre-upload recognize
|
||||
score) if the candidate is more novel (lower score); falling back to
|
||||
blur-score comparison when no Frigate scores are available; manually
|
||||
added files are never touched
|
||||
↳ at cap + QUALITY_REPLACEMENT=false — skip this person
|
||||
• Object mode: save crops to disk → place into your Frigate data directory
|
||||
```
|
||||
@@ -183,14 +190,16 @@ In scheduled mode the process (and loaded models) stays resident between runs. T
|
||||
|
||||
| Variable | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `MIN_FACE_WIDTH` | `50` | Minimum face crop width in pixels |
|
||||
| `MIN_FACE_WIDTH` | `90` | Minimum face crop width in pixels |
|
||||
| `FACE_MARGIN` | `0.15` | Padding around bounding box crop (fraction of face size) |
|
||||
| `ENABLE_FACE_ALIGNMENT` | `true` | Align to ArcFace 112×112 format using facial landmarks |
|
||||
| `USE_FULL_RESOLUTION` | `true` | Download full-resolution originals rather than preview thumbnails |
|
||||
| `MIN_CONFIDENCE` | `0.7` | Minimum Immich face detection confidence |
|
||||
| `BLUR_THRESHOLD` | `100.0` | Laplacian variance threshold — lower accepts more blur |
|
||||
| `BLUR_THRESHOLD` | `120.0` | Laplacian variance threshold — lower accepts more blur |
|
||||
| `MAX_AUTO_IMAGES` | `80` | Maximum training images per person in Frigate |
|
||||
| `QUALITY_REPLACEMENT` | `true` | When at cap, swap the lowest-scoring tracked image for a better candidate. Never touches manually added Frigate files. Set `false` to skip people already at cap |
|
||||
| `QUALITY_REPLACEMENT` | `true` | When at cap, swap a weaker tracked image for a better candidate. With Frigate scoring active, targets the most redundant image (highest pre-upload recognize score); otherwise uses blur score. Never touches manually added Frigate files. Set `false` to skip people at cap |
|
||||
| `FRIGATE_SCORE_CEILING` | `0.0` | Skip uploads whose pre-upload Frigate recognize score exceeds this value — they are already well-covered. `0` disables; requires at least one prior run to have scores |
|
||||
| `ENABLE_FRIGATE_SCORES` | `true` | Call Frigate's recognize endpoint pre-upload to store diversity scores used for quality replacement. Adds ~200 ms per upload. Disable to use blur-score replacement only |
|
||||
|
||||
### GPU & Models
|
||||
|
||||
@@ -215,7 +224,7 @@ In scheduled mode the process (and loaded models) stays resident between runs. T
|
||||
| :--- | :--- | :--- |
|
||||
| `DRY_RUN` | `false` | Preview selection without downloading or uploading |
|
||||
| `RETRY_REJECTED` | `false` | Re-attempt assets previously rejected by Frigate |
|
||||
| `RESET_PERSON` | *(unset)* | Clear upload and rejection history for one person by name |
|
||||
| `RESET_PERSON` | *(unset)* | Clear upload history for one person and delete their winnow-managed Frigate training files so the next run starts fresh. Manually added Frigate files are never touched |
|
||||
|
||||
### Scheduling
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "winnow"
|
||||
version = "0.3.2"
|
||||
version = "0.4.2"
|
||||
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
|
||||
license = "AGPL-3.0-or-later"
|
||||
requires-python = ">=3.13"
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
winnow inference benchmark: GPU vs CPU throughput.
|
||||
|
||||
Measures InsightFace (face mode) and SigLIP (object mode) latency and
|
||||
throughput. Run with FORCE_CPU=true for CPU-only baseline.
|
||||
|
||||
Usage inside container:
|
||||
# GPU mode:
|
||||
docker exec winnow python /app/scripts/benchmark.py
|
||||
|
||||
# CPU mode:
|
||||
docker exec -e FORCE_CPU=true winnow python /app/scripts/benchmark.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def _mode_label() -> str:
|
||||
if os.getenv("FORCE_CPU", "").lower() in ("true", "1", "yes"):
|
||||
return "CPU (FORCE_CPU=true)"
|
||||
return "GPU (auto)"
|
||||
|
||||
|
||||
def make_face_image(size: int = 640) -> Image.Image:
|
||||
"""Synthetic face-like image: skin-tone rectangle with landmark blobs."""
|
||||
img = Image.new("RGB", (size, size), (200, 170, 140))
|
||||
draw = ImageDraw.Draw(img)
|
||||
# Head oval
|
||||
cx, cy = size // 2, size // 2
|
||||
hw, hh = int(size * 0.3), int(size * 0.38)
|
||||
draw.ellipse([cx - hw, cy - hh, cx + hw, cy + hh], fill=(220, 185, 155))
|
||||
# Eyes
|
||||
for ex in [cx - int(size * 0.1), cx + int(size * 0.1)]:
|
||||
ey = cy - int(size * 0.05)
|
||||
r = max(4, size // 40)
|
||||
draw.ellipse([ex - r, ey - r, ex + r, ey + r], fill=(40, 30, 20))
|
||||
# Nose
|
||||
draw.ellipse([cx - 5, cy + 5, cx + 5, cy + 15], fill=(180, 140, 110))
|
||||
# Mouth
|
||||
draw.arc([cx - 20, cy + 25, cx + 20, cy + 45], start=0, end=180, fill=(160, 80, 80), width=3)
|
||||
return img
|
||||
|
||||
|
||||
def make_random_image(width: int = 224, height: int = 224) -> Image.Image:
|
||||
rng = np.random.default_rng(42)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8), "RGB")
|
||||
|
||||
|
||||
def _stats(times_s: list[float]) -> dict:
|
||||
arr = np.array(times_s) * 1000 # ms
|
||||
return {
|
||||
"median_ms": float(np.median(arr)),
|
||||
"mean_ms": float(np.mean(arr)),
|
||||
"min_ms": float(np.min(arr)),
|
||||
"p95_ms": float(np.percentile(arr, 95)),
|
||||
"ips": 1000.0 / float(np.median(arr)),
|
||||
}
|
||||
|
||||
|
||||
def bench_insightface(n_warmup: int = 5, n_runs: int = 30) -> None:
|
||||
import cv2
|
||||
|
||||
import winnow.embeddings as emb_mod
|
||||
from winnow.embeddings import get_insightface_app
|
||||
|
||||
# Reset singleton so we get a fresh load
|
||||
emb_mod._insightface_app = None
|
||||
emb_mod._insightface_loaded = False
|
||||
|
||||
print(" Loading model...")
|
||||
t_load = time.perf_counter()
|
||||
app = get_insightface_app()
|
||||
load_s = time.perf_counter() - t_load
|
||||
|
||||
if app is None:
|
||||
print(" SKIP: InsightFace failed to load")
|
||||
return
|
||||
|
||||
img_pil = make_face_image(640)
|
||||
img_bgr = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
|
||||
|
||||
# Warmup
|
||||
for _ in range(n_warmup):
|
||||
app.get(img_bgr)
|
||||
|
||||
# Timed — single image 640×640
|
||||
times: list[float] = []
|
||||
for _ in range(n_runs):
|
||||
t0 = time.perf_counter()
|
||||
app.get(img_bgr)
|
||||
times.append(time.perf_counter() - t0)
|
||||
|
||||
s = _stats(times)
|
||||
print(f" Model load time : {load_s:.2f} s")
|
||||
print(" Input size : 640×640")
|
||||
print(f" Runs : {n_runs} (after {n_warmup} warmup)")
|
||||
print(f" Median latency : {s['median_ms']:.1f} ms")
|
||||
print(f" Mean / p95 : {s['mean_ms']:.1f} ms / {s['p95_ms']:.1f} ms")
|
||||
print(f" Min latency : {s['min_ms']:.1f} ms")
|
||||
print(f" Throughput : {s['ips']:.1f} images/s")
|
||||
|
||||
# Also test at 320×320
|
||||
img_sm = make_face_image(320)
|
||||
img_sm_bgr = cv2.cvtColor(np.asarray(img_sm), cv2.COLOR_RGB2BGR)
|
||||
for _ in range(n_warmup):
|
||||
app.get(img_sm_bgr)
|
||||
times_sm: list[float] = []
|
||||
for _ in range(n_runs):
|
||||
t0 = time.perf_counter()
|
||||
app.get(img_sm_bgr)
|
||||
times_sm.append(time.perf_counter() - t0)
|
||||
s2 = _stats(times_sm)
|
||||
print(f" 320×320 median : {s2['median_ms']:.1f} ms ({s2['ips']:.1f} img/s)")
|
||||
|
||||
|
||||
def bench_siglip(
|
||||
n_warmup: int = 3,
|
||||
n_runs: int = 20,
|
||||
batch_sizes: tuple = (1, 4, 8, 16, 32),
|
||||
) -> None:
|
||||
import torch
|
||||
|
||||
import winnow.embeddings as emb_mod
|
||||
emb_mod._siglip_model = None
|
||||
emb_mod._siglip_processor = None
|
||||
emb_mod._siglip_loaded = False
|
||||
|
||||
print(" Loading model...")
|
||||
t_load = time.perf_counter()
|
||||
model, processor = emb_mod.get_siglip_model()
|
||||
load_s = time.perf_counter() - t_load
|
||||
|
||||
if model is None:
|
||||
print(" SKIP: SigLIP failed to load")
|
||||
return
|
||||
|
||||
device = next(model.parameters()).device
|
||||
print(f" Model load time : {load_s:.2f} s (device: {device})")
|
||||
|
||||
print(f" {'Batch':>5} {'ms/batch':>10} {'ms/img':>8} {'img/s':>8} {'p95/img':>9}")
|
||||
for bs in batch_sizes:
|
||||
imgs = [make_random_image(224, 224) for _ in range(bs)]
|
||||
inputs = processor(images=imgs, return_tensors="pt")
|
||||
inputs = {k: v.to(device) for k, v in inputs.items()}
|
||||
|
||||
# Warmup
|
||||
for _ in range(n_warmup):
|
||||
with torch.no_grad():
|
||||
model(**inputs)
|
||||
if str(device) != "cpu":
|
||||
torch.cuda.synchronize()
|
||||
|
||||
times: list[float] = []
|
||||
for _ in range(n_runs):
|
||||
if str(device) != "cpu":
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
with torch.no_grad():
|
||||
model(**inputs)
|
||||
if str(device) != "cpu":
|
||||
torch.cuda.synchronize()
|
||||
times.append(time.perf_counter() - t0)
|
||||
|
||||
s = _stats(times)
|
||||
print(
|
||||
f" {bs:>5} {s['median_ms']:>10.1f} {s['median_ms']/bs:>8.2f}"
|
||||
f" {bs * 1000 / s['median_ms']:>8.1f} {s['p95_ms']/bs:>9.2f}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("=" * 56)
|
||||
print(" winnow inference benchmark")
|
||||
print(f" Mode: {_mode_label()}")
|
||||
print("=" * 56)
|
||||
print()
|
||||
|
||||
print("── InsightFace Buffalo_L (face detection + ArcFace) ──")
|
||||
bench_insightface()
|
||||
print()
|
||||
|
||||
print("── SigLIP google/siglip-base-patch16-224 (objects) ───")
|
||||
bench_siglip()
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Add winnow to path when run directly inside container
|
||||
sys.path.insert(0, "/app")
|
||||
main()
|
||||
@@ -17,9 +17,9 @@ def test_config_loads_defaults(monkeypatch):
|
||||
assert cfg.API_KEY == "test-key"
|
||||
assert cfg.OUTPUT_DIR == "./frigate_train"
|
||||
assert cfg.YEARS_FILTER == 10
|
||||
assert cfg.MIN_FACE_WIDTH == 50
|
||||
assert cfg.MIN_FACE_WIDTH == 90
|
||||
assert cfg.MIN_FACE_COUNT == 0
|
||||
assert cfg.BLUR_THRESHOLD == 100.0
|
||||
assert cfg.BLUR_THRESHOLD == 120.0
|
||||
assert cfg.MIN_CONFIDENCE == 0.7
|
||||
assert cfg.MAX_AUTO_IMAGES == 80
|
||||
assert cfg.QUALITY_REPLACEMENT is True
|
||||
|
||||
@@ -218,3 +218,45 @@ def test_get_lowest_quality_exclude_all_returns_none():
|
||||
mark_uploaded("asset-a", person_name="Alice", score=0.50)
|
||||
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
||||
assert get_lowest_quality_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
|
||||
|
||||
|
||||
# ── get_most_redundant_mapped_file ────────────────────────────────────────────
|
||||
|
||||
def test_get_most_redundant_none_when_no_frigate_scores():
|
||||
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
||||
mark_uploaded("asset-a", person_name="Alice", score=0.80)
|
||||
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
||||
# blur score only, no frigate_score → no candidates
|
||||
assert get_most_redundant_mapped_file("Alice") is None
|
||||
|
||||
|
||||
def test_get_most_redundant_returns_highest_frigate_score():
|
||||
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
||||
mark_uploaded("asset-novel", person_name="Alice", score=0.50, frigate_score=0.31)
|
||||
mark_uploaded("asset-redundant", person_name="Alice", score=0.90, frigate_score=0.88)
|
||||
record_frigate_file("Alice", "Alice-novel.webp", "asset-novel")
|
||||
record_frigate_file("Alice", "Alice-redundant.webp", "asset-redundant")
|
||||
result = get_most_redundant_mapped_file("Alice")
|
||||
assert result is not None
|
||||
frigate_filename, asset_id, score = result
|
||||
assert frigate_filename == "Alice-redundant.webp"
|
||||
assert asset_id == "asset-redundant"
|
||||
assert score == pytest.approx(0.88, abs=0.001)
|
||||
|
||||
|
||||
def test_get_most_redundant_exclude_skips_file():
|
||||
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
||||
mark_uploaded("asset-hi", person_name="Alice", score=0.9, frigate_score=0.85)
|
||||
mark_uploaded("asset-lo", person_name="Alice", score=0.5, frigate_score=0.40)
|
||||
record_frigate_file("Alice", "Alice-hi.webp", "asset-hi")
|
||||
record_frigate_file("Alice", "Alice-lo.webp", "asset-lo")
|
||||
result = get_most_redundant_mapped_file("Alice", exclude={"Alice-hi.webp"})
|
||||
assert result is not None
|
||||
assert result[1] == "asset-lo" # hi excluded; lo is next highest
|
||||
|
||||
|
||||
def test_get_most_redundant_exclude_all_returns_none():
|
||||
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
||||
mark_uploaded("asset-a", person_name="Alice", score=0.5, frigate_score=0.70)
|
||||
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
||||
assert get_most_redundant_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
|
||||
|
||||
@@ -1970,15 +1970,16 @@ version = "2.12.0"
|
||||
source = { registry = "https://download.pytorch.org/whl/cpu" }
|
||||
resolution-markers = [
|
||||
"platform_machine != 's390x' and sys_platform == 'darwin'",
|
||||
"platform_machine == 's390x' and sys_platform == 'darwin'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "filelock", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "fsspec", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "jinja2", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "networkx", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "setuptools", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "sympy", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "typing-extensions", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "filelock", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "fsspec", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "jinja2", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "networkx", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "setuptools", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "sympy", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:90dd587a5f61bfe1307148b581e2084fc5bc4a06e2b90a20e9a36b81087ff16b", upload-time = "2026-05-12T16:20:17Z" },
|
||||
@@ -2041,16 +2042,15 @@ resolution-markers = [
|
||||
"platform_machine == 's390x' and sys_platform == 'win32'",
|
||||
"platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"platform_machine == 's390x' and sys_platform == 'darwin'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "filelock", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "fsspec", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "jinja2", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "networkx", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "setuptools", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "sympy", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "typing-extensions", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "filelock", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "fsspec", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "jinja2", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "networkx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "setuptools", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "sympy", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "typing-extensions", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:5e0da19e1c3bfdc9b92638c552579eac678354485d61fc8921b0461fd6c40449", upload-time = "2026-05-12T23:17:05Z" },
|
||||
@@ -2116,11 +2116,12 @@ version = "0.27.0"
|
||||
source = { registry = "https://download.pytorch.org/whl/cpu" }
|
||||
resolution-markers = [
|
||||
"platform_machine != 's390x' and sys_platform == 'darwin'",
|
||||
"platform_machine == 's390x' and sys_platform == 'darwin'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "pillow", marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "numpy", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "pillow", marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.27.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:41d6dae73e1af09fa82ded597ae57f2a2314285acde54b25890a8f8e51b999d7", upload-time = "2026-05-12T16:20:37Z" },
|
||||
@@ -2171,12 +2172,11 @@ resolution-markers = [
|
||||
"platform_machine == 's390x' and sys_platform == 'win32'",
|
||||
"platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"platform_machine == 's390x' and sys_platform == 'darwin'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "pillow", marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "numpy", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "pillow", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.27.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:69093b64b2762c43df17be2db2be163029963d90bc3f1801500fdeb723e54833", upload-time = "2026-05-12T16:20:36Z" },
|
||||
@@ -2306,13 +2306,13 @@ dependencies = [
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
{ name = "scipy" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (platform_machine == 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (platform_machine == 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "ultralytics-thop" },
|
||||
]
|
||||
@@ -2327,9 +2327,9 @@ version = "2.0.20"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "numpy" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (platform_machine == 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/98/c6/d25cb53e141242f74950744179c21b8798bb09b9e7161465ecda4f577ddf/ultralytics_thop-2.0.20.tar.gz", hash = "sha256:f3595e0d8c6fd0b9f62fc2cd9be921755e2649a05c34f1fabaea0bff7295d641", size = 34682, upload-time = "2026-06-06T11:42:42.184Z" }
|
||||
@@ -2348,7 +2348,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.3.2"
|
||||
version = "0.4.1"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "croniter" },
|
||||
@@ -2363,13 +2363,13 @@ dependencies = [
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "requests" },
|
||||
{ name = "rich" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (platform_machine == 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torch", version = "2.12.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 's390x' and sys_platform == 'darwin') or (platform_machine == 's390x' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin' or (extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (platform_machine == 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux') or (sys_platform == 'darwin' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform == 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "torchvision", version = "0.27.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu') or (sys_platform != 'linux' and extra == 'project-11-onnxruntime' and extra == 'project-15-onnxruntime-gpu')" },
|
||||
{ name = "transformers" },
|
||||
{ name = "ultralytics" },
|
||||
|
||||
@@ -41,6 +41,8 @@ def _handle_trace_crop(size_str: str) -> None:
|
||||
rprint(f" Immich URL: {immich_url}/photos/{m['asset_id']}")
|
||||
blur = m.get("blur_score")
|
||||
rprint(f" Blur score: {blur:.1f}" if blur is not None else " Blur score: unknown")
|
||||
fscore = m.get("frigate_score")
|
||||
rprint(f" Frigate score: {fscore:.2f}" if fscore is not None else " Frigate score: unknown")
|
||||
if m.get("frigate_filename"):
|
||||
rprint(f" Frigate file: {m['frigate_filename']}")
|
||||
else:
|
||||
|
||||
+8
-4
@@ -26,11 +26,13 @@ class _Config:
|
||||
YEARS_FILTER: int = 10
|
||||
|
||||
# Quality filtering
|
||||
MIN_FACE_WIDTH: int = 50
|
||||
BLUR_THRESHOLD: float = 100.0
|
||||
MIN_FACE_WIDTH: int = 90
|
||||
BLUR_THRESHOLD: float = 120.0
|
||||
MIN_CONFIDENCE: float = 0.7
|
||||
MAX_AUTO_IMAGES: int = 80
|
||||
QUALITY_REPLACEMENT: bool = True
|
||||
FRIGATE_SCORE_CEILING: float = 0.0
|
||||
ENABLE_FRIGATE_SCORES: bool = True
|
||||
|
||||
# People filtering
|
||||
MIN_FACE_COUNT: int = 0
|
||||
@@ -56,12 +58,14 @@ class _Config:
|
||||
self.API_KEY = os.getenv("API_KEY")
|
||||
self.OUTPUT_DIR = os.getenv("OUTPUT_DIR", "./frigate_train")
|
||||
self.YEARS_FILTER = int(os.getenv("YEARS_FILTER", "10"))
|
||||
self.MIN_FACE_WIDTH = int(os.getenv("MIN_FACE_WIDTH", "50"))
|
||||
self.MIN_FACE_WIDTH = int(os.getenv("MIN_FACE_WIDTH", "90"))
|
||||
self.MIN_FACE_COUNT = int(os.getenv("MIN_FACE_COUNT", "0"))
|
||||
self.BLUR_THRESHOLD = float(os.getenv("BLUR_THRESHOLD", "100.0"))
|
||||
self.BLUR_THRESHOLD = float(os.getenv("BLUR_THRESHOLD", "120.0"))
|
||||
self.MIN_CONFIDENCE = float(os.getenv("MIN_CONFIDENCE", "0.7"))
|
||||
self.MAX_AUTO_IMAGES = int(os.getenv("MAX_AUTO_IMAGES", "80"))
|
||||
self.QUALITY_REPLACEMENT = os.getenv("QUALITY_REPLACEMENT", "true").lower() in ("true", "1", "yes")
|
||||
self.FRIGATE_SCORE_CEILING = float(os.getenv("FRIGATE_SCORE_CEILING", "0.0"))
|
||||
self.ENABLE_FRIGATE_SCORES = os.getenv("ENABLE_FRIGATE_SCORES", "true").lower() in ("true", "1", "yes")
|
||||
self.FACE_MARGIN = float(os.getenv("FACE_MARGIN", "0.15"))
|
||||
self.USE_FULL_RESOLUTION = os.getenv("USE_FULL_RESOLUTION", "true").lower() in ("true", "1", "yes")
|
||||
self.ENABLE_FACE_ALIGNMENT = os.getenv("ENABLE_FACE_ALIGNMENT", "true").lower() in ("true", "1", "yes")
|
||||
|
||||
+142
-28
@@ -13,15 +13,22 @@ from rich import print as rprint
|
||||
from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
|
||||
|
||||
from .config import Config, get_headers
|
||||
from .frigate_api import delete_frigate_person_files, get_frigate_person_files
|
||||
from .frigate_api import (
|
||||
delete_frigate_person_files,
|
||||
get_all_frigate_person_files,
|
||||
get_frigate_person_files,
|
||||
recognize_face,
|
||||
)
|
||||
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
||||
from .immich_api import fetch_face_data, fetch_full_image
|
||||
from .log_config import console
|
||||
from .quality import assess_quality
|
||||
from .upload_tracker import (
|
||||
get_lowest_quality_mapped_file,
|
||||
get_most_redundant_mapped_file,
|
||||
get_tracked_frigate_file_count,
|
||||
get_tracked_frigate_filenames,
|
||||
has_frigate_scores,
|
||||
mark_rejected,
|
||||
mark_uploaded,
|
||||
record_frigate_file,
|
||||
@@ -182,6 +189,17 @@ def execute_jobs(jobs: list[dict]) -> None:
|
||||
# from the Immich faces API (not included in search/metadata results)
|
||||
if mode == "face":
|
||||
asset = _enrich_asset_with_face_data(asset, person)
|
||||
# Skip download if detection confidence already disqualifies
|
||||
# the asset — avoids fetching a large image we'll discard.
|
||||
conf = asset.get("face_confidence")
|
||||
if conf is not None and conf < Config.MIN_CONFIDENCE:
|
||||
progress.console.print(
|
||||
f"[yellow]Skipped {asset['id']}"
|
||||
f" (detection confidence {conf:.2f} < {Config.MIN_CONFIDENCE})[/yellow]"
|
||||
)
|
||||
progress.advance(job_task)
|
||||
progress.advance(overall_task)
|
||||
continue
|
||||
|
||||
# Use full-resolution for final output when configured
|
||||
if use_full_res:
|
||||
@@ -305,6 +323,10 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
||||
uploaded, failed = 0, 0
|
||||
max_retries = 2
|
||||
|
||||
# Fetch all Frigate training files once — avoids one GET /api/faces per person.
|
||||
# Falls back to per-person calls inside the loop if this fetch fails.
|
||||
all_frigate_files = get_all_frigate_person_files()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
@@ -342,7 +364,10 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
||||
# Snapshot live Frigate files for post-upload reconciliation diff only.
|
||||
# effective_count is sourced from the tracker (mapped files) so that
|
||||
# manually-added Frigate files don't consume winnow's managed quota.
|
||||
_snapshot = get_frigate_person_files(name)
|
||||
_snapshot = (
|
||||
all_frigate_files.get(name, []) if all_frigate_files is not None
|
||||
else get_frigate_person_files(name)
|
||||
)
|
||||
if _snapshot is None:
|
||||
# Frigate GET is down; fall back to the tracker's mapped filenames
|
||||
# as the pre-upload baseline. reconciliation will still work unless
|
||||
@@ -354,11 +379,28 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
||||
known_frigate_files_at_start: set[str] = get_tracked_frigate_filenames(name)
|
||||
else:
|
||||
known_frigate_files_at_start: set[str] = set(_snapshot)
|
||||
# Remove tracker mappings for files that no longer exist in Frigate
|
||||
# (manually deleted, or cleaned up outside winnow). This corrects the
|
||||
# effective_count so those slots are available for new uploads.
|
||||
stale = get_tracked_frigate_filenames(name) - known_frigate_files_at_start
|
||||
for stale_fn in stale:
|
||||
remove_frigate_file(name, stale_fn)
|
||||
if stale:
|
||||
progress.console.print(
|
||||
f" [dim]{name}: cleared {len(stale)} stale mapping(s)"
|
||||
" (file(s) no longer in Frigate)[/dim]"
|
||||
)
|
||||
effective_count = get_tracked_frigate_file_count(name)
|
||||
pre_run_count = effective_count
|
||||
quality_replacement = job.get("config", {}).get("quality_replacement", False)
|
||||
if Config.ENABLE_FRIGATE_SCORES and pre_run_count == 0:
|
||||
progress.console.print(
|
||||
f" [dim]{name}: first run — Frigate diversity scoring will apply from the next run[/dim]"
|
||||
)
|
||||
actually_uploaded: list[tuple[str, str | None]] = []
|
||||
failed_deletes: set[str] = set()
|
||||
min_quality_score_for_slot: float | None = None
|
||||
person_has_fscores: bool = has_frigate_scores(name)
|
||||
|
||||
for fname in person_files:
|
||||
fpath = os.path.join(person_dir, fname)
|
||||
@@ -378,40 +420,109 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
||||
continue
|
||||
|
||||
at_cap = effective_count >= Config.MAX_AUTO_IMAGES
|
||||
|
||||
# Pre-upload Frigate score — clean measurement (image not yet in training set).
|
||||
# Called for all below-cap uploads (seeds frigate_scores for future at-cap
|
||||
# replacement) and for at-cap uploads when scores already exist. Skipped on
|
||||
# the first run (pre_run_count == 0) since Frigate has no model yet.
|
||||
# recognize_face returns (face_name, score); we only use the score when the
|
||||
# best match is for the correct person. Mismatches (or "unknown") are treated
|
||||
# as None so a wrong-person score never drives a ceiling skip or replacement.
|
||||
# Frigate rebuilds its model asynchronously after any delete (clear + background
|
||||
# thread), so the first recognize call after a deletion returns None — our code
|
||||
# handles this conservatively by skipping that candidate until the next run.
|
||||
pre_fscore: float | None = None
|
||||
if Config.ENABLE_FRIGATE_SCORES and pre_run_count > 0:
|
||||
if not at_cap or person_has_fscores:
|
||||
_result = recognize_face(fpath)
|
||||
if _result is not None and (_result[0] or "").casefold() == name.casefold():
|
||||
pre_fscore = _result[1]
|
||||
|
||||
# Ceiling check: skip if the existing training set already covers this
|
||||
# face condition well. Applies below cap only — at cap, replacement logic
|
||||
# drives the decision.
|
||||
if not at_cap and Config.FRIGATE_SCORE_CEILING > 0 and pre_run_count > 0:
|
||||
if pre_fscore is not None and pre_fscore > Config.FRIGATE_SCORE_CEILING:
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: Frigate score {pre_fscore:.2f}"
|
||||
f" > ceiling {Config.FRIGATE_SCORE_CEILING:.2f}, already covered[/dim]"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
|
||||
if at_cap:
|
||||
if not quality_replacement:
|
||||
progress.console.print(f" [dim]⏭ {fname}: at cap, quality replacement disabled[/dim]")
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
new_score = score_map.get(fname)
|
||||
if new_score is None:
|
||||
progress.console.print(f" [dim]⏭ {fname}: no confidence score, skipping replacement[/dim]")
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
worst = get_lowest_quality_mapped_file(name, exclude=failed_deletes)
|
||||
if worst is None or new_score <= worst[2]:
|
||||
worst_score_str = f"{worst[2]:.3f}" if worst is not None else "N/A"
|
||||
|
||||
using_fscore = person_has_fscores and Config.ENABLE_FRIGATE_SCORES
|
||||
if using_fscore:
|
||||
candidate_score = pre_fscore
|
||||
if candidate_score is None:
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: Frigate recognize unavailable, skipping replacement[/dim]"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
# Low score = more novel than the most redundant mapped file = replace
|
||||
target = get_most_redundant_mapped_file(name, exclude=failed_deletes)
|
||||
if target is None or candidate_score >= target[2]:
|
||||
target_score_str = f"{target[2]:.3f}" if target is not None else "N/A"
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: frigate {candidate_score:.3f} ≥ most redundant"
|
||||
f" {target_score_str}, not more novel[/dim]"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
target_frigate_file, _target_asset_id, target_score = target
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: score {new_score:.3f} ≤ worst mapped"
|
||||
f" {worst_score_str}, skipping[/dim]"
|
||||
f" 🔄 {fname}: frigate {candidate_score:.3f} < {target_score:.3f},"
|
||||
f" replacing {target_frigate_file} (more novel)"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
# Delete the worst mapped file to make room for the better one
|
||||
worst_frigate_file, _worst_asset_id, worst_score = worst
|
||||
progress.console.print(
|
||||
f" 🔄 {fname}: score {new_score:.3f} > {worst_score:.3f},"
|
||||
f" replacing {worst_frigate_file}"
|
||||
)
|
||||
if delete_frigate_person_files(name, [worst_frigate_file]):
|
||||
remove_frigate_file(name, worst_frigate_file)
|
||||
effective_count -= 1
|
||||
min_quality_score_for_slot = worst_score
|
||||
if delete_frigate_person_files(name, [target_frigate_file]):
|
||||
remove_frigate_file(name, target_frigate_file)
|
||||
person_has_fscores = has_frigate_scores(name)
|
||||
effective_count -= 1
|
||||
# clear any blur-mode slot floor — Frigate uses a different score metric
|
||||
min_quality_score_for_slot = None
|
||||
else:
|
||||
logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement")
|
||||
failed_deletes.add(target_frigate_file)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
else:
|
||||
logger.warning(f"Failed to delete {worst_frigate_file} for {name}, skipping replacement")
|
||||
failed_deletes.add(worst_frigate_file)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
candidate_score = score_map.get(fname)
|
||||
if candidate_score is None:
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: no quality score, skipping replacement[/dim]"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
target = get_lowest_quality_mapped_file(name, exclude=failed_deletes)
|
||||
if target is None or candidate_score <= target[2]:
|
||||
target_score_str = f"{target[2]:.3f}" if target is not None else "N/A"
|
||||
progress.console.print(
|
||||
f" [dim]⏭ {fname}: blur {candidate_score:.3f} ≤ worst"
|
||||
f" {target_score_str}, skipping[/dim]"
|
||||
)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
target_frigate_file, _target_asset_id, target_score = target
|
||||
progress.console.print(
|
||||
f" 🔄 {fname}: blur {candidate_score:.3f} > {target_score:.3f},"
|
||||
f" replacing {target_frigate_file}"
|
||||
)
|
||||
if delete_frigate_person_files(name, [target_frigate_file]):
|
||||
remove_frigate_file(name, target_frigate_file)
|
||||
person_has_fscores = has_frigate_scores(name)
|
||||
effective_count -= 1
|
||||
min_quality_score_for_slot = score_map.get(fname)
|
||||
else:
|
||||
logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement")
|
||||
failed_deletes.add(target_frigate_file)
|
||||
progress.advance(upload_task)
|
||||
continue
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
@@ -434,7 +545,10 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
||||
person_name=name,
|
||||
score=score_map.get(fname),
|
||||
crop_dims=dims_map.get(fname),
|
||||
frigate_score=pre_fscore,
|
||||
)
|
||||
if pre_fscore is not None:
|
||||
person_has_fscores = True
|
||||
actually_uploaded.append((fname, asset_id))
|
||||
|
||||
break
|
||||
|
||||
+49
-5
@@ -22,11 +22,11 @@ def _get_faces_data() -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
def get_frigate_face_counts() -> dict[str, int] | None:
|
||||
"""Return {person_name: training_image_count} from Frigate's train directory.
|
||||
def get_all_frigate_person_files() -> dict[str, list[str]] | None:
|
||||
"""Return {person_name: [filename, ...]} for every person in Frigate.
|
||||
|
||||
Returns None if FRIGATE_URL is not set or the API is unreachable, so callers
|
||||
can distinguish "API unavailable" from "person has 0 images."
|
||||
Single call used to build per-person snapshots before the upload loop,
|
||||
avoiding one GET /api/faces per person. Returns None if unavailable.
|
||||
"""
|
||||
data = _get_faces_data()
|
||||
if data is None:
|
||||
@@ -34,12 +34,24 @@ def get_frigate_face_counts() -> dict[str, int] | None:
|
||||
# Response: {person_name: [file, ...], "train": [...], ...}
|
||||
# "train" is a flat pending list, not a person — skip it.
|
||||
return {
|
||||
name: len(files)
|
||||
name: files
|
||||
for name, files in data.items()
|
||||
if name != "train" and isinstance(files, list)
|
||||
}
|
||||
|
||||
|
||||
def get_frigate_face_counts() -> dict[str, int] | None:
|
||||
"""Return {person_name: training_image_count} from Frigate's train directory.
|
||||
|
||||
Returns None if FRIGATE_URL is not set or the API is unreachable, so callers
|
||||
can distinguish "API unavailable" from "person has 0 images."
|
||||
"""
|
||||
all_files = get_all_frigate_person_files()
|
||||
if all_files is None:
|
||||
return None
|
||||
return {name: len(files) for name, files in all_files.items()}
|
||||
|
||||
|
||||
def get_frigate_person_files(person_name: str) -> list[str] | None:
|
||||
"""Return the list of training filenames for a person in Frigate.
|
||||
|
||||
@@ -53,6 +65,38 @@ def get_frigate_person_files(person_name: str) -> list[str] | None:
|
||||
return files if isinstance(files, list) else []
|
||||
|
||||
|
||||
def recognize_face(file_path: str) -> tuple[str | None, float] | None:
|
||||
"""Submit an image to Frigate's recognize endpoint.
|
||||
|
||||
Returns (face_name, score) where face_name is the best-matching person
|
||||
(may be "unknown" if below Frigate's confidence threshold) and score is
|
||||
the sigmoid-mapped cosine similarity (0-1) against that person's mean
|
||||
embedding.
|
||||
|
||||
Returns None if FRIGATE_URL is unset, the API is unreachable, no face is
|
||||
detected, or face recognition is not enabled in Frigate.
|
||||
"""
|
||||
frigate_url = os.environ.get("FRIGATE_URL", "").rstrip("/")
|
||||
if not frigate_url:
|
||||
return None
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
resp = requests.post(
|
||||
f"{frigate_url}/api/faces/recognize",
|
||||
files={"file": (os.path.basename(file_path), f, "image/jpeg")},
|
||||
timeout=15,
|
||||
)
|
||||
if not resp.ok:
|
||||
return None
|
||||
data = resp.json()
|
||||
if data.get("success") and "score" in data:
|
||||
return (data.get("face_name"), round(float(data["score"]), 4))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Frigate recognize failed for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def delete_frigate_person_files(person_name: str, filenames: list[str]) -> bool:
|
||||
"""Delete specific training files for a person from Frigate.
|
||||
|
||||
|
||||
+105
-28
@@ -11,21 +11,29 @@ Both are excluded from future candidate pools. To reset:
|
||||
|
||||
by_person schema (frigate_uploaded_ids.json):
|
||||
{
|
||||
"asset_ids": ["immich-id-1", ...], # all assets we attempted to upload
|
||||
"scores": {"immich-id-1": 450.3}, # Laplacian blur variance at upload time
|
||||
"frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID
|
||||
"crop_dims": {"immich-id-1": [640, 480]}, # crop pixel dimensions at upload time
|
||||
"frigate_count": 42 # last known Frigate training image count
|
||||
"asset_ids": ["immich-id-1", ...], # all assets we attempted to upload
|
||||
"scores": {"immich-id-1": 450.3}, # Laplacian blur variance at upload time
|
||||
"frigate_scores": {"immich-id-1": 0.87}, # Frigate recognition confidence (0-1) pre-upload
|
||||
"frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID
|
||||
"crop_dims": {"immich-id-1": [640, 480]}, # crop pixel dimensions at upload time
|
||||
"frigate_count": 42 # last known Frigate training image count
|
||||
}
|
||||
|
||||
frigate_scores stores pre-upload recognize scores (0-1 sigmoid-mapped cosine
|
||||
similarity). High score = the existing training set already covers this face
|
||||
condition well. Low score = a gap — novel/diverse for the training set.
|
||||
|
||||
frigate_files only contains files winnow uploaded — files added manually through
|
||||
Frigate's UI are never mapped here and are never touched by quality replacement.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .frigate_api import delete_frigate_person_files
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UPLOAD_TRACKER_FILE = "frigate_uploaded_ids.json"
|
||||
@@ -77,9 +85,10 @@ def _get_ids(entry: list | dict) -> list[str]:
|
||||
def _migrate_entry(entry: list | dict) -> dict:
|
||||
"""Ensure by_person entry is in the current dict format."""
|
||||
if isinstance(entry, list):
|
||||
return {"asset_ids": sorted(entry), "scores": {}, "frigate_files": {}, "crop_dims": {}}
|
||||
return {"asset_ids": sorted(entry), "scores": {}, "frigate_scores": {}, "frigate_files": {}, "crop_dims": {}}
|
||||
entry.setdefault("asset_ids", [])
|
||||
entry.setdefault("scores", {})
|
||||
entry.setdefault("frigate_scores", {})
|
||||
entry.setdefault("frigate_files", {})
|
||||
entry.setdefault("crop_dims", {})
|
||||
return entry
|
||||
@@ -91,6 +100,7 @@ def _mark(
|
||||
person_name: str | None,
|
||||
score: float | None = None,
|
||||
crop_dims: tuple[int, int] | None = None,
|
||||
frigate_score: float | None = None,
|
||||
) -> None:
|
||||
data = _load(filename)
|
||||
flat_key = _flat_key(filename)
|
||||
@@ -107,6 +117,8 @@ def _mark(
|
||||
entry["scores"][asset_id] = round(score, 4)
|
||||
if crop_dims is not None:
|
||||
entry["crop_dims"][asset_id] = [crop_dims[0], crop_dims[1]]
|
||||
if frigate_score is not None:
|
||||
entry["frigate_scores"][asset_id] = round(frigate_score, 4)
|
||||
by_person[person_name] = entry
|
||||
_save(filename, data)
|
||||
|
||||
@@ -126,8 +138,9 @@ def mark_uploaded(
|
||||
person_name: str | None = None,
|
||||
score: float | None = None,
|
||||
crop_dims: tuple[int, int] | None = None,
|
||||
frigate_score: float | None = None,
|
||||
) -> None:
|
||||
_mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims)
|
||||
_mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims, frigate_score=frigate_score)
|
||||
logger.debug(f"Marked {asset_id} as uploaded ({person_name})")
|
||||
|
||||
|
||||
@@ -136,6 +149,7 @@ def mark_rejected(asset_id: str, person_name: str | None = None) -> None:
|
||||
logger.debug(f"Marked {asset_id} as rejected ({person_name})")
|
||||
|
||||
|
||||
|
||||
def record_frigate_file(person_name: str, frigate_filename: str, asset_id: str) -> None:
|
||||
"""Record the mapping from a Frigate training filename to an Immich asset ID."""
|
||||
data = _load(UPLOAD_TRACKER_FILE)
|
||||
@@ -156,7 +170,9 @@ def remove_frigate_file(person_name: str, frigate_filename: str) -> None:
|
||||
data = _load(UPLOAD_TRACKER_FILE)
|
||||
by_person = data.get("by_person", {})
|
||||
entry = _migrate_entry(by_person.get(person_name, {}))
|
||||
entry["frigate_files"].pop(frigate_filename, None)
|
||||
asset_id = entry["frigate_files"].pop(frigate_filename, None)
|
||||
if asset_id:
|
||||
entry["frigate_scores"].pop(asset_id, None)
|
||||
by_person[person_name] = entry
|
||||
_save(UPLOAD_TRACKER_FILE, data)
|
||||
logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})")
|
||||
@@ -184,27 +200,64 @@ def get_tracked_frigate_filenames(person_name: str) -> set[str]:
|
||||
return set(entry["frigate_files"].keys())
|
||||
|
||||
|
||||
def has_frigate_scores(person_name: str) -> bool:
|
||||
"""Return True if any mapped file for this person has a stored Frigate recognition score."""
|
||||
data = _load(UPLOAD_TRACKER_FILE)
|
||||
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
||||
frigate_files = entry.get("frigate_files", {})
|
||||
frigate_scores = entry.get("frigate_scores", {})
|
||||
return any(asset_id in frigate_scores for asset_id in frigate_files.values())
|
||||
|
||||
|
||||
def _pick_mapped_file(
|
||||
person_name: str, score_key: str, *, highest: bool, exclude: set[str] | None = None
|
||||
) -> tuple[str, str, float] | None:
|
||||
data = _load(UPLOAD_TRACKER_FILE)
|
||||
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
||||
scores = entry.get(score_key, {})
|
||||
candidates = [
|
||||
(ff, asset_id, scores[asset_id])
|
||||
for ff, asset_id in entry.get("frigate_files", {}).items()
|
||||
if (exclude is None or ff not in exclude) and asset_id in scores
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda x: x[2]) if highest else min(candidates, key=lambda x: x[2])
|
||||
|
||||
|
||||
def get_lowest_quality_mapped_file(
|
||||
person_name: str, exclude: set[str] | None = None
|
||||
) -> tuple[str, str, float] | None:
|
||||
"""Return (frigate_filename, asset_id, score) for the mapped file with the lowest
|
||||
quality score, or None if no mapped files with known scores exist.
|
||||
blur score, or None if no mapped files with known scores exist.
|
||||
|
||||
Pass `exclude` to skip files that failed to delete this run without removing
|
||||
them from the tracker — they remain candidates on the next run.
|
||||
Used for quality replacement when no Frigate scores are available.
|
||||
Pass `exclude` to skip files that failed to delete this run.
|
||||
"""
|
||||
return _pick_mapped_file(person_name, "scores", highest=False, exclude=exclude)
|
||||
|
||||
|
||||
def get_most_redundant_mapped_file(
|
||||
person_name: str, exclude: set[str] | None = None
|
||||
) -> tuple[str, str, float] | None:
|
||||
"""Return (frigate_filename, asset_id, score) for the mapped file with the highest
|
||||
Frigate recognition score, or None if no mapped files with Frigate scores exist.
|
||||
|
||||
High Frigate score = the training set already covers this face condition well
|
||||
= the most redundant file and therefore the best replacement target.
|
||||
Pass `exclude` to skip files that failed to delete this run.
|
||||
"""
|
||||
return _pick_mapped_file(person_name, "frigate_scores", highest=True, exclude=exclude)
|
||||
|
||||
|
||||
def get_frigate_filename_for_asset(person_name: str, asset_id: str) -> str | None:
|
||||
"""Return the Frigate training filename mapped to this asset ID, or None."""
|
||||
data = _load(UPLOAD_TRACKER_FILE)
|
||||
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
||||
frigate_files = entry.get("frigate_files", {})
|
||||
scores = entry.get("scores", {})
|
||||
candidates = [
|
||||
(frigate_filename, asset_id, scores[asset_id])
|
||||
for frigate_filename, asset_id in frigate_files.items()
|
||||
if asset_id in scores and (exclude is None or frigate_filename not in exclude)
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return min(candidates, key=lambda x: x[2])
|
||||
for frigate_filename, aid in entry["frigate_files"].items():
|
||||
if aid == asset_id:
|
||||
return frigate_filename
|
||||
return None
|
||||
|
||||
|
||||
def find_by_crop_dimension(size: int) -> list[dict]:
|
||||
@@ -220,6 +273,7 @@ def find_by_crop_dimension(size: int) -> list[dict]:
|
||||
scores = entry.get("scores", {})
|
||||
frigate_files = entry.get("frigate_files", {})
|
||||
asset_to_frigate = {v: k for k, v in frigate_files.items()}
|
||||
frigate_scores = entry.get("frigate_scores", {})
|
||||
for asset_id, dims in entry.get("crop_dims", {}).items():
|
||||
w, h = dims[0], dims[1]
|
||||
if w == size or h == size:
|
||||
@@ -229,6 +283,7 @@ def find_by_crop_dimension(size: int) -> list[dict]:
|
||||
"width": w,
|
||||
"height": h,
|
||||
"blur_score": scores.get(asset_id),
|
||||
"frigate_score": frigate_scores.get(asset_id),
|
||||
"frigate_filename": asset_to_frigate.get(asset_id),
|
||||
})
|
||||
return results
|
||||
@@ -245,19 +300,41 @@ def update_frigate_count(person_name: str, count: int) -> None:
|
||||
|
||||
|
||||
def reset_person(person_name: str) -> None:
|
||||
"""Remove all uploaded and rejected records for a given person."""
|
||||
for filename in (UPLOAD_TRACKER_FILE, REJECT_TRACKER_FILE):
|
||||
data = _load(filename)
|
||||
"""Remove all uploaded and rejected records for a given person.
|
||||
|
||||
Also deletes winnow-managed Frigate training files so the next run starts
|
||||
clean rather than uploading on top of orphaned files. Manually-added Frigate
|
||||
files (not in frigate_files) are never touched. Proceeds with tracker reset
|
||||
even if Frigate is unreachable.
|
||||
"""
|
||||
upload_data = _load(UPLOAD_TRACKER_FILE)
|
||||
entry = _migrate_entry(upload_data.get("by_person", {}).get(person_name, {}))
|
||||
frigate_filenames = list(entry.get("frigate_files", {}).keys())
|
||||
if frigate_filenames:
|
||||
if not os.environ.get("FRIGATE_URL", "").strip():
|
||||
logger.info(f"FRIGATE_URL not set — skipping Frigate file deletion for {person_name}")
|
||||
elif delete_frigate_person_files(person_name, frigate_filenames):
|
||||
logger.info(f"Deleted {len(frigate_filenames)} Frigate file(s) for {person_name}")
|
||||
else:
|
||||
logger.warning(f"Could not delete Frigate files for {person_name} — tracker reset proceeding anyway")
|
||||
|
||||
changed = False
|
||||
tracker_files = ((UPLOAD_TRACKER_FILE, upload_data), (REJECT_TRACKER_FILE, _load(REJECT_TRACKER_FILE)))
|
||||
for filename, data in tracker_files:
|
||||
flat_key = _flat_key(filename)
|
||||
by_person = data.get("by_person", {})
|
||||
entry = by_person.pop(person_name, None)
|
||||
if entry is not None:
|
||||
person_ids = set(_get_ids(entry))
|
||||
tracker_entry = by_person.pop(person_name, None)
|
||||
if tracker_entry is not None:
|
||||
person_ids = set(_get_ids(tracker_entry))
|
||||
flat = set(data.get(flat_key, [])) - person_ids
|
||||
data[flat_key] = sorted(flat)
|
||||
data["by_person"] = by_person
|
||||
_save(filename, data)
|
||||
logger.info(f"Reset tracking data for {person_name}")
|
||||
changed = True
|
||||
if changed:
|
||||
logger.info(f"Reset tracking data for {person_name}")
|
||||
else:
|
||||
logger.debug(f"reset_person: no tracking data found for {person_name}")
|
||||
|
||||
|
||||
def get_person_summary() -> dict[str, dict]:
|
||||
|
||||
Reference in New Issue
Block a user