Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
326fdbdf38 | ||
|
|
91e0858aa6 | ||
|
|
71df0e81de | ||
|
|
9bb0727807 | ||
|
|
7c306a4423 | ||
|
|
c22857b912 | ||
|
|
c53172f5ff | ||
|
|
9598142997 |
@@ -2,7 +2,7 @@ name: Publish Docker Image
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: ["dev"]
|
branches: ["main", "dev"]
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- "**.md"
|
- "**.md"
|
||||||
- "docs/**"
|
- "docs/**"
|
||||||
@@ -15,16 +15,6 @@ on:
|
|||||||
- "uv-cpu.lock"
|
- "uv-cpu.lock"
|
||||||
- "uv-rocm.lock"
|
- "uv-rocm.lock"
|
||||||
- "uv-intel.lock"
|
- "uv-intel.lock"
|
||||||
workflow_call:
|
|
||||||
inputs:
|
|
||||||
tag:
|
|
||||||
type: string
|
|
||||||
required: false
|
|
||||||
description: "Release tag, e.g. v0.4.1 — triggers :latest + versioned image tags"
|
|
||||||
version:
|
|
||||||
type: string
|
|
||||||
required: false
|
|
||||||
description: "Version string without v prefix, e.g. 0.4.1"
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: docker-${{ github.ref }}
|
group: docker-${{ github.ref }}
|
||||||
@@ -36,13 +26,15 @@ env:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
name: Build (linux/amd64)
|
name: Build (${{ matrix.platform }})
|
||||||
runs-on: ubuntu-latest
|
runs-on: ${{ matrix.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- platform: linux/amd64
|
- platform: linux/amd64
|
||||||
runner: ubuntu-latest
|
runner: ubuntu-latest
|
||||||
|
- platform: linux/arm64
|
||||||
|
runner: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
@@ -58,8 +50,10 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
ref: ${{ inputs.tag || github.ref }}
|
- name: Set up QEMU
|
||||||
|
if: matrix.platform == 'linux/arm64'
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -71,15 +65,6 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Compute build version
|
|
||||||
id: version
|
|
||||||
run: |
|
|
||||||
if [ -n "${{ inputs.version }}" ]; then
|
|
||||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Build and push by digest
|
- name: Build and push by digest
|
||||||
id: build
|
id: build
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@v7
|
||||||
@@ -87,7 +72,6 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: ${{ matrix.platform }}
|
platforms: ${{ matrix.platform }}
|
||||||
build-args: VERSION=${{ steps.version.outputs.value }}
|
|
||||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||||
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -102,7 +86,7 @@ jobs:
|
|||||||
- name: Upload digest
|
- name: Upload digest
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: digest-amd64
|
name: digest-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||||
path: /tmp/digests/*
|
path: /tmp/digests/*
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
@@ -136,26 +120,22 @@ jobs:
|
|||||||
- name: Determine image tags
|
- name: Determine image tags
|
||||||
id: tags
|
id: tags
|
||||||
run: |
|
run: |
|
||||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||||
INPUT_TAG="${{ inputs.tag }}"
|
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev" >> "$GITHUB_OUTPUT"
|
||||||
if [ -n "$INPUT_TAG" ]; then
|
|
||||||
echo "tag_args=-t ${IMAGE}:latest -t ${IMAGE}:${INPUT_TAG}" >> "$GITHUB_OUTPUT"
|
|
||||||
echo "inspect_tag=${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
else
|
||||||
echo "tag_args=-t ${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> "$GITHUB_OUTPUT"
|
||||||
echo "inspect_tag=${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Create and push multi-arch manifest
|
- name: Create and push multi-arch manifest
|
||||||
working-directory: /tmp/digests
|
working-directory: /tmp/digests
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools create \
|
docker buildx imagetools create \
|
||||||
${{ steps.tags.outputs.tag_args }} \
|
-t ${{ steps.tags.outputs.tags }} \
|
||||||
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||||
|
|
||||||
- name: Inspect image
|
- name: Inspect image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.tags.outputs.inspect_tag }}
|
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -182,8 +162,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
ref: ${{ inputs.tag || github.ref }}
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v4
|
uses: docker/setup-qemu-action@v4
|
||||||
@@ -198,30 +176,13 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine CPU image tags
|
- name: Determine CPU image tag
|
||||||
id: cpu-tags
|
id: cpu-tag
|
||||||
run: |
|
run: |
|
||||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||||
INPUT_TAG="${{ inputs.tag }}"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||||
if [ -n "$INPUT_TAG" ]; then
|
|
||||||
{
|
|
||||||
echo "tags<<EOF"
|
|
||||||
printf '%s\n' "${IMAGE}:cpu" "${IMAGE}:${INPUT_TAG}-cpu"
|
|
||||||
echo "EOF"
|
|
||||||
} >> "$GITHUB_OUTPUT"
|
|
||||||
echo "inspect_tag=${IMAGE}:cpu" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
else
|
||||||
echo "tags=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cpu" >> "$GITHUB_OUTPUT"
|
||||||
echo "inspect_tag=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Compute build version
|
|
||||||
id: version
|
|
||||||
run: |
|
|
||||||
if [ -n "${{ inputs.version }}" ]; then
|
|
||||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push CPU image
|
- name: Build and push CPU image
|
||||||
@@ -230,18 +191,16 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
build-args: |
|
build-args: VARIANT=cpu
|
||||||
VARIANT=cpu
|
|
||||||
VERSION=${{ steps.version.outputs.value }}
|
|
||||||
cache-from: type=gha,scope=cpu
|
cache-from: type=gha,scope=cpu
|
||||||
cache-to: type=gha,mode=max,scope=cpu
|
cache-to: type=gha,mode=max,scope=cpu
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.cpu-tags.outputs.tags }}
|
tags: ${{ steps.cpu-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Inspect CPU image
|
- name: Inspect CPU image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.cpu-tags.outputs.inspect_tag }}
|
docker buildx imagetools inspect ${{ steps.cpu-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -268,8 +227,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
ref: ${{ inputs.tag || github.ref }}
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -281,30 +238,13 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine ROCm image tags
|
- name: Determine ROCm image tag
|
||||||
id: rocm-tags
|
id: rocm-tag
|
||||||
run: |
|
run: |
|
||||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||||
INPUT_TAG="${{ inputs.tag }}"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||||
if [ -n "$INPUT_TAG" ]; then
|
|
||||||
{
|
|
||||||
echo "tags<<EOF"
|
|
||||||
printf '%s\n' "${IMAGE}:rocm" "${IMAGE}:${INPUT_TAG}-rocm"
|
|
||||||
echo "EOF"
|
|
||||||
} >> "$GITHUB_OUTPUT"
|
|
||||||
echo "inspect_tag=${IMAGE}:rocm" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
else
|
||||||
echo "tags=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:rocm" >> "$GITHUB_OUTPUT"
|
||||||
echo "inspect_tag=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Compute build version
|
|
||||||
id: version
|
|
||||||
run: |
|
|
||||||
if [ -n "${{ inputs.version }}" ]; then
|
|
||||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push ROCm image
|
- name: Build and push ROCm image
|
||||||
@@ -313,18 +253,16 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
build-args: |
|
build-args: VARIANT=rocm
|
||||||
VARIANT=rocm
|
|
||||||
VERSION=${{ steps.version.outputs.value }}
|
|
||||||
cache-from: type=gha,scope=linux/amd64-rocm
|
cache-from: type=gha,scope=linux/amd64-rocm
|
||||||
cache-to: type=gha,mode=max,scope=linux/amd64-rocm
|
cache-to: type=gha,mode=max,scope=linux/amd64-rocm
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.rocm-tags.outputs.tags }}
|
tags: ${{ steps.rocm-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Inspect ROCm image
|
- name: Inspect ROCm image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.rocm-tags.outputs.inspect_tag }}
|
docker buildx imagetools inspect ${{ steps.rocm-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -351,8 +289,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
ref: ${{ inputs.tag || github.ref }}
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -364,30 +300,13 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine Intel image tags
|
- name: Determine Intel image tag
|
||||||
id: intel-tags
|
id: intel-tag
|
||||||
run: |
|
run: |
|
||||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
if [ "${{ github.ref_name }}" = "dev" ]; then
|
||||||
INPUT_TAG="${{ inputs.tag }}"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||||
if [ -n "$INPUT_TAG" ]; then
|
|
||||||
{
|
|
||||||
echo "tags<<EOF"
|
|
||||||
printf '%s\n' "${IMAGE}:intel" "${IMAGE}:${INPUT_TAG}-intel"
|
|
||||||
echo "EOF"
|
|
||||||
} >> "$GITHUB_OUTPUT"
|
|
||||||
echo "inspect_tag=${IMAGE}:intel" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
else
|
||||||
echo "tags=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:intel" >> "$GITHUB_OUTPUT"
|
||||||
echo "inspect_tag=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Compute build version
|
|
||||||
id: version
|
|
||||||
run: |
|
|
||||||
if [ -n "${{ inputs.version }}" ]; then
|
|
||||||
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "value=dev" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push Intel image
|
- name: Build and push Intel image
|
||||||
@@ -396,18 +315,16 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
build-args: |
|
build-args: VARIANT=intel
|
||||||
VARIANT=intel
|
|
||||||
VERSION=${{ steps.version.outputs.value }}
|
|
||||||
cache-from: type=gha,scope=linux/amd64-intel
|
cache-from: type=gha,scope=linux/amd64-intel
|
||||||
cache-to: type=gha,mode=max,scope=linux/amd64-intel
|
cache-to: type=gha,mode=max,scope=linux/amd64-intel
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.intel-tags.outputs.tags }}
|
tags: ${{ steps.intel-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Inspect Intel image
|
- name: Inspect Intel image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.intel-tags.outputs.inspect_tag }}
|
docker buildx imagetools inspect ${{ steps.intel-tag.outputs.tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -127,14 +127,188 @@ jobs:
|
|||||||
prerelease: false,
|
prerelease: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
build-images:
|
build-gpu:
|
||||||
name: Build and push Docker images
|
name: Build GPU image
|
||||||
needs: release
|
needs: release
|
||||||
uses: ./.github/workflows/docker-publish.yml
|
runs-on: ubuntu-latest
|
||||||
with:
|
|
||||||
tag: ${{ needs.release.outputs.tag }}
|
|
||||||
version: ${{ needs.release.outputs.version }}
|
|
||||||
secrets: inherit
|
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
contents: read
|
steps:
|
||||||
|
- name: Free up disk space
|
||||||
|
run: |
|
||||||
|
sudo rm -rf /usr/share/dotnet
|
||||||
|
sudo rm -rf /opt/ghc
|
||||||
|
sudo rm -rf "/usr/local/share/boost"
|
||||||
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||||
|
echo "Disk space freed."
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build and push GPU image (latest)
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
build-args: VERSION=${{ needs.release.outputs.version }}
|
||||||
|
cache-from: type=gha,scope=release-gpu
|
||||||
|
cache-to: type=gha,mode=max,scope=release-gpu
|
||||||
|
tags: |
|
||||||
|
ghcr.io/sudolulo/winnow:latest
|
||||||
|
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}
|
||||||
|
|
||||||
|
build-cpu:
|
||||||
|
name: Build CPU image
|
||||||
|
needs: release
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
packages: write
|
||||||
|
steps:
|
||||||
|
- name: Free up disk space
|
||||||
|
run: |
|
||||||
|
sudo rm -rf /usr/share/dotnet
|
||||||
|
sudo rm -rf /opt/ghc
|
||||||
|
sudo rm -rf "/usr/local/share/boost"
|
||||||
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||||
|
echo "Disk space freed."
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build and push CPU image
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
build-args: |
|
||||||
|
VARIANT=cpu
|
||||||
|
VERSION=${{ needs.release.outputs.version }}
|
||||||
|
cache-from: type=gha,scope=release-cpu
|
||||||
|
cache-to: type=gha,mode=max,scope=release-cpu
|
||||||
|
tags: |
|
||||||
|
ghcr.io/sudolulo/winnow:cpu
|
||||||
|
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-cpu
|
||||||
|
|
||||||
|
build-rocm:
|
||||||
|
name: Build ROCm image
|
||||||
|
needs: release
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
packages: write
|
||||||
|
steps:
|
||||||
|
- name: Free up disk space
|
||||||
|
run: |
|
||||||
|
sudo rm -rf /usr/share/dotnet
|
||||||
|
sudo rm -rf /opt/ghc
|
||||||
|
sudo rm -rf "/usr/local/share/boost"
|
||||||
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||||
|
echo "Disk space freed."
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build and push ROCm image
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
build-args: |
|
||||||
|
VARIANT=rocm
|
||||||
|
VERSION=${{ needs.release.outputs.version }}
|
||||||
|
cache-from: type=gha,scope=release-rocm
|
||||||
|
cache-to: type=gha,mode=max,scope=release-rocm
|
||||||
|
tags: |
|
||||||
|
ghcr.io/sudolulo/winnow:rocm
|
||||||
|
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-rocm
|
||||||
|
|
||||||
|
build-intel:
|
||||||
|
name: Build Intel image
|
||||||
|
needs: release
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
packages: write
|
||||||
|
steps:
|
||||||
|
- name: Free up disk space
|
||||||
|
run: |
|
||||||
|
sudo rm -rf /usr/share/dotnet
|
||||||
|
sudo rm -rf /opt/ghc
|
||||||
|
sudo rm -rf "/usr/local/share/boost"
|
||||||
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||||
|
echo "Disk space freed."
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build and push Intel image
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
build-args: |
|
||||||
|
VARIANT=intel
|
||||||
|
VERSION=${{ needs.release.outputs.version }}
|
||||||
|
cache-from: type=gha,scope=release-intel
|
||||||
|
cache-to: type=gha,mode=max,scope=release-intel
|
||||||
|
tags: |
|
||||||
|
ghcr.io/sudolulo/winnow:intel
|
||||||
|
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-intel
|
||||||
|
|||||||
@@ -7,23 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
## [0.4.3] - 2026-06-14
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **InsightFace landmark-based face crop alignment**: face crops for Frigate training are now aligned using InsightFace's `norm_crop` (ArcFace 112×112 alignment with 5-point facial landmarks). Previously, Immich's API returned only bounding boxes with no landmarks, so `align_face()` was dead code and crops were plain bbox slices — resulting in misaligned or partial crops (e.g. foreheads). The fix runs InsightFace detection on an expanded region around the Immich bbox, finds the nearest face, and uses its keypoints for proper alignment. Controlled by `ENABLE_FACE_ALIGNMENT` (default `true`).
|
|
||||||
- **Duplicate Immich person detection and handling**: when multiple Immich person records share the same name, winnow now detects this at startup and warns with a per-group summary. Without handling, two jobs would run for the same Frigate folder and overwrite each other's output. By default (`MERGE_DUPLICATE_PEOPLE=false`) only the first person per name is processed. Set `MERGE_DUPLICATE_PEOPLE=true` to permanently merge duplicate records inside Immich (keeps the person with the most assets).
|
|
||||||
|
|
||||||
## [0.4.2] - 2026-06-13
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **GPU image now uses CUDA 12.8.1** (was 13.3): CUDA 13.3 requires driver ≥ 575; driver 570 (the current stable release) was incorrectly rejected with "CUDA driver version is insufficient" at startup. The `:latest` image now works with any NVIDIA driver ≥ 570.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **`scripts/benchmark.py`**: measures InsightFace and SigLIP inference latency and throughput across GPU and CPU modes. Run inside the container with `python /app/scripts/benchmark.py`. RTX 2070 SUPER results: InsightFace 12.8 ms / 78 img/s (8× CPU), SigLIP batch 32 at 5.4 ms/img / 187 img/s (33× CPU).
|
|
||||||
|
|
||||||
## [0.4.1] - 2026-06-13
|
## [0.4.1] - 2026-06-13
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
@@ -36,10 +36,6 @@ CI runs both on every push and PR to `main` and `dev`. PRs must pass before merg
|
|||||||
- Keep the `CHANGELOG.md` entry in the `[Unreleased]` section updated.
|
- Keep the `CHANGELOG.md` entry in the `[Unreleased]` section updated.
|
||||||
- Commit messages should be plain English describing what changed and why.
|
- Commit messages should be plain English describing what changed and why.
|
||||||
|
|
||||||
## Development Tooling
|
|
||||||
|
|
||||||
Development uses Claude Code (Anthropic) for implementation assistance. All code is reviewed and the final call on design, behavior, and what ships is made by the maintainer. Contributions from humans are equally welcome.
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
By submitting a contribution you agree that your work will be released under the project's [AGPLv3+ license](LICENSE).
|
By submitting a contribution you agree that your work will be released under the project's [AGPLv3+ license](LICENSE).
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
# ── Base images ───────────────────────────────────────────────────────────────
|
# ── Base images ───────────────────────────────────────────────────────────────
|
||||||
# amd64 + gpu: NVIDIA CUDA 12.8 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
# amd64 + gpu: NVIDIA CUDA 13.3 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
||||||
# amd64 + rocm: Ubuntu 22.04 (AMD GPU via ROCm — pass /dev/kfd and /dev/dri)
|
# amd64 + rocm: Ubuntu 22.04 (AMD GPU via ROCm — pass /dev/kfd and /dev/dri)
|
||||||
# amd64 + intel: Ubuntu 22.04 (Intel Arc / iGPU via OpenVINO — pass /dev/dri)
|
# amd64 + intel: Ubuntu 22.04 (Intel Arc / iGPU via OpenVINO — pass /dev/dri)
|
||||||
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
||||||
@@ -7,7 +7,7 @@
|
|||||||
|
|
||||||
ARG VARIANT=gpu
|
ARG VARIANT=gpu
|
||||||
|
|
||||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
FROM --platform=$BUILDPLATFORM nvidia/cuda:13.3.0-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
||||||
FROM ubuntu:22.04 AS base-amd64-rocm
|
FROM ubuntu:22.04 AS base-amd64-rocm
|
||||||
FROM ubuntu:22.04 AS base-amd64-intel
|
FROM ubuntu:22.04 AS base-amd64-intel
|
||||||
FROM ubuntu:22.04 AS base-amd64-cpu
|
FROM ubuntu:22.04 AS base-amd64-cpu
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ services:
|
|||||||
# - SKIP_PEOPLE=Unknown # Comma-separated; skip these people
|
# - SKIP_PEOPLE=Unknown # Comma-separated; skip these people
|
||||||
# - MIN_FACE_COUNT=5 # Skip people with fewer than N assets in Immich
|
# - MIN_FACE_COUNT=5 # Skip people with fewer than N assets in Immich
|
||||||
# - YEARS_FILTER=10 # Only include images from the last N years (default: 10)
|
# - YEARS_FILTER=10 # Only include images from the last N years (default: 10)
|
||||||
# - MERGE_DUPLICATE_PEOPLE=true # Auto-merge Immich people with the same name (keeps most assets)
|
|
||||||
|
|
||||||
# ── Image Quality ─────────────────────────────────────────────────────
|
# ── Image Quality ─────────────────────────────────────────────────────
|
||||||
# - MIN_FACE_WIDTH=50 # Minimum face width in pixels (default: 50)
|
# - MIN_FACE_WIDTH=50 # Minimum face width in pixels (default: 50)
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.4.3"
|
version = "0.4.1"
|
||||||
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
|
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|||||||
@@ -1,196 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
winnow inference benchmark: GPU vs CPU throughput.
|
|
||||||
|
|
||||||
Measures InsightFace (face mode) and SigLIP (object mode) latency and
|
|
||||||
throughput. Run with FORCE_CPU=true for CPU-only baseline.
|
|
||||||
|
|
||||||
Usage inside container:
|
|
||||||
# GPU mode:
|
|
||||||
docker exec winnow python /app/scripts/benchmark.py
|
|
||||||
|
|
||||||
# CPU mode:
|
|
||||||
docker exec -e FORCE_CPU=true winnow python /app/scripts/benchmark.py
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
from PIL import Image, ImageDraw
|
|
||||||
|
|
||||||
|
|
||||||
def _mode_label() -> str:
|
|
||||||
if os.getenv("FORCE_CPU", "").lower() in ("true", "1", "yes"):
|
|
||||||
return "CPU (FORCE_CPU=true)"
|
|
||||||
return "GPU (auto)"
|
|
||||||
|
|
||||||
|
|
||||||
def make_face_image(size: int = 640) -> Image.Image:
|
|
||||||
"""Synthetic face-like image: skin-tone rectangle with landmark blobs."""
|
|
||||||
img = Image.new("RGB", (size, size), (200, 170, 140))
|
|
||||||
draw = ImageDraw.Draw(img)
|
|
||||||
# Head oval
|
|
||||||
cx, cy = size // 2, size // 2
|
|
||||||
hw, hh = int(size * 0.3), int(size * 0.38)
|
|
||||||
draw.ellipse([cx - hw, cy - hh, cx + hw, cy + hh], fill=(220, 185, 155))
|
|
||||||
# Eyes
|
|
||||||
for ex in [cx - int(size * 0.1), cx + int(size * 0.1)]:
|
|
||||||
ey = cy - int(size * 0.05)
|
|
||||||
r = max(4, size // 40)
|
|
||||||
draw.ellipse([ex - r, ey - r, ex + r, ey + r], fill=(40, 30, 20))
|
|
||||||
# Nose
|
|
||||||
draw.ellipse([cx - 5, cy + 5, cx + 5, cy + 15], fill=(180, 140, 110))
|
|
||||||
# Mouth
|
|
||||||
draw.arc([cx - 20, cy + 25, cx + 20, cy + 45], start=0, end=180, fill=(160, 80, 80), width=3)
|
|
||||||
return img
|
|
||||||
|
|
||||||
|
|
||||||
def make_random_image(width: int = 224, height: int = 224) -> Image.Image:
|
|
||||||
rng = np.random.default_rng(42)
|
|
||||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8), "RGB")
|
|
||||||
|
|
||||||
|
|
||||||
def _stats(times_s: list[float]) -> dict:
|
|
||||||
arr = np.array(times_s) * 1000 # ms
|
|
||||||
return {
|
|
||||||
"median_ms": float(np.median(arr)),
|
|
||||||
"mean_ms": float(np.mean(arr)),
|
|
||||||
"min_ms": float(np.min(arr)),
|
|
||||||
"p95_ms": float(np.percentile(arr, 95)),
|
|
||||||
"ips": 1000.0 / float(np.median(arr)),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def bench_insightface(n_warmup: int = 5, n_runs: int = 30) -> None:
|
|
||||||
import cv2
|
|
||||||
|
|
||||||
import winnow.embeddings as emb_mod
|
|
||||||
from winnow.embeddings import get_insightface_app
|
|
||||||
|
|
||||||
# Reset singleton so we get a fresh load
|
|
||||||
emb_mod._insightface_app = None
|
|
||||||
emb_mod._insightface_loaded = False
|
|
||||||
|
|
||||||
print(" Loading model...")
|
|
||||||
t_load = time.perf_counter()
|
|
||||||
app = get_insightface_app()
|
|
||||||
load_s = time.perf_counter() - t_load
|
|
||||||
|
|
||||||
if app is None:
|
|
||||||
print(" SKIP: InsightFace failed to load")
|
|
||||||
return
|
|
||||||
|
|
||||||
img_pil = make_face_image(640)
|
|
||||||
img_bgr = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
|
|
||||||
|
|
||||||
# Warmup
|
|
||||||
for _ in range(n_warmup):
|
|
||||||
app.get(img_bgr)
|
|
||||||
|
|
||||||
# Timed — single image 640×640
|
|
||||||
times: list[float] = []
|
|
||||||
for _ in range(n_runs):
|
|
||||||
t0 = time.perf_counter()
|
|
||||||
app.get(img_bgr)
|
|
||||||
times.append(time.perf_counter() - t0)
|
|
||||||
|
|
||||||
s = _stats(times)
|
|
||||||
print(f" Model load time : {load_s:.2f} s")
|
|
||||||
print(" Input size : 640×640")
|
|
||||||
print(f" Runs : {n_runs} (after {n_warmup} warmup)")
|
|
||||||
print(f" Median latency : {s['median_ms']:.1f} ms")
|
|
||||||
print(f" Mean / p95 : {s['mean_ms']:.1f} ms / {s['p95_ms']:.1f} ms")
|
|
||||||
print(f" Min latency : {s['min_ms']:.1f} ms")
|
|
||||||
print(f" Throughput : {s['ips']:.1f} images/s")
|
|
||||||
|
|
||||||
# Also test at 320×320
|
|
||||||
img_sm = make_face_image(320)
|
|
||||||
img_sm_bgr = cv2.cvtColor(np.asarray(img_sm), cv2.COLOR_RGB2BGR)
|
|
||||||
for _ in range(n_warmup):
|
|
||||||
app.get(img_sm_bgr)
|
|
||||||
times_sm: list[float] = []
|
|
||||||
for _ in range(n_runs):
|
|
||||||
t0 = time.perf_counter()
|
|
||||||
app.get(img_sm_bgr)
|
|
||||||
times_sm.append(time.perf_counter() - t0)
|
|
||||||
s2 = _stats(times_sm)
|
|
||||||
print(f" 320×320 median : {s2['median_ms']:.1f} ms ({s2['ips']:.1f} img/s)")
|
|
||||||
|
|
||||||
|
|
||||||
def bench_siglip(
|
|
||||||
n_warmup: int = 3,
|
|
||||||
n_runs: int = 20,
|
|
||||||
batch_sizes: tuple = (1, 4, 8, 16, 32),
|
|
||||||
) -> None:
|
|
||||||
import torch
|
|
||||||
|
|
||||||
import winnow.embeddings as emb_mod
|
|
||||||
emb_mod._siglip_model = None
|
|
||||||
emb_mod._siglip_processor = None
|
|
||||||
emb_mod._siglip_loaded = False
|
|
||||||
|
|
||||||
print(" Loading model...")
|
|
||||||
t_load = time.perf_counter()
|
|
||||||
model, processor = emb_mod.get_siglip_model()
|
|
||||||
load_s = time.perf_counter() - t_load
|
|
||||||
|
|
||||||
if model is None:
|
|
||||||
print(" SKIP: SigLIP failed to load")
|
|
||||||
return
|
|
||||||
|
|
||||||
device = next(model.parameters()).device
|
|
||||||
print(f" Model load time : {load_s:.2f} s (device: {device})")
|
|
||||||
|
|
||||||
print(f" {'Batch':>5} {'ms/batch':>10} {'ms/img':>8} {'img/s':>8} {'p95/img':>9}")
|
|
||||||
for bs in batch_sizes:
|
|
||||||
imgs = [make_random_image(224, 224) for _ in range(bs)]
|
|
||||||
inputs = processor(images=imgs, return_tensors="pt")
|
|
||||||
inputs = {k: v.to(device) for k, v in inputs.items()}
|
|
||||||
|
|
||||||
# Warmup
|
|
||||||
for _ in range(n_warmup):
|
|
||||||
with torch.no_grad():
|
|
||||||
model(**inputs)
|
|
||||||
if str(device) != "cpu":
|
|
||||||
torch.cuda.synchronize()
|
|
||||||
|
|
||||||
times: list[float] = []
|
|
||||||
for _ in range(n_runs):
|
|
||||||
if str(device) != "cpu":
|
|
||||||
torch.cuda.synchronize()
|
|
||||||
t0 = time.perf_counter()
|
|
||||||
with torch.no_grad():
|
|
||||||
model(**inputs)
|
|
||||||
if str(device) != "cpu":
|
|
||||||
torch.cuda.synchronize()
|
|
||||||
times.append(time.perf_counter() - t0)
|
|
||||||
|
|
||||||
s = _stats(times)
|
|
||||||
print(
|
|
||||||
f" {bs:>5} {s['median_ms']:>10.1f} {s['median_ms']/bs:>8.2f}"
|
|
||||||
f" {bs * 1000 / s['median_ms']:>8.1f} {s['p95_ms']/bs:>9.2f}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
print("=" * 56)
|
|
||||||
print(" winnow inference benchmark")
|
|
||||||
print(f" Mode: {_mode_label()}")
|
|
||||||
print("=" * 56)
|
|
||||||
print()
|
|
||||||
|
|
||||||
print("── InsightFace Buffalo_L (face detection + ArcFace) ──")
|
|
||||||
bench_insightface()
|
|
||||||
print()
|
|
||||||
|
|
||||||
print("── SigLIP google/siglip-base-patch16-224 (objects) ───")
|
|
||||||
bench_siglip()
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# Add winnow to path when run directly inside container
|
|
||||||
sys.path.insert(0, "/app")
|
|
||||||
main()
|
|
||||||
@@ -2348,7 +2348,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.4.2"
|
version = "0.4.0"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "croniter" },
|
{ name = "croniter" },
|
||||||
|
|||||||
+1
-82
@@ -9,7 +9,7 @@ from rich.prompt import Confirm
|
|||||||
|
|
||||||
from .config import Config, ConfigManager
|
from .config import Config, ConfigManager
|
||||||
from .executor import execute_jobs, upload_to_frigate
|
from .executor import execute_jobs, upload_to_frigate
|
||||||
from .immich_api import get_people, merge_people
|
from .immich_api import get_people
|
||||||
from .jobs import _show_preview, auto_configure, interactive_configure
|
from .jobs import _show_preview, auto_configure, interactive_configure
|
||||||
from .log_config import console, setup_logging
|
from .log_config import console, setup_logging
|
||||||
from .upload_tracker import find_by_crop_dimension, get_person_summary, reset_person
|
from .upload_tracker import find_by_crop_dimension, get_person_summary, reset_person
|
||||||
@@ -51,85 +51,6 @@ def _handle_trace_crop(size_str: str) -> None:
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
def _handle_duplicate_people(people: list[dict]) -> list[dict]:
|
|
||||||
"""Warn about or merge Immich people that share the same name.
|
|
||||||
|
|
||||||
Duplicates arise when Immich creates separate person records for the same
|
|
||||||
individual (e.g. unmerged face clusters). Without handling, winnow would
|
|
||||||
run multiple jobs for the same Frigate folder and overwrite its own output,
|
|
||||||
leaving far fewer training images than expected.
|
|
||||||
|
|
||||||
With MERGE_DUPLICATE_PEOPLE=false (default): prints a warning, skips the
|
|
||||||
smaller duplicates so only the person with the most assets is processed,
|
|
||||||
and returns a deduplicated people list.
|
|
||||||
With MERGE_DUPLICATE_PEOPLE=true: merges each duplicate group inside
|
|
||||||
Immich via its API (permanently combines the face records), then
|
|
||||||
re-fetches the people list so the rest of the run sees the merged state.
|
|
||||||
"""
|
|
||||||
from collections import defaultdict
|
|
||||||
|
|
||||||
by_name: dict[str, list[dict]] = defaultdict(list)
|
|
||||||
for p in people:
|
|
||||||
name = (p.get("name") or "").strip()
|
|
||||||
if name:
|
|
||||||
by_name[name].append(p)
|
|
||||||
|
|
||||||
duplicates = {name: ps for name, ps in by_name.items() if len(ps) > 1}
|
|
||||||
if not duplicates:
|
|
||||||
return people
|
|
||||||
|
|
||||||
if not Config.MERGE_DUPLICATE_PEOPLE:
|
|
||||||
rprint("\n[bold yellow]⚠ Duplicate person names detected in Immich:[/bold yellow]")
|
|
||||||
for name, ps in sorted(duplicates.items()):
|
|
||||||
ordered = sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)
|
|
||||||
entries = ", ".join(
|
|
||||||
f"[dim]{p['id'][:8]}…[/dim] ({p.get('assetCount', 0)} assets)"
|
|
||||||
for p in ordered
|
|
||||||
)
|
|
||||||
rprint(f" [yellow]{name}[/yellow] → {len(ps)} people: {entries}")
|
|
||||||
skipped = ordered[1:]
|
|
||||||
rprint(
|
|
||||||
f" [dim] Processing largest only "
|
|
||||||
f"({ordered[0].get('assetCount', 0)} assets). "
|
|
||||||
f"Skipping {len(skipped)} smaller duplicate(s) to avoid overwriting output.[/dim]"
|
|
||||||
)
|
|
||||||
rprint(
|
|
||||||
" [dim]Set MERGE_DUPLICATE_PEOPLE=true to permanently merge duplicates "
|
|
||||||
"inside Immich (keeps the person with the most assets).[/dim]\n"
|
|
||||||
)
|
|
||||||
# Return deduplicated list — keep only the largest per name so that
|
|
||||||
# downstream job creation never runs two jobs for the same Frigate folder.
|
|
||||||
skip_ids = {
|
|
||||||
p["id"]
|
|
||||||
for ps in duplicates.values()
|
|
||||||
for p in sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)[1:]
|
|
||||||
}
|
|
||||||
return [p for p in people if p["id"] not in skip_ids]
|
|
||||||
|
|
||||||
# Auto-merge: survivor = largest asset count, rest merge into it inside Immich
|
|
||||||
merged_any = False
|
|
||||||
for name, ps in sorted(duplicates.items()):
|
|
||||||
ordered = sorted(ps, key=lambda x: x.get("assetCount", 0), reverse=True)
|
|
||||||
survivor = ordered[0]
|
|
||||||
merge_ids = [p["id"] for p in ordered[1:]]
|
|
||||||
rprint(
|
|
||||||
f" [cyan]Merging {name!r} inside Immich:[/cyan] keeping "
|
|
||||||
f"[dim]{survivor['id'][:8]}…[/dim] ({survivor.get('assetCount', 0)} assets), "
|
|
||||||
f"absorbing {len(merge_ids)} smaller duplicate(s)..."
|
|
||||||
)
|
|
||||||
if merge_people(survivor["id"], merge_ids):
|
|
||||||
rprint(f" [green]✓ Merged {name!r}[/green]")
|
|
||||||
merged_any = True
|
|
||||||
else:
|
|
||||||
rprint(f" [red]✗ Failed to merge {name!r}[/red]")
|
|
||||||
|
|
||||||
if merged_any:
|
|
||||||
rprint(" [dim]Re-fetching people after merge...[/dim]")
|
|
||||||
return get_people()
|
|
||||||
|
|
||||||
return people
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
"""Entry point for winnow CLI."""
|
"""Entry point for winnow CLI."""
|
||||||
try:
|
try:
|
||||||
@@ -182,8 +103,6 @@ def main() -> None:
|
|||||||
rprint("[bold red]Could not fetch people from Immich. Check URL/Key.[/bold red]")
|
rprint("[bold red]Could not fetch people from Immich. Check URL/Key.[/bold red]")
|
||||||
return
|
return
|
||||||
|
|
||||||
people = _handle_duplicate_people(people)
|
|
||||||
|
|
||||||
# Auto mode when no TTY (Docker, cron, pipes) — the primary use case.
|
# Auto mode when no TTY (Docker, cron, pipes) — the primary use case.
|
||||||
# A TTY means local interactive use; AUTO_MODE=true overrides that for scripting.
|
# A TTY means local interactive use; AUTO_MODE=true overrides that for scripting.
|
||||||
auto_mode = not sys.stdin.isatty() or os.environ.get("AUTO_MODE", "").lower() in ("true", "1", "yes")
|
auto_mode = not sys.stdin.isatty() or os.environ.get("AUTO_MODE", "").lower() in ("true", "1", "yes")
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ class _Config:
|
|||||||
|
|
||||||
# People filtering
|
# People filtering
|
||||||
MIN_FACE_COUNT: int = 0
|
MIN_FACE_COUNT: int = 0
|
||||||
MERGE_DUPLICATE_PEOPLE: bool = False
|
|
||||||
|
|
||||||
# Output quality
|
# Output quality
|
||||||
FACE_MARGIN: float = 0.15
|
FACE_MARGIN: float = 0.15
|
||||||
@@ -61,7 +60,6 @@ class _Config:
|
|||||||
self.YEARS_FILTER = int(os.getenv("YEARS_FILTER", "10"))
|
self.YEARS_FILTER = int(os.getenv("YEARS_FILTER", "10"))
|
||||||
self.MIN_FACE_WIDTH = int(os.getenv("MIN_FACE_WIDTH", "90"))
|
self.MIN_FACE_WIDTH = int(os.getenv("MIN_FACE_WIDTH", "90"))
|
||||||
self.MIN_FACE_COUNT = int(os.getenv("MIN_FACE_COUNT", "0"))
|
self.MIN_FACE_COUNT = int(os.getenv("MIN_FACE_COUNT", "0"))
|
||||||
self.MERGE_DUPLICATE_PEOPLE = os.getenv("MERGE_DUPLICATE_PEOPLE", "false").lower() in ("true", "1", "yes")
|
|
||||||
self.BLUR_THRESHOLD = float(os.getenv("BLUR_THRESHOLD", "120.0"))
|
self.BLUR_THRESHOLD = float(os.getenv("BLUR_THRESHOLD", "120.0"))
|
||||||
self.MIN_CONFIDENCE = float(os.getenv("MIN_CONFIDENCE", "0.7"))
|
self.MIN_CONFIDENCE = float(os.getenv("MIN_CONFIDENCE", "0.7"))
|
||||||
self.MAX_AUTO_IMAGES = int(os.getenv("MAX_AUTO_IMAGES", "80"))
|
self.MAX_AUTO_IMAGES = int(os.getenv("MAX_AUTO_IMAGES", "80"))
|
||||||
|
|||||||
+1
-15
@@ -155,18 +155,6 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
|
|
||||||
use_full_res = Config.USE_FULL_RESOLUTION
|
use_full_res = Config.USE_FULL_RESOLUTION
|
||||||
|
|
||||||
# Load InsightFace app for landmark-based crop alignment (face mode only).
|
|
||||||
# The model is already resident from the diversity/embedding phase, so this
|
|
||||||
# is just a singleton lookup — no load cost.
|
|
||||||
insightface_app = None
|
|
||||||
if any(j["config"].get("mode", "face") == "face" for j in jobs) and Config.ENABLE_FACE_ALIGNMENT:
|
|
||||||
try:
|
|
||||||
from .embeddings import get_insightface_app
|
|
||||||
|
|
||||||
insightface_app = get_insightface_app()
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"InsightFace unavailable for crop alignment: {e}")
|
|
||||||
|
|
||||||
with Progress(
|
with Progress(
|
||||||
SpinnerColumn(),
|
SpinnerColumn(),
|
||||||
TextColumn("[progress.description]{task.description}"),
|
TextColumn("[progress.description]{task.description}"),
|
||||||
@@ -228,7 +216,7 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
progress.console.print(f"[red]Failed download {asset['id']}[/red]")
|
progress.console.print(f"[red]Failed download {asset['id']}[/red]")
|
||||||
else:
|
else:
|
||||||
saved = (
|
saved = (
|
||||||
process_face_mode(img, asset, person, person_dir, count, insightface_app=insightface_app)
|
process_face_mode(img, asset, person, person_dir, count)
|
||||||
if mode == "face"
|
if mode == "face"
|
||||||
else process_object_mode(img, config, person_dir, count)
|
else process_object_mode(img, config, person_dir, count)
|
||||||
if mode == "object"
|
if mode == "object"
|
||||||
@@ -376,8 +364,6 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
# Snapshot live Frigate files for post-upload reconciliation diff only.
|
# Snapshot live Frigate files for post-upload reconciliation diff only.
|
||||||
# effective_count is sourced from the tracker (mapped files) so that
|
# effective_count is sourced from the tracker (mapped files) so that
|
||||||
# manually-added Frigate files don't consume winnow's managed quota.
|
# manually-added Frigate files don't consume winnow's managed quota.
|
||||||
# Replacement targets also come exclusively from the tracker, so manually
|
|
||||||
# added files are never selected for deletion — only winnow-uploaded ones.
|
|
||||||
_snapshot = (
|
_snapshot = (
|
||||||
all_frigate_files.get(name, []) if all_frigate_files is not None
|
all_frigate_files.get(name, []) if all_frigate_files is not None
|
||||||
else get_frigate_person_files(name)
|
else get_frigate_person_files(name)
|
||||||
|
|||||||
@@ -69,15 +69,13 @@ def process_face_mode(
|
|||||||
output_dir: str,
|
output_dir: str,
|
||||||
count: int,
|
count: int,
|
||||||
min_width: int | None = None,
|
min_width: int | None = None,
|
||||||
insightface_app=None,
|
|
||||||
) -> tuple[int, int] | None:
|
) -> tuple[int, int] | None:
|
||||||
"""Crop face based on Immich metadata and save to output directory.
|
"""Crop face based on Immich metadata and save to output directory.
|
||||||
|
|
||||||
Returns (width, height) of the saved crop, or None if no crop was saved.
|
Returns (width, height) of the saved crop, or None if no crop was saved.
|
||||||
When insightface_app is provided and ENABLE_FACE_ALIGNMENT is True,
|
If face alignment is enabled and landmarks are available, produces
|
||||||
re-detects the face in the Immich bbox region using InsightFace to get
|
an aligned 112x112 crop. Otherwise falls back to bounding box crop
|
||||||
precise landmarks for a proper 112x112 aligned crop. Falls back to
|
with configurable margin.
|
||||||
bounding box crop with configurable margin if alignment is unavailable.
|
|
||||||
"""
|
"""
|
||||||
min_width = min_width or Config.MIN_FACE_WIDTH
|
min_width = min_width or Config.MIN_FACE_WIDTH
|
||||||
|
|
||||||
@@ -111,51 +109,18 @@ def process_face_mode(
|
|||||||
logger.debug(f"Face too small ({face_w:.1f}x{face_h:.1f})")
|
logger.debug(f"Face too small ({face_w:.1f}x{face_h:.1f})")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Re-detect face with InsightFace for landmark-based alignment.
|
# Try face alignment if enabled and landmarks available
|
||||||
# Immich's /api/faces endpoint does not include landmarks, so the
|
|
||||||
# align_face fallback below never fires without this step.
|
|
||||||
if insightface_app is not None and Config.ENABLE_FACE_ALIGNMENT:
|
|
||||||
try:
|
|
||||||
# Expand the Immich bbox by 50% to give InsightFace enough context
|
|
||||||
# for detection and alignment, then search for the face nearest the
|
|
||||||
# centre of that region (handles group photos at the boundary).
|
|
||||||
pad_x, pad_y = face_w * 0.5, face_h * 0.5
|
|
||||||
search_box = (
|
|
||||||
max(0, x1 - pad_x),
|
|
||||||
max(0, y1 - pad_y),
|
|
||||||
min(img_w, x2 + pad_x),
|
|
||||||
min(img_h, y2 + pad_y),
|
|
||||||
)
|
|
||||||
search_crop = img.crop(search_box)
|
|
||||||
detected = insightface_app.get(np.asarray(search_crop))
|
|
||||||
if detected:
|
|
||||||
cx, cy = search_crop.width / 2, search_crop.height / 2
|
|
||||||
best = min(
|
|
||||||
detected,
|
|
||||||
key=lambda f: abs((f.bbox[0] + f.bbox[2]) / 2 - cx)
|
|
||||||
+ abs((f.bbox[1] + f.bbox[3]) / 2 - cy),
|
|
||||||
)
|
|
||||||
kps = getattr(best, "kps", None)
|
|
||||||
if kps is not None and np.asarray(kps).shape == (5, 2):
|
|
||||||
aligned = align_face(search_crop, kps)
|
|
||||||
if aligned is not None:
|
|
||||||
_save_jpeg(aligned, os.path.join(output_dir, f"{count}.jpg"))
|
|
||||||
return aligned.size
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"InsightFace re-detection failed for {asset.get('id')}: {e}")
|
|
||||||
|
|
||||||
# Landmark alignment from Immich metadata (Immich does not currently
|
|
||||||
# expose landmarks, so this path is a future-proofing fallback)
|
|
||||||
if Config.ENABLE_FACE_ALIGNMENT:
|
if Config.ENABLE_FACE_ALIGNMENT:
|
||||||
landmarks = face_info.get("landmarks") or face_info.get("landmark")
|
landmarks = face_info.get("landmarks") or face_info.get("landmark")
|
||||||
if landmarks:
|
if landmarks:
|
||||||
|
# Scale landmarks
|
||||||
scaled_landmarks = [[lm[0] * scale_x, lm[1] * scale_y] for lm in landmarks]
|
scaled_landmarks = [[lm[0] * scale_x, lm[1] * scale_y] for lm in landmarks]
|
||||||
aligned = align_face(img, scaled_landmarks)
|
aligned = align_face(img, scaled_landmarks)
|
||||||
if aligned is not None:
|
if aligned is not None:
|
||||||
_save_jpeg(aligned, os.path.join(output_dir, f"{count}.jpg"))
|
_save_jpeg(aligned, os.path.join(output_dir, f"{count}.jpg"))
|
||||||
return aligned.size
|
return aligned.size
|
||||||
|
|
||||||
# Final fallback: bounding box crop with configurable margin
|
# Fall back to bounding box crop with configurable margin
|
||||||
margin = Config.FACE_MARGIN
|
margin = Config.FACE_MARGIN
|
||||||
margin_x, margin_y = face_w * margin, face_h * margin
|
margin_x, margin_y = face_w * margin, face_h * margin
|
||||||
crop_box = (
|
crop_box = (
|
||||||
|
|||||||
@@ -45,26 +45,6 @@ def get_people() -> list[dict]:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def merge_people(survivor_id: str, merge_ids: list[str]) -> bool:
|
|
||||||
"""Merge duplicate people into survivor via Immich's merge endpoint.
|
|
||||||
|
|
||||||
The survivor (identified by survivor_id) absorbs all faces and assets
|
|
||||||
from the people in merge_ids, which are then removed from Immich.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
resp = requests.put(
|
|
||||||
f"{Config.IMMICH_URL}/api/people/{survivor_id}/merge",
|
|
||||||
headers={**get_headers(), "Content-Type": "application/json"},
|
|
||||||
json={"ids": merge_ids},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
resp.raise_for_status()
|
|
||||||
return True
|
|
||||||
except requests.RequestException as e:
|
|
||||||
logger.error(f"Failed to merge people into {survivor_id}: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_all_assets(person: dict) -> list[dict]:
|
def fetch_all_assets(person: dict) -> list[dict]:
|
||||||
"""Fetch all assets for a person with pagination."""
|
"""Fetch all assets for a person with pagination."""
|
||||||
name = person.get("name", "Unknown")
|
name = person.get("name", "Unknown")
|
||||||
|
|||||||
Reference in New Issue
Block a user