Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f86c1054a | ||
|
|
634688fc93 | ||
|
|
9f0a78522f | ||
|
|
8d1f5da05a | ||
|
|
a7257cf031 | ||
|
|
326fdbdf38 | ||
|
|
405413490b | ||
|
|
91e0858aa6 | ||
|
|
e67f2d9638 | ||
|
|
71df0e81de | ||
|
|
bbbac18207 |
@@ -2,7 +2,7 @@ name: Publish Docker Image
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: ["main", "dev"]
|
branches: ["dev"]
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- "**.md"
|
- "**.md"
|
||||||
- "docs/**"
|
- "docs/**"
|
||||||
@@ -15,6 +15,16 @@ on:
|
|||||||
- "uv-cpu.lock"
|
- "uv-cpu.lock"
|
||||||
- "uv-rocm.lock"
|
- "uv-rocm.lock"
|
||||||
- "uv-intel.lock"
|
- "uv-intel.lock"
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
tag:
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
description: "Release tag, e.g. v0.4.1 — triggers :latest + versioned image tags"
|
||||||
|
version:
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
description: "Version string without v prefix, e.g. 0.4.1"
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: docker-${{ github.ref }}
|
group: docker-${{ github.ref }}
|
||||||
@@ -50,6 +60,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.tag || github.ref }}
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
if: matrix.platform == 'linux/arm64'
|
if: matrix.platform == 'linux/arm64'
|
||||||
@@ -65,6 +77,15 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Compute build version
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
if [ -n "${{ inputs.version }}" ]; then
|
||||||
|
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Build and push by digest
|
- name: Build and push by digest
|
||||||
id: build
|
id: build
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@v7
|
||||||
@@ -72,6 +93,7 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: ${{ matrix.platform }}
|
platforms: ${{ matrix.platform }}
|
||||||
|
build-args: VERSION=${{ steps.version.outputs.value }}
|
||||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||||
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -120,22 +142,26 @@ jobs:
|
|||||||
- name: Determine image tags
|
- name: Determine image tags
|
||||||
id: tags
|
id: tags
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||||
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev" >> "$GITHUB_OUTPUT"
|
INPUT_TAG="${{ inputs.tag }}"
|
||||||
|
if [ -n "$INPUT_TAG" ]; then
|
||||||
|
echo "tag_args=-t ${IMAGE}:latest -t ${IMAGE}:${INPUT_TAG}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> "$GITHUB_OUTPUT"
|
echo "tag_args=-t ${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Create and push multi-arch manifest
|
- name: Create and push multi-arch manifest
|
||||||
working-directory: /tmp/digests
|
working-directory: /tmp/digests
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools create \
|
docker buildx imagetools create \
|
||||||
-t ${{ steps.tags.outputs.tags }} \
|
${{ steps.tags.outputs.tag_args }} \
|
||||||
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||||
|
|
||||||
- name: Inspect image
|
- name: Inspect image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.tags.outputs.tags }}
|
docker buildx imagetools inspect ${{ steps.tags.outputs.inspect_tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -162,6 +188,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.tag || github.ref }}
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v4
|
uses: docker/setup-qemu-action@v4
|
||||||
@@ -176,13 +204,30 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine CPU image tag
|
- name: Determine CPU image tags
|
||||||
id: cpu-tag
|
id: cpu-tags
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-cpu" >> "$GITHUB_OUTPUT"
|
INPUT_TAG="${{ inputs.tag }}"
|
||||||
|
if [ -n "$INPUT_TAG" ]; then
|
||||||
|
{
|
||||||
|
echo "tags<<EOF"
|
||||||
|
printf '%s\n' "${IMAGE}:cpu" "${IMAGE}:${INPUT_TAG}-cpu"
|
||||||
|
echo "EOF"
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:cpu" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cpu" >> "$GITHUB_OUTPUT"
|
echo "tags=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:dev-cpu" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Compute build version
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
if [ -n "${{ inputs.version }}" ]; then
|
||||||
|
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push CPU image
|
- name: Build and push CPU image
|
||||||
@@ -191,16 +236,18 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
build-args: VARIANT=cpu
|
build-args: |
|
||||||
|
VARIANT=cpu
|
||||||
|
VERSION=${{ steps.version.outputs.value }}
|
||||||
cache-from: type=gha,scope=cpu
|
cache-from: type=gha,scope=cpu
|
||||||
cache-to: type=gha,mode=max,scope=cpu
|
cache-to: type=gha,mode=max,scope=cpu
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.cpu-tag.outputs.tag }}
|
tags: ${{ steps.cpu-tags.outputs.tags }}
|
||||||
|
|
||||||
- name: Inspect CPU image
|
- name: Inspect CPU image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.cpu-tag.outputs.tag }}
|
docker buildx imagetools inspect ${{ steps.cpu-tags.outputs.inspect_tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -227,6 +274,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.tag || github.ref }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -238,13 +287,30 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine ROCm image tag
|
- name: Determine ROCm image tags
|
||||||
id: rocm-tag
|
id: rocm-tags
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-rocm" >> "$GITHUB_OUTPUT"
|
INPUT_TAG="${{ inputs.tag }}"
|
||||||
|
if [ -n "$INPUT_TAG" ]; then
|
||||||
|
{
|
||||||
|
echo "tags<<EOF"
|
||||||
|
printf '%s\n' "${IMAGE}:rocm" "${IMAGE}:${INPUT_TAG}-rocm"
|
||||||
|
echo "EOF"
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:rocm" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:rocm" >> "$GITHUB_OUTPUT"
|
echo "tags=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:dev-rocm" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Compute build version
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
if [ -n "${{ inputs.version }}" ]; then
|
||||||
|
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push ROCm image
|
- name: Build and push ROCm image
|
||||||
@@ -253,16 +319,18 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
build-args: VARIANT=rocm
|
build-args: |
|
||||||
|
VARIANT=rocm
|
||||||
|
VERSION=${{ steps.version.outputs.value }}
|
||||||
cache-from: type=gha,scope=linux/amd64-rocm
|
cache-from: type=gha,scope=linux/amd64-rocm
|
||||||
cache-to: type=gha,mode=max,scope=linux/amd64-rocm
|
cache-to: type=gha,mode=max,scope=linux/amd64-rocm
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.rocm-tag.outputs.tag }}
|
tags: ${{ steps.rocm-tags.outputs.tags }}
|
||||||
|
|
||||||
- name: Inspect ROCm image
|
- name: Inspect ROCm image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.rocm-tag.outputs.tag }}
|
docker buildx imagetools inspect ${{ steps.rocm-tags.outputs.inspect_tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
@@ -289,6 +357,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.tag || github.ref }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -300,13 +370,30 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Determine Intel image tag
|
- name: Determine Intel image tags
|
||||||
id: intel-tag
|
id: intel-tags
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ github.ref_name }}" = "dev" ]; then
|
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev-intel" >> "$GITHUB_OUTPUT"
|
INPUT_TAG="${{ inputs.tag }}"
|
||||||
|
if [ -n "$INPUT_TAG" ]; then
|
||||||
|
{
|
||||||
|
echo "tags<<EOF"
|
||||||
|
printf '%s\n' "${IMAGE}:intel" "${IMAGE}:${INPUT_TAG}-intel"
|
||||||
|
echo "EOF"
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:intel" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:intel" >> "$GITHUB_OUTPUT"
|
echo "tags=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "inspect_tag=${IMAGE}:dev-intel" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Compute build version
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
if [ -n "${{ inputs.version }}" ]; then
|
||||||
|
echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "value=dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Build and push Intel image
|
- name: Build and push Intel image
|
||||||
@@ -315,16 +402,18 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
build-args: VARIANT=intel
|
build-args: |
|
||||||
|
VARIANT=intel
|
||||||
|
VERSION=${{ steps.version.outputs.value }}
|
||||||
cache-from: type=gha,scope=linux/amd64-intel
|
cache-from: type=gha,scope=linux/amd64-intel
|
||||||
cache-to: type=gha,mode=max,scope=linux/amd64-intel
|
cache-to: type=gha,mode=max,scope=linux/amd64-intel
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.intel-tag.outputs.tag }}
|
tags: ${{ steps.intel-tags.outputs.tags }}
|
||||||
|
|
||||||
- name: Inspect Intel image
|
- name: Inspect Intel image
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.intel-tag.outputs.tag }}
|
docker buildx imagetools inspect ${{ steps.intel-tags.outputs.inspect_tag }}
|
||||||
|
|
||||||
- name: Ensure package is public
|
- name: Ensure package is public
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -127,188 +127,14 @@ jobs:
|
|||||||
prerelease: false,
|
prerelease: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
build-gpu:
|
build-images:
|
||||||
name: Build GPU image
|
name: Build and push Docker images
|
||||||
needs: release
|
needs: release
|
||||||
runs-on: ubuntu-latest
|
uses: ./.github/workflows/docker-publish.yml
|
||||||
|
with:
|
||||||
|
tag: ${{ needs.release.outputs.tag }}
|
||||||
|
version: ${{ needs.release.outputs.version }}
|
||||||
|
secrets: inherit
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
contents: read
|
||||||
- name: Free up disk space
|
|
||||||
run: |
|
|
||||||
sudo rm -rf /usr/share/dotnet
|
|
||||||
sudo rm -rf /opt/ghc
|
|
||||||
sudo rm -rf "/usr/local/share/boost"
|
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
||||||
echo "Disk space freed."
|
|
||||||
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push GPU image (latest)
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
platforms: linux/amd64,linux/arm64
|
|
||||||
push: true
|
|
||||||
build-args: VERSION=${{ needs.release.outputs.version }}
|
|
||||||
cache-from: type=gha,scope=release-gpu
|
|
||||||
cache-to: type=gha,mode=max,scope=release-gpu
|
|
||||||
tags: |
|
|
||||||
ghcr.io/sudolulo/winnow:latest
|
|
||||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}
|
|
||||||
|
|
||||||
build-cpu:
|
|
||||||
name: Build CPU image
|
|
||||||
needs: release
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- name: Free up disk space
|
|
||||||
run: |
|
|
||||||
sudo rm -rf /usr/share/dotnet
|
|
||||||
sudo rm -rf /opt/ghc
|
|
||||||
sudo rm -rf "/usr/local/share/boost"
|
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
||||||
echo "Disk space freed."
|
|
||||||
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push CPU image
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
platforms: linux/amd64,linux/arm64
|
|
||||||
push: true
|
|
||||||
build-args: |
|
|
||||||
VARIANT=cpu
|
|
||||||
VERSION=${{ needs.release.outputs.version }}
|
|
||||||
cache-from: type=gha,scope=release-cpu
|
|
||||||
cache-to: type=gha,mode=max,scope=release-cpu
|
|
||||||
tags: |
|
|
||||||
ghcr.io/sudolulo/winnow:cpu
|
|
||||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-cpu
|
|
||||||
|
|
||||||
build-rocm:
|
|
||||||
name: Build ROCm image
|
|
||||||
needs: release
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- name: Free up disk space
|
|
||||||
run: |
|
|
||||||
sudo rm -rf /usr/share/dotnet
|
|
||||||
sudo rm -rf /opt/ghc
|
|
||||||
sudo rm -rf "/usr/local/share/boost"
|
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
||||||
echo "Disk space freed."
|
|
||||||
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push ROCm image
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
platforms: linux/amd64
|
|
||||||
push: true
|
|
||||||
build-args: |
|
|
||||||
VARIANT=rocm
|
|
||||||
VERSION=${{ needs.release.outputs.version }}
|
|
||||||
cache-from: type=gha,scope=release-rocm
|
|
||||||
cache-to: type=gha,mode=max,scope=release-rocm
|
|
||||||
tags: |
|
|
||||||
ghcr.io/sudolulo/winnow:rocm
|
|
||||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-rocm
|
|
||||||
|
|
||||||
build-intel:
|
|
||||||
name: Build Intel image
|
|
||||||
needs: release
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- name: Free up disk space
|
|
||||||
run: |
|
|
||||||
sudo rm -rf /usr/share/dotnet
|
|
||||||
sudo rm -rf /opt/ghc
|
|
||||||
sudo rm -rf "/usr/local/share/boost"
|
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
||||||
echo "Disk space freed."
|
|
||||||
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push Intel image
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
platforms: linux/amd64
|
|
||||||
push: true
|
|
||||||
build-args: |
|
|
||||||
VARIANT=intel
|
|
||||||
VERSION=${{ needs.release.outputs.version }}
|
|
||||||
cache-from: type=gha,scope=release-intel
|
|
||||||
cache-to: type=gha,mode=max,scope=release-intel
|
|
||||||
tags: |
|
|
||||||
ghcr.io/sudolulo/winnow:intel
|
|
||||||
ghcr.io/sudolulo/winnow:${{ needs.release.outputs.tag }}-intel
|
|
||||||
|
|||||||
@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.4.2] - 2026-06-13
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **GPU image now uses CUDA 12.8.1** (was 13.3): CUDA 13.3 requires driver ≥ 575; driver 570 (the current stable release) was incorrectly rejected with "CUDA driver version is insufficient" at startup. The `:latest` image now works with any NVIDIA driver ≥ 570.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **`scripts/benchmark.py`**: measures InsightFace and SigLIP inference latency and throughput across GPU and CPU modes. Run inside the container with `python /app/scripts/benchmark.py`. RTX 2070 SUPER results: InsightFace 12.8 ms / 78 img/s (8× CPU), SigLIP batch 32 at 5.4 ms/img / 187 img/s (33× CPU).
|
||||||
|
|
||||||
|
## [0.4.1] - 2026-06-13
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **`RESET_PERSON` no longer creates duplicate Frigate files**: previously, resetting a person only wiped the local tracker — existing Frigate training files were left as unmanaged orphans, causing the next run to upload a full new batch on top of them. `reset_person` now deletes all winnow-managed files for that person from Frigate before clearing the tracker. Manually-added Frigate files are unaffected.
|
||||||
|
- **No spurious warning when `FRIGATE_URL` is unset and `RESET_PERSON` is used**: the deletion step is now skipped silently at info level rather than logging a misleading "could not delete" warning.
|
||||||
|
|
||||||
## [0.4.0] - 2026-06-13
|
## [0.4.0] - 2026-06-13
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
# ── Base images ───────────────────────────────────────────────────────────────
|
# ── Base images ───────────────────────────────────────────────────────────────
|
||||||
# amd64 + gpu: NVIDIA CUDA 13.3 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
# amd64 + gpu: NVIDIA CUDA 12.8 + cuDNN (GPU acceleration via NVIDIA Container Toolkit)
|
||||||
# amd64 + rocm: Ubuntu 22.04 (AMD GPU via ROCm — pass /dev/kfd and /dev/dri)
|
# amd64 + rocm: Ubuntu 22.04 (AMD GPU via ROCm — pass /dev/kfd and /dev/dri)
|
||||||
# amd64 + intel: Ubuntu 22.04 (Intel Arc / iGPU via OpenVINO — pass /dev/dri)
|
# amd64 + intel: Ubuntu 22.04 (Intel Arc / iGPU via OpenVINO — pass /dev/dri)
|
||||||
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
# amd64 + cpu: Ubuntu 22.04 (CPU-only, ~2 GB smaller image)
|
||||||
@@ -7,7 +7,7 @@
|
|||||||
|
|
||||||
ARG VARIANT=gpu
|
ARG VARIANT=gpu
|
||||||
|
|
||||||
FROM --platform=$BUILDPLATFORM nvidia/cuda:13.3.0-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
FROM --platform=$BUILDPLATFORM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04 AS base-amd64-gpu
|
||||||
FROM ubuntu:22.04 AS base-amd64-rocm
|
FROM ubuntu:22.04 AS base-amd64-rocm
|
||||||
FROM ubuntu:22.04 AS base-amd64-intel
|
FROM ubuntu:22.04 AS base-amd64-intel
|
||||||
FROM ubuntu:22.04 AS base-amd64-cpu
|
FROM ubuntu:22.04 AS base-amd64-cpu
|
||||||
|
|||||||
@@ -11,6 +11,8 @@
|
|||||||
|
|
||||||
Frigate's face recognition is only as good as its training data — and the key quality metric is **diversity**, not volume. A hundred photos from the same week teach the model one lighting condition. What you need is a spread: different years, different angles, different lighting, different contexts. Your photo library already has that data. winnow finds and delivers the right subset automatically.
|
Frigate's face recognition is only as good as its training data — and the key quality metric is **diversity**, not volume. A hundred photos from the same week teach the model one lighting condition. What you need is a spread: different years, different angles, different lighting, different contexts. Your photo library already has that data. winnow finds and delivers the right subset automatically.
|
||||||
|
|
||||||
|
> **winnow only touches files it uploaded.** Faces added to Frigate manually through its UI are never deleted, replaced, or modified — not by quality replacement, not by `RESET_PERSON`, not by stale cleanup. If you have a curated training set you want to keep, it is safe.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
@@ -222,7 +224,7 @@ In scheduled mode the process (and loaded models) stays resident between runs. T
|
|||||||
| :--- | :--- | :--- |
|
| :--- | :--- | :--- |
|
||||||
| `DRY_RUN` | `false` | Preview selection without downloading or uploading |
|
| `DRY_RUN` | `false` | Preview selection without downloading or uploading |
|
||||||
| `RETRY_REJECTED` | `false` | Re-attempt assets previously rejected by Frigate |
|
| `RETRY_REJECTED` | `false` | Re-attempt assets previously rejected by Frigate |
|
||||||
| `RESET_PERSON` | *(unset)* | Clear upload and rejection history for one person by name |
|
| `RESET_PERSON` | *(unset)* | Clear upload history for one person and delete their winnow-managed Frigate training files so the next run starts fresh. Manually added Frigate files are never touched |
|
||||||
|
|
||||||
### Scheduling
|
### Scheduling
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.4.0"
|
version = "0.4.2"
|
||||||
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
|
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification."
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
winnow inference benchmark: GPU vs CPU throughput.
|
||||||
|
|
||||||
|
Measures InsightFace (face mode) and SigLIP (object mode) latency and
|
||||||
|
throughput. Run with FORCE_CPU=true for CPU-only baseline.
|
||||||
|
|
||||||
|
Usage inside container:
|
||||||
|
# GPU mode:
|
||||||
|
docker exec winnow python /app/scripts/benchmark.py
|
||||||
|
|
||||||
|
# CPU mode:
|
||||||
|
docker exec -e FORCE_CPU=true winnow python /app/scripts/benchmark.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from PIL import Image, ImageDraw
|
||||||
|
|
||||||
|
|
||||||
|
def _mode_label() -> str:
|
||||||
|
if os.getenv("FORCE_CPU", "").lower() in ("true", "1", "yes"):
|
||||||
|
return "CPU (FORCE_CPU=true)"
|
||||||
|
return "GPU (auto)"
|
||||||
|
|
||||||
|
|
||||||
|
def make_face_image(size: int = 640) -> Image.Image:
|
||||||
|
"""Synthetic face-like image: skin-tone rectangle with landmark blobs."""
|
||||||
|
img = Image.new("RGB", (size, size), (200, 170, 140))
|
||||||
|
draw = ImageDraw.Draw(img)
|
||||||
|
# Head oval
|
||||||
|
cx, cy = size // 2, size // 2
|
||||||
|
hw, hh = int(size * 0.3), int(size * 0.38)
|
||||||
|
draw.ellipse([cx - hw, cy - hh, cx + hw, cy + hh], fill=(220, 185, 155))
|
||||||
|
# Eyes
|
||||||
|
for ex in [cx - int(size * 0.1), cx + int(size * 0.1)]:
|
||||||
|
ey = cy - int(size * 0.05)
|
||||||
|
r = max(4, size // 40)
|
||||||
|
draw.ellipse([ex - r, ey - r, ex + r, ey + r], fill=(40, 30, 20))
|
||||||
|
# Nose
|
||||||
|
draw.ellipse([cx - 5, cy + 5, cx + 5, cy + 15], fill=(180, 140, 110))
|
||||||
|
# Mouth
|
||||||
|
draw.arc([cx - 20, cy + 25, cx + 20, cy + 45], start=0, end=180, fill=(160, 80, 80), width=3)
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def make_random_image(width: int = 224, height: int = 224) -> Image.Image:
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8), "RGB")
|
||||||
|
|
||||||
|
|
||||||
|
def _stats(times_s: list[float]) -> dict:
|
||||||
|
arr = np.array(times_s) * 1000 # ms
|
||||||
|
return {
|
||||||
|
"median_ms": float(np.median(arr)),
|
||||||
|
"mean_ms": float(np.mean(arr)),
|
||||||
|
"min_ms": float(np.min(arr)),
|
||||||
|
"p95_ms": float(np.percentile(arr, 95)),
|
||||||
|
"ips": 1000.0 / float(np.median(arr)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def bench_insightface(n_warmup: int = 5, n_runs: int = 30) -> None:
|
||||||
|
import cv2
|
||||||
|
|
||||||
|
import winnow.embeddings as emb_mod
|
||||||
|
from winnow.embeddings import get_insightface_app
|
||||||
|
|
||||||
|
# Reset singleton so we get a fresh load
|
||||||
|
emb_mod._insightface_app = None
|
||||||
|
emb_mod._insightface_loaded = False
|
||||||
|
|
||||||
|
print(" Loading model...")
|
||||||
|
t_load = time.perf_counter()
|
||||||
|
app = get_insightface_app()
|
||||||
|
load_s = time.perf_counter() - t_load
|
||||||
|
|
||||||
|
if app is None:
|
||||||
|
print(" SKIP: InsightFace failed to load")
|
||||||
|
return
|
||||||
|
|
||||||
|
img_pil = make_face_image(640)
|
||||||
|
img_bgr = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
|
||||||
|
|
||||||
|
# Warmup
|
||||||
|
for _ in range(n_warmup):
|
||||||
|
app.get(img_bgr)
|
||||||
|
|
||||||
|
# Timed — single image 640×640
|
||||||
|
times: list[float] = []
|
||||||
|
for _ in range(n_runs):
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
app.get(img_bgr)
|
||||||
|
times.append(time.perf_counter() - t0)
|
||||||
|
|
||||||
|
s = _stats(times)
|
||||||
|
print(f" Model load time : {load_s:.2f} s")
|
||||||
|
print(" Input size : 640×640")
|
||||||
|
print(f" Runs : {n_runs} (after {n_warmup} warmup)")
|
||||||
|
print(f" Median latency : {s['median_ms']:.1f} ms")
|
||||||
|
print(f" Mean / p95 : {s['mean_ms']:.1f} ms / {s['p95_ms']:.1f} ms")
|
||||||
|
print(f" Min latency : {s['min_ms']:.1f} ms")
|
||||||
|
print(f" Throughput : {s['ips']:.1f} images/s")
|
||||||
|
|
||||||
|
# Also test at 320×320
|
||||||
|
img_sm = make_face_image(320)
|
||||||
|
img_sm_bgr = cv2.cvtColor(np.asarray(img_sm), cv2.COLOR_RGB2BGR)
|
||||||
|
for _ in range(n_warmup):
|
||||||
|
app.get(img_sm_bgr)
|
||||||
|
times_sm: list[float] = []
|
||||||
|
for _ in range(n_runs):
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
app.get(img_sm_bgr)
|
||||||
|
times_sm.append(time.perf_counter() - t0)
|
||||||
|
s2 = _stats(times_sm)
|
||||||
|
print(f" 320×320 median : {s2['median_ms']:.1f} ms ({s2['ips']:.1f} img/s)")
|
||||||
|
|
||||||
|
|
||||||
|
def bench_siglip(
|
||||||
|
n_warmup: int = 3,
|
||||||
|
n_runs: int = 20,
|
||||||
|
batch_sizes: tuple = (1, 4, 8, 16, 32),
|
||||||
|
) -> None:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
import winnow.embeddings as emb_mod
|
||||||
|
emb_mod._siglip_model = None
|
||||||
|
emb_mod._siglip_processor = None
|
||||||
|
emb_mod._siglip_loaded = False
|
||||||
|
|
||||||
|
print(" Loading model...")
|
||||||
|
t_load = time.perf_counter()
|
||||||
|
model, processor = emb_mod.get_siglip_model()
|
||||||
|
load_s = time.perf_counter() - t_load
|
||||||
|
|
||||||
|
if model is None:
|
||||||
|
print(" SKIP: SigLIP failed to load")
|
||||||
|
return
|
||||||
|
|
||||||
|
device = next(model.parameters()).device
|
||||||
|
print(f" Model load time : {load_s:.2f} s (device: {device})")
|
||||||
|
|
||||||
|
print(f" {'Batch':>5} {'ms/batch':>10} {'ms/img':>8} {'img/s':>8} {'p95/img':>9}")
|
||||||
|
for bs in batch_sizes:
|
||||||
|
imgs = [make_random_image(224, 224) for _ in range(bs)]
|
||||||
|
inputs = processor(images=imgs, return_tensors="pt")
|
||||||
|
inputs = {k: v.to(device) for k, v in inputs.items()}
|
||||||
|
|
||||||
|
# Warmup
|
||||||
|
for _ in range(n_warmup):
|
||||||
|
with torch.no_grad():
|
||||||
|
model(**inputs)
|
||||||
|
if str(device) != "cpu":
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
|
||||||
|
times: list[float] = []
|
||||||
|
for _ in range(n_runs):
|
||||||
|
if str(device) != "cpu":
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
with torch.no_grad():
|
||||||
|
model(**inputs)
|
||||||
|
if str(device) != "cpu":
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
times.append(time.perf_counter() - t0)
|
||||||
|
|
||||||
|
s = _stats(times)
|
||||||
|
print(
|
||||||
|
f" {bs:>5} {s['median_ms']:>10.1f} {s['median_ms']/bs:>8.2f}"
|
||||||
|
f" {bs * 1000 / s['median_ms']:>8.1f} {s['p95_ms']/bs:>9.2f}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
print("=" * 56)
|
||||||
|
print(" winnow inference benchmark")
|
||||||
|
print(f" Mode: {_mode_label()}")
|
||||||
|
print("=" * 56)
|
||||||
|
print()
|
||||||
|
|
||||||
|
print("── InsightFace Buffalo_L (face detection + ArcFace) ──")
|
||||||
|
bench_insightface()
|
||||||
|
print()
|
||||||
|
|
||||||
|
print("── SigLIP google/siglip-base-patch16-224 (objects) ───")
|
||||||
|
bench_siglip()
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Add winnow to path when run directly inside container
|
||||||
|
sys.path.insert(0, "/app")
|
||||||
|
main()
|
||||||
@@ -2348,7 +2348,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.3.3"
|
version = "0.4.1"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "croniter" },
|
{ name = "croniter" },
|
||||||
|
|||||||
+16
-5
@@ -13,7 +13,12 @@ from rich import print as rprint
|
|||||||
from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
|
from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
|
||||||
|
|
||||||
from .config import Config, get_headers
|
from .config import Config, get_headers
|
||||||
from .frigate_api import delete_frigate_person_files, get_all_frigate_person_files, get_frigate_person_files, recognize_face
|
from .frigate_api import (
|
||||||
|
delete_frigate_person_files,
|
||||||
|
get_all_frigate_person_files,
|
||||||
|
get_frigate_person_files,
|
||||||
|
recognize_face,
|
||||||
|
)
|
||||||
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
from .image_processing import process_face_mode, process_full_mode, process_object_mode
|
||||||
from .immich_api import fetch_face_data, fetch_full_image
|
from .immich_api import fetch_face_data, fetch_full_image
|
||||||
from .log_config import console
|
from .log_config import console
|
||||||
@@ -395,6 +400,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
actually_uploaded: list[tuple[str, str | None]] = []
|
actually_uploaded: list[tuple[str, str | None]] = []
|
||||||
failed_deletes: set[str] = set()
|
failed_deletes: set[str] = set()
|
||||||
min_quality_score_for_slot: float | None = None
|
min_quality_score_for_slot: float | None = None
|
||||||
|
person_has_fscores: bool = has_frigate_scores(name)
|
||||||
|
|
||||||
for fname in person_files:
|
for fname in person_files:
|
||||||
fpath = os.path.join(person_dir, fname)
|
fpath = os.path.join(person_dir, fname)
|
||||||
@@ -427,9 +433,9 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
# handles this conservatively by skipping that candidate until the next run.
|
# handles this conservatively by skipping that candidate until the next run.
|
||||||
pre_fscore: float | None = None
|
pre_fscore: float | None = None
|
||||||
if Config.ENABLE_FRIGATE_SCORES and pre_run_count > 0:
|
if Config.ENABLE_FRIGATE_SCORES and pre_run_count > 0:
|
||||||
if not at_cap or has_frigate_scores(name):
|
if not at_cap or person_has_fscores:
|
||||||
_result = recognize_face(fpath)
|
_result = recognize_face(fpath)
|
||||||
if _result is not None and _result[0] == name:
|
if _result is not None and (_result[0] or "").casefold() == name.casefold():
|
||||||
pre_fscore = _result[1]
|
pre_fscore = _result[1]
|
||||||
|
|
||||||
# Ceiling check: skip if the existing training set already covers this
|
# Ceiling check: skip if the existing training set already covers this
|
||||||
@@ -450,7 +456,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
progress.advance(upload_task)
|
progress.advance(upload_task)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
using_fscore = has_frigate_scores(name) and Config.ENABLE_FRIGATE_SCORES
|
using_fscore = person_has_fscores and Config.ENABLE_FRIGATE_SCORES
|
||||||
if using_fscore:
|
if using_fscore:
|
||||||
candidate_score = pre_fscore
|
candidate_score = pre_fscore
|
||||||
if candidate_score is None:
|
if candidate_score is None:
|
||||||
@@ -476,8 +482,10 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
)
|
)
|
||||||
if delete_frigate_person_files(name, [target_frigate_file]):
|
if delete_frigate_person_files(name, [target_frigate_file]):
|
||||||
remove_frigate_file(name, target_frigate_file)
|
remove_frigate_file(name, target_frigate_file)
|
||||||
|
person_has_fscores = has_frigate_scores(name)
|
||||||
effective_count -= 1
|
effective_count -= 1
|
||||||
min_quality_score_for_slot = None # clear any blur-mode slot floor — Frigate uses a different score metric
|
# clear any blur-mode slot floor — Frigate uses a different score metric
|
||||||
|
min_quality_score_for_slot = None
|
||||||
else:
|
else:
|
||||||
logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement")
|
logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement")
|
||||||
failed_deletes.add(target_frigate_file)
|
failed_deletes.add(target_frigate_file)
|
||||||
@@ -507,6 +515,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
)
|
)
|
||||||
if delete_frigate_person_files(name, [target_frigate_file]):
|
if delete_frigate_person_files(name, [target_frigate_file]):
|
||||||
remove_frigate_file(name, target_frigate_file)
|
remove_frigate_file(name, target_frigate_file)
|
||||||
|
person_has_fscores = has_frigate_scores(name)
|
||||||
effective_count -= 1
|
effective_count -= 1
|
||||||
min_quality_score_for_slot = score_map.get(fname)
|
min_quality_score_for_slot = score_map.get(fname)
|
||||||
else:
|
else:
|
||||||
@@ -538,6 +547,8 @@ def upload_to_frigate(jobs: list[dict]) -> None:
|
|||||||
crop_dims=dims_map.get(fname),
|
crop_dims=dims_map.get(fname),
|
||||||
frigate_score=pre_fscore,
|
frigate_score=pre_fscore,
|
||||||
)
|
)
|
||||||
|
if pre_fscore is not None:
|
||||||
|
person_has_fscores = True
|
||||||
actually_uploaded.append((fname, asset_id))
|
actually_uploaded.append((fname, asset_id))
|
||||||
|
|
||||||
break
|
break
|
||||||
|
|||||||
+14
-18
@@ -22,24 +22,6 @@ def _get_faces_data() -> dict | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_frigate_face_counts() -> dict[str, int] | None:
|
|
||||||
"""Return {person_name: training_image_count} from Frigate's train directory.
|
|
||||||
|
|
||||||
Returns None if FRIGATE_URL is not set or the API is unreachable, so callers
|
|
||||||
can distinguish "API unavailable" from "person has 0 images."
|
|
||||||
"""
|
|
||||||
data = _get_faces_data()
|
|
||||||
if data is None:
|
|
||||||
return None
|
|
||||||
# Response: {person_name: [file, ...], "train": [...], ...}
|
|
||||||
# "train" is a flat pending list, not a person — skip it.
|
|
||||||
return {
|
|
||||||
name: len(files)
|
|
||||||
for name, files in data.items()
|
|
||||||
if name != "train" and isinstance(files, list)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def get_all_frigate_person_files() -> dict[str, list[str]] | None:
|
def get_all_frigate_person_files() -> dict[str, list[str]] | None:
|
||||||
"""Return {person_name: [filename, ...]} for every person in Frigate.
|
"""Return {person_name: [filename, ...]} for every person in Frigate.
|
||||||
|
|
||||||
@@ -49,6 +31,8 @@ def get_all_frigate_person_files() -> dict[str, list[str]] | None:
|
|||||||
data = _get_faces_data()
|
data = _get_faces_data()
|
||||||
if data is None:
|
if data is None:
|
||||||
return None
|
return None
|
||||||
|
# Response: {person_name: [file, ...], "train": [...], ...}
|
||||||
|
# "train" is a flat pending list, not a person — skip it.
|
||||||
return {
|
return {
|
||||||
name: files
|
name: files
|
||||||
for name, files in data.items()
|
for name, files in data.items()
|
||||||
@@ -56,6 +40,18 @@ def get_all_frigate_person_files() -> dict[str, list[str]] | None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_frigate_face_counts() -> dict[str, int] | None:
|
||||||
|
"""Return {person_name: training_image_count} from Frigate's train directory.
|
||||||
|
|
||||||
|
Returns None if FRIGATE_URL is not set or the API is unreachable, so callers
|
||||||
|
can distinguish "API unavailable" from "person has 0 images."
|
||||||
|
"""
|
||||||
|
all_files = get_all_frigate_person_files()
|
||||||
|
if all_files is None:
|
||||||
|
return None
|
||||||
|
return {name: len(files) for name, files in all_files.items()}
|
||||||
|
|
||||||
|
|
||||||
def get_frigate_person_files(person_name: str) -> list[str] | None:
|
def get_frigate_person_files(person_name: str) -> list[str] | None:
|
||||||
"""Return the list of training filenames for a person in Frigate.
|
"""Return the list of training filenames for a person in Frigate.
|
||||||
|
|
||||||
|
|||||||
+53
-38
@@ -29,8 +29,11 @@ Frigate's UI are never mapped here and are never touched by quality replacement.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .frigate_api import delete_frigate_person_files
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
UPLOAD_TRACKER_FILE = "frigate_uploaded_ids.json"
|
UPLOAD_TRACKER_FILE = "frigate_uploaded_ids.json"
|
||||||
@@ -167,7 +170,9 @@ def remove_frigate_file(person_name: str, frigate_filename: str) -> None:
|
|||||||
data = _load(UPLOAD_TRACKER_FILE)
|
data = _load(UPLOAD_TRACKER_FILE)
|
||||||
by_person = data.get("by_person", {})
|
by_person = data.get("by_person", {})
|
||||||
entry = _migrate_entry(by_person.get(person_name, {}))
|
entry = _migrate_entry(by_person.get(person_name, {}))
|
||||||
entry["frigate_files"].pop(frigate_filename, None)
|
asset_id = entry["frigate_files"].pop(frigate_filename, None)
|
||||||
|
if asset_id:
|
||||||
|
entry["frigate_scores"].pop(asset_id, None)
|
||||||
by_person[person_name] = entry
|
by_person[person_name] = entry
|
||||||
_save(UPLOAD_TRACKER_FILE, data)
|
_save(UPLOAD_TRACKER_FILE, data)
|
||||||
logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})")
|
logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})")
|
||||||
@@ -204,6 +209,22 @@ def has_frigate_scores(person_name: str) -> bool:
|
|||||||
return any(asset_id in frigate_scores for asset_id in frigate_files.values())
|
return any(asset_id in frigate_scores for asset_id in frigate_files.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_mapped_file(
|
||||||
|
person_name: str, score_key: str, *, highest: bool, exclude: set[str] | None = None
|
||||||
|
) -> tuple[str, str, float] | None:
|
||||||
|
data = _load(UPLOAD_TRACKER_FILE)
|
||||||
|
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
||||||
|
scores = entry.get(score_key, {})
|
||||||
|
candidates = [
|
||||||
|
(ff, asset_id, scores[asset_id])
|
||||||
|
for ff, asset_id in entry.get("frigate_files", {}).items()
|
||||||
|
if (exclude is None or ff not in exclude) and asset_id in scores
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
return max(candidates, key=lambda x: x[2]) if highest else min(candidates, key=lambda x: x[2])
|
||||||
|
|
||||||
|
|
||||||
def get_lowest_quality_mapped_file(
|
def get_lowest_quality_mapped_file(
|
||||||
person_name: str, exclude: set[str] | None = None
|
person_name: str, exclude: set[str] | None = None
|
||||||
) -> tuple[str, str, float] | None:
|
) -> tuple[str, str, float] | None:
|
||||||
@@ -213,21 +234,7 @@ def get_lowest_quality_mapped_file(
|
|||||||
Used for quality replacement when no Frigate scores are available.
|
Used for quality replacement when no Frigate scores are available.
|
||||||
Pass `exclude` to skip files that failed to delete this run.
|
Pass `exclude` to skip files that failed to delete this run.
|
||||||
"""
|
"""
|
||||||
data = _load(UPLOAD_TRACKER_FILE)
|
return _pick_mapped_file(person_name, "scores", highest=False, exclude=exclude)
|
||||||
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
|
||||||
frigate_files = entry.get("frigate_files", {})
|
|
||||||
blur_scores = entry.get("scores", {})
|
|
||||||
|
|
||||||
candidates = [
|
|
||||||
(ff, asset_id, blur_scores[asset_id])
|
|
||||||
for ff, asset_id in frigate_files.items()
|
|
||||||
if (exclude is None or ff not in exclude)
|
|
||||||
and asset_id in blur_scores
|
|
||||||
]
|
|
||||||
|
|
||||||
if not candidates:
|
|
||||||
return None
|
|
||||||
return min(candidates, key=lambda x: x[2])
|
|
||||||
|
|
||||||
|
|
||||||
def get_most_redundant_mapped_file(
|
def get_most_redundant_mapped_file(
|
||||||
@@ -240,21 +247,7 @@ def get_most_redundant_mapped_file(
|
|||||||
= the most redundant file and therefore the best replacement target.
|
= the most redundant file and therefore the best replacement target.
|
||||||
Pass `exclude` to skip files that failed to delete this run.
|
Pass `exclude` to skip files that failed to delete this run.
|
||||||
"""
|
"""
|
||||||
data = _load(UPLOAD_TRACKER_FILE)
|
return _pick_mapped_file(person_name, "frigate_scores", highest=True, exclude=exclude)
|
||||||
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
|
|
||||||
frigate_files = entry.get("frigate_files", {})
|
|
||||||
frigate_scores = entry.get("frigate_scores", {})
|
|
||||||
|
|
||||||
candidates = [
|
|
||||||
(ff, asset_id, frigate_scores[asset_id])
|
|
||||||
for ff, asset_id in frigate_files.items()
|
|
||||||
if (exclude is None or ff not in exclude)
|
|
||||||
and asset_id in frigate_scores
|
|
||||||
]
|
|
||||||
|
|
||||||
if not candidates:
|
|
||||||
return None
|
|
||||||
return max(candidates, key=lambda x: x[2])
|
|
||||||
|
|
||||||
|
|
||||||
def get_frigate_filename_for_asset(person_name: str, asset_id: str) -> str | None:
|
def get_frigate_filename_for_asset(person_name: str, asset_id: str) -> str | None:
|
||||||
@@ -307,19 +300,41 @@ def update_frigate_count(person_name: str, count: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def reset_person(person_name: str) -> None:
|
def reset_person(person_name: str) -> None:
|
||||||
"""Remove all uploaded and rejected records for a given person."""
|
"""Remove all uploaded and rejected records for a given person.
|
||||||
for filename in (UPLOAD_TRACKER_FILE, REJECT_TRACKER_FILE):
|
|
||||||
data = _load(filename)
|
Also deletes winnow-managed Frigate training files so the next run starts
|
||||||
|
clean rather than uploading on top of orphaned files. Manually-added Frigate
|
||||||
|
files (not in frigate_files) are never touched. Proceeds with tracker reset
|
||||||
|
even if Frigate is unreachable.
|
||||||
|
"""
|
||||||
|
upload_data = _load(UPLOAD_TRACKER_FILE)
|
||||||
|
entry = _migrate_entry(upload_data.get("by_person", {}).get(person_name, {}))
|
||||||
|
frigate_filenames = list(entry.get("frigate_files", {}).keys())
|
||||||
|
if frigate_filenames:
|
||||||
|
if not os.environ.get("FRIGATE_URL", "").strip():
|
||||||
|
logger.info(f"FRIGATE_URL not set — skipping Frigate file deletion for {person_name}")
|
||||||
|
elif delete_frigate_person_files(person_name, frigate_filenames):
|
||||||
|
logger.info(f"Deleted {len(frigate_filenames)} Frigate file(s) for {person_name}")
|
||||||
|
else:
|
||||||
|
logger.warning(f"Could not delete Frigate files for {person_name} — tracker reset proceeding anyway")
|
||||||
|
|
||||||
|
changed = False
|
||||||
|
tracker_files = ((UPLOAD_TRACKER_FILE, upload_data), (REJECT_TRACKER_FILE, _load(REJECT_TRACKER_FILE)))
|
||||||
|
for filename, data in tracker_files:
|
||||||
flat_key = _flat_key(filename)
|
flat_key = _flat_key(filename)
|
||||||
by_person = data.get("by_person", {})
|
by_person = data.get("by_person", {})
|
||||||
entry = by_person.pop(person_name, None)
|
tracker_entry = by_person.pop(person_name, None)
|
||||||
if entry is not None:
|
if tracker_entry is not None:
|
||||||
person_ids = set(_get_ids(entry))
|
person_ids = set(_get_ids(tracker_entry))
|
||||||
flat = set(data.get(flat_key, [])) - person_ids
|
flat = set(data.get(flat_key, [])) - person_ids
|
||||||
data[flat_key] = sorted(flat)
|
data[flat_key] = sorted(flat)
|
||||||
data["by_person"] = by_person
|
data["by_person"] = by_person
|
||||||
_save(filename, data)
|
_save(filename, data)
|
||||||
logger.info(f"Reset tracking data for {person_name}")
|
changed = True
|
||||||
|
if changed:
|
||||||
|
logger.info(f"Reset tracking data for {person_name}")
|
||||||
|
else:
|
||||||
|
logger.debug(f"reset_person: no tracking data found for {person_name}")
|
||||||
|
|
||||||
|
|
||||||
def get_person_summary() -> dict[str, dict]:
|
def get_person_summary() -> dict[str, dict]:
|
||||||
|
|||||||
Reference in New Issue
Block a user