flan/winnow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Immich v2.7.5 compat, supply-chain hardening, and quality fixes (0.5.2) (#23)

Browse Source 
* fix: Immich v2.7.5 compat, supply-chain hardening, and quality fixes (0.5.2)

- Remove assetCount pre-filter broken by Immich v2.7.5 API change; check
  MIN_FACE_COUNT after fetch_all_assets instead
- Replace curl|sh uv installer with COPY --from Docker stage (supply chain)
- Fix HEALTHCHECK to use kill -0 on PID file instead of static file test
- Fix CONFIG_FILE path to resolve inside DATA_DIR for volume persistence
- Fix EmbeddingCache singleton to re-init when cache_dir changes
- Fix fd leak in _suppress_output() with nested finally closes
- Fix silent exception on SQLite connection close in upload_tracker
- Log unexpected Frigate API keys at DEBUG in get_all_frigate_person_files
- Add reconcile FIFO-mapping debug log
- Pin all CI action SHAs; update setup-uv v8.2.0, upload/download-artifact,
  ruff-action v4.0.0

* chore: update lockfile

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
flan
2026-06-14 19:15:20 -04:00
committed by GitHub
co-authored by github-actions[bot]
parent d99d607fc8
commit 8acf8b52b8
17 changed files with 144 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-publish.yml
+21 -21
View File
@@ -54,15 +54,15 @@ jobs:
df -h
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.tag || github.ref }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -79,7 +79,7 @@ jobs:
- name: Build and push by digest
id: build
uses: docker/build-push-action@v7
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
file: ./Dockerfile
@@ -97,7 +97,7 @@ jobs:
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: digest-amd64
path: /tmp/digests/*
@@ -114,17 +114,17 @@ jobs:
steps:
- name: Download digests
uses: actions/download-artifact@v4
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: /tmp/digests
pattern: digest-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -178,18 +178,18 @@ jobs:
df -h
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.tag || github.ref }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -222,7 +222,7 @@ jobs:
fi
- name: Build and push CPU image
uses: docker/build-push-action@v7
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
file: ./Dockerfile
@@ -264,15 +264,15 @@ jobs:
df -h
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.tag || github.ref }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -305,7 +305,7 @@ jobs:
fi
- name: Build and push ROCm image
uses: docker/build-push-action@v7
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
file: ./Dockerfile
@@ -347,15 +347,15 @@ jobs:
df -h
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.tag || github.ref }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -388,7 +388,7 @@ jobs:
fi
- name: Build and push Intel image
uses: docker/build-push-action@v7
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
file: ./Dockerfile
.github/workflows/lint.yml
+2 -2
View File
@@ -23,10 +23,10 @@ jobs:
echo "Disk space freed."
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- name: Run Ruff
uses: astral-sh/ruff-action@v3
uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
with:
args: "check"
.github/workflows/release.yml
+3 -3
View File
@@ -32,12 +32,12 @@ jobs:
echo "Disk space freed."
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@v7
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
- name: Set up Python
run: uv python install 3.13
@@ -100,7 +100,7 @@ jobs:
fi
- name: Create GitHub Release
uses: actions/github-script@v9
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
RELEASE_TAG: ${{ steps.tag.outputs.TAG }}
RELEASE_NOTES: ${{ steps.changelog.outputs.NOTES }}
.github/workflows/test.yml
+2 -2
View File
@@ -13,10 +13,10 @@ jobs:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- name: Install uv
uses: astral-sh/setup-uv@v7
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
- name: Set up Python
run: uv python install 3.13
.github/workflows/update-lockfile.yml
+2 -2
View File
@@ -16,10 +16,10 @@ jobs:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- name: Install uv
uses: astral-sh/setup-uv@v7
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
- name: Set up Python
run: uv python install 3.13
CHANGELOG.md
+24
View File
@@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [0.5.2] - 2026-06-14
### Fixed
- **Immich v2.7.5 compatibility**: `auto_configure` no longer pre-filters people by `assetCount` from the `/api/people` response, which Immich v2.7.5 dropped. The `MIN_FACE_COUNT` check now runs after `fetch_all_assets` so the actual asset count is used instead of the missing field.
- **Dockerfile supply-chain**: replaced `curl | sh` uv installer with `COPY --from=ghcr.io/astral-sh/uv:0.11.21` to eliminate the network-executed script.
- **HEALTHCHECK**: replaced the static file-existence check with `kill -0 $(cat /tmp/winnow.pid)` so the container reports unhealthy when the scheduler process actually dies, not just when a script file is missing.
- **`CONFIG_FILE` volume safety**: the config file path now resolves to `DATA_DIR/.immich_config.json` so it persists across container restarts. The legacy CWD location is still read as a fallback for existing setups.
- **EmbeddingCache singleton isolation**: `get_cache()` now tracks the `cache_dir` argument and re-creates the cache when it changes, preventing test runs from sharing state across different `DATA_DIR` values.
- **File descriptor leak in `_suppress_output()`**: `devnull_fd`, `saved_out`, and `saved_err` are now all closed in a nested `finally` chain, preventing fd exhaustion on long runs.
- **Silent exception in `upload_tracker`**: `except Exception: pass` on SQLite connection close is now `except Exception as e: logger.debug(...)` so connection errors are visible in debug logs.
- **Frigate API unknown-key logging**: `get_all_frigate_person_files` now logs unexpected non-list keys at DEBUG level instead of silently skipping them.
- **Reconcile debug log**: added a debug log entry before the FIFO timestamp mapping step in `reconcile_frigate_mappings` to make the mapping assumption visible in logs.
- **CI action SHA pinning**: all five GitHub Actions workflows now pin every third-party action to a full commit SHA. Updated `setup-uv` v7→v8.2.0, `upload-artifact` v4→v7.0.1, `download-artifact` v4→v8.0.1, `ruff-action` v3→v4.0.0.
## [0.5.1] - 2026-06-14
### Changed
Dockerfile
+3 -3
View File
@@ -38,8 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/* \
&& ln -sf /usr/bin/python3.13 /usr/bin/python3
RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
&& cp /root/.local/bin/uv /usr/local/bin/uv
COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /usr/local/bin/uv
WORKDIR /app
@@ -130,5 +129,6 @@ USER appuser
# the dist-info. Explicitly adding /app lets Python find winnow/__init__.py there.
ENV INSIGHTFACE_HOME=/models/.insightface PYTHONPATH=/app
HEALTHCHECK CMD test -f /app/entrypoint.sh || exit 1
HEALTHCHECK --interval=60s --timeout=5s --start-period=120s --retries=3 \
CMD sh -c 'if [ -f /tmp/winnow.pid ]; then kill -0 "$(cat /tmp/winnow.pid)"; fi'
ENTRYPOINT ["tini", "--", "/app/entrypoint.sh"]
pyproject.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "winnow"
version = "0.5.1"
version = "0.5.2"
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
license = "AGPL-3.0-or-later"
requires-python = ">=3.13"
scheduler.py
+34 -24
View File
@@ -15,36 +15,46 @@ except ImportError:
# resident in memory across all subsequent scheduled runs.
from winnow.cli import main
SCHEDULE = os.environ["CRON_SCHEDULE"]
INSIGHTFACE_HOME = os.environ.get("INSIGHTFACE_HOME", "/models/.insightface")
logger = logging.getLogger(__name__)
def check_models() -> None:
buffalo = Path(INSIGHTFACE_HOME) / "models" / "buffalo_l"
def _check_models() -> None:
insightface_home = os.environ.get("INSIGHTFACE_HOME", "/models/.insightface")
buffalo = Path(insightface_home) / "models" / "buffalo_l"
if not buffalo.exists():
print(" InsightFace Buffalo_L not found — will download on first run", flush=True)
NOW = time.time()
cron = croniter(SCHEDULE, NOW)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
def _run_scheduler() -> None:
schedule = os.environ.get("CRON_SCHEDULE")
if not schedule:
print("Error: CRON_SCHEDULE environment variable is required.", flush=True)
sys.exit(1)
Path("/tmp/winnow.pid").write_text(str(os.getpid()))
while True:
now = time.time()
if now >= next_run:
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
check_models()
try:
main()
print("winnow run complete", flush=True)
except KeyboardInterrupt:
raise
except Exception as e:
logger.error("winnow run failed: %s", e, exc_info=True)
print(f"winnow run failed: {e}", flush=True)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
time.sleep(max(1, next_run - time.time()))
cron = croniter(schedule, now)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
while True:
now = time.time()
if now >= next_run:
print(f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting winnow run...", flush=True)
_check_models()
try:
main()
print("winnow run complete", flush=True)
except KeyboardInterrupt:
raise
except Exception as e:
logger.error("winnow run failed: %s", e, exc_info=True)
print(f"winnow run failed: {e}", flush=True)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
time.sleep(max(1, next_run - time.time()))
if __name__ == "__main__":
_run_scheduler()
uv.lock
Generated
+1 -1
View File
@@ -862,7 +862,7 @@ wheels = [
[[package]]
name = "winnow"
version = "0.5.1"
version = "0.5.2"
source = { editable = "." }
dependencies = [
{ name = "croniter" },
winnow/cache.py
+7 -6
View File
@@ -101,17 +101,18 @@ class EmbeddingCache:
# Singleton instance
_cache: EmbeddingCache | None = None
_cache_dir: str | None = None
def get_cache(cache_dir: str = ".if_cache") -> EmbeddingCache:
"""Get or create the singleton cache instance.
Note: The ``cache_dir`` parameter is only used when creating the
singleton for the first time. Subsequent calls return the existing
instance regardless of ``cache_dir``. If you need a cache with a
different directory, instantiate ``EmbeddingCache`` directly.
Re-creates the instance when ``cache_dir`` changes so that test
isolation (which resets Config.DATA_DIR via _Config.reset()) always
writes to the correct directory rather than a stale one.
"""
global _cache
if _cache is None:
global _cache, _cache_dir
if _cache is None or _cache_dir != cache_dir:
_cache = EmbeddingCache(cache_dir)
_cache_dir = cache_dir
return _cache
winnow/config.py
+13 -5
View File
@@ -9,7 +9,7 @@ from typing import ClassVar
from dotenv import load_dotenv
from rich.prompt import Prompt
CONFIG_FILE = Path(".immich_config.json")
_LEGACY_CONFIG_FILE = Path(".immich_config.json") # pre-v0.6: lived in process CWD, not on a volume
class _Config:
@@ -113,9 +113,13 @@ class _Config:
# Fall back to config file only when the env var is genuinely absent (None).
# An explicitly empty env var (IMMICH_URL="") takes priority over the file.
if CONFIG_FILE.exists():
# Prefer DATA_DIR/.immich_config.json (volume-safe in Docker) and fall back
# to the legacy CWD path so existing installations continue to work.
_data_cfg = Path(self.DATA_DIR) / ".immich_config.json"
config_file = _data_cfg if _data_cfg.exists() else _LEGACY_CONFIG_FILE
if config_file.exists():
try:
data = json.loads(CONFIG_FILE.read_text())
data = json.loads(config_file.read_text())
if self.IMMICH_URL is None:
self.IMMICH_URL = data.get("IMMICH_URL")
if os.getenv("OUTPUT_DIR") is None:
@@ -135,9 +139,13 @@ class _Config:
API_KEY is intentionally excluded — store it in .env or as an
environment variable instead of a plain-text config file.
Writes to DATA_DIR/.immich_config.json so the file survives container
restarts when DATA_DIR is a mounted volume.
"""
config_file = Path(self.DATA_DIR) / ".immich_config.json"
try:
CONFIG_FILE.write_text(
Path(self.DATA_DIR).mkdir(parents=True, exist_ok=True)
config_file.write_text(
json.dumps(
{
"IMMICH_URL": self.IMMICH_URL,
@@ -146,7 +154,7 @@ class _Config:
indent=2,
)
)
logging.info("Configuration saved to %s", CONFIG_FILE)
logging.info("Configuration saved to %s", config_file)
except OSError as e:
logging.error("Failed to save config: %s", e)
winnow/embeddings.py
+6 -4
View File
@@ -35,10 +35,12 @@ def _suppress_output():
try:
os.dup2(saved_out, 1)
finally:
os.dup2(saved_err, 2)
os.close(devnull_fd)
os.close(saved_out)
os.close(saved_err)
try:
os.dup2(saved_err, 2)
finally:
os.close(devnull_fd)
os.close(saved_out)
os.close(saved_err)
# Lazy-loaded singleton
winnow/frigate_api.py
+10 -8
View File
@@ -52,14 +52,16 @@ def get_all_frigate_person_files() -> dict[str, list[str]] | None:
# Response: {person_name: [file, ...], "train": [...], ...}
# "train" is a flat pending list, not a person — skip it.
# TODO(frigate-api): "train" is the only known special key as of Frigate v0.16.
# If Frigate adds other top-level non-person keys, they'll be silently treated
# as person names here. Switch to an allowlist or a typed schema when Frigate
# documents its response contract.
return {
name: files
for name, files in data.items()
if name != "train" and isinstance(files, list)
}
# Log unexpected non-list values so future Frigate schema additions are visible.
result = {}
for name, files in data.items():
if name == "train":
continue
if isinstance(files, list):
result[name] = files
else:
logger.debug("Frigate API: skipping unexpected key %r (got %s, not list)", name, type(files).__name__)
return result
def get_frigate_face_counts() -> dict[str, int] | None:
winnow/jobs.py
+7 -8
View File
@@ -239,15 +239,7 @@ def auto_configure(people: list[dict]) -> list[dict]:
if skip:
valid_people = [p for p in valid_people if p["name"] not in skip]
# Filter by minimum face count (Issue #6: previously unimplemented)
min_face_count = Config.MIN_FACE_COUNT
if min_face_count > 0:
valid_people = [p for p in valid_people if p.get("assetCount", 0) >= min_face_count]
if valid_people:
rprint(
f" Filtered to {len(valid_people)} people with"
f" ≥{min_face_count} assets (MIN_FACE_COUNT={min_face_count})"
)
frigate_counts = get_frigate_face_counts()
# Persist each count to tracker so the last known value survives Frigate downtime
@@ -264,6 +256,13 @@ def auto_configure(people: list[dict]) -> list[dict]:
rprint(f" {name}: {len(all_assets)} total, {len(recent_assets)} recent")
# MIN_FACE_COUNT guard: skip people with too few Immich assets.
# Done here (after fetch) rather than upfront because Immich v2.7.5+
# dropped assetCount from the /api/people response.
if min_face_count > 0 and len(all_assets) < min_face_count:
rprint(f" [dim]Skipping {name} ({len(all_assets)} assets < MIN_FACE_COUNT={min_face_count}).[/dim]")
continue
# Enforce MAX_AUTO_IMAGES against the tracked file count only.
# Manually-added Frigate files are invisible to this cap so users can
# curate their own files without shrinking winnow's managed quota.
winnow/reconcile.py
+6
View File
@@ -60,6 +60,12 @@ def reconcile_frigate_mappings(
except (ValueError, IndexError):
return 0.0
logger.debug(
"%s: mapping %s file(s) by filename timestamp — assumes Frigate processes"
" uploads in FIFO order; mapping may be wrong if that ever changes",
person_name,
target,
)
mappings = {
frigate_file: asset_id
for (_, asset_id), frigate_file in zip(uploaded, sorted(new_files, key=_ts))
winnow/upload_tracker.py
+2 -2
View File
@@ -74,8 +74,8 @@ def _get_conn() -> sqlite3.Connection:
if _conn is not None and _conn_path != db_path:
try:
_conn.close()
except Exception:
pass
except Exception as e:
logger.debug("Failed to close previous SQLite connection: %s", e)
_conn = None
if _conn is None: