From cc092ec97227794958019820823846f1e0989974 Mon Sep 17 00:00:00 2001 From: Holden Date: Sun, 14 Jun 2026 03:49:20 +0000 Subject: [PATCH] fix: cap fetch_all_assets at 5000 items; filter non-dict page entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetching up to MAX_PAGES*page_size (1M) assets before the 3000-item diversity pool cap was applied could exhaust memory on large Immich libraries. Early-exit once 5000 items are collected — the pool cap of 3000 makes anything beyond that wasteful. Also filter null/non-dict items from page responses at fetch time. --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- winnow/immich_api.py | 5 +++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e9f90e..c2aead5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.6] - 2026-06-14 + +### Fixed + +- **OOM when Immich returns many pages per person**: `fetch_all_assets` now stops fetching once 5000 assets have been collected — the diversity selection pool is already capped at 3000 items, so fetching up to 1,000,000 was wasteful and could exhaust memory on large libraries. 5000 provides ample headroom for the pool cap while bounding per-person memory to ~2 MB. +- **Non-dict items in Immich asset pages silently skipped**: a malformed or partially-null Immich response page could include `null` or non-object items in the assets array. These are now filtered at fetch time rather than causing `AttributeError` downstream. + ## [0.4.5] - 2026-06-14 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index f23f811..22d2d6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "winnow" -version = "0.4.5" +version = "0.4.6" description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition and object classification." license = "AGPL-3.0-or-later" requires-python = ">=3.13" diff --git a/winnow/immich_api.py b/winnow/immich_api.py index e60e5b6..b3f7aed 100644 --- a/winnow/immich_api.py +++ b/winnow/immich_api.py @@ -14,6 +14,7 @@ from .config import Config, get_headers logger = logging.getLogger(__name__) MAX_PAGES = 1000 # Safety limit for pagination +_MAX_ASSETS_PER_PERSON = 5000 # Stop fetching after this many — diversity pool is capped at 3000 anyway @dataclass @@ -95,10 +96,10 @@ def fetch_all_assets(person: dict) -> list[dict]: if not page_assets: break - assets.extend(page_assets) + assets.extend(a for a in page_assets if isinstance(a, dict)) logger.debug(f"Fetched page {page}, total: {len(assets)}") - if len(page_assets) < page_size: + if len(page_assets) < page_size or len(assets) >= _MAX_ASSETS_PER_PERSON: break except (requests.RequestException, ValueError) as e: