release: v0.5.15 — fix cache regression and structural cleanup
- cache.py: fix np.save extension bug from v0.5.13 — tmp path used final+".tmp" (abc.npy.tmp) but np.save auto-appends .npy to paths not ending in .npy, writing to abc.npy.tmp.npy instead; os.replace then raised FileNotFoundError silently, making every cache write a no-op and leaking *.npy.tmp.npy files. Fixed by inserting .tmp before .npy: tmp = final[:-4] + ".tmp.npy" - config.py: remove str(default) round-trip in _getenv_int/_getenv_float — use raw = os.getenv(name); return default if raw is None else int(raw) so a future float default can't cause a spurious "not a valid integer" warning and return the wrong type - executor.py: consolidate 4 progress.remove_task calls into one try/finally around the per-job body; continue inside try/finally executes the finally before the next iteration, making the invariant structurally enforced rather than relying on discipline across 4 sites
This commit is contained in:
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.5.15] - 2026-06-15
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **Embedding cache writes were silently no-ops since v0.5.13**: the atomic-write path used `tmp = final + ".tmp"` where `final` ends in `.npy` (e.g. `abc.npy`), producing a tmp path of `abc.npy.tmp`. `np.save` auto-appends `.npy` to paths not already ending in `.npy`, so it wrote to `abc.npy.tmp.npy` instead. The subsequent `os.replace("abc.npy.tmp", "abc.npy")` then raised `FileNotFoundError` (caught silently at DEBUG), meaning no cache entry was ever committed and leaked `*.npy.tmp.npy` files accumulated on disk. The fix inserts `.tmp` before the `.npy` extension: `tmp = final[:-4] + ".tmp.npy"` so `np.save` sees a path already ending in `.npy` and does not re-append.
|
||||||
|
|
||||||
|
- **`_getenv_int`/`_getenv_float` no longer route the default through `str()` conversion**: the previous form `os.getenv(name, str(default))` converted the default to a string so it could be fed through `int()`/`float()` — an unnecessary round-trip that would cause `_getenv_int("FOO", 4.0)` to log a spurious "not a valid integer" warning and return the float. The helpers now use `raw = os.getenv(name); return default if raw is None else int(raw)`, passing the typed default through directly.
|
||||||
|
|
||||||
|
- **`execute_jobs` progress task now removed via `try/finally`**: `progress.remove_task(job_task)` was duplicated in three early-exit paths (ValueError, symlink TOCTOU, OSError) plus once at normal completion. The entire per-job body is now wrapped in `try/finally: progress.remove_task(job_task)`; the three inner `continue` statements trigger the `finally` automatically before advancing to the next job, making the invariant structurally impossible to violate by a future code path.
|
||||||
|
|
||||||
## [0.5.14] - 2026-06-15
|
## [0.5.14] - 2026-06-15
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.5.14"
|
version = "0.5.15"
|
||||||
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
|
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|||||||
+3
-1
@@ -83,7 +83,9 @@ class EmbeddingCache:
|
|||||||
"""Store an embedding in the cache."""
|
"""Store an embedding in the cache."""
|
||||||
self._ensure_dir()
|
self._ensure_dir()
|
||||||
final = self._path(asset_id, model)
|
final = self._path(asset_id, model)
|
||||||
tmp = final + ".tmp"
|
# Insert .tmp before .npy so np.save doesn't auto-append another .npy extension
|
||||||
|
# (np.save appends .npy to paths that don't already end in .npy).
|
||||||
|
tmp = final[:-4] + ".tmp.npy"
|
||||||
try:
|
try:
|
||||||
np.save(tmp, embedding)
|
np.save(tmp, embedding)
|
||||||
os.replace(tmp, final)
|
os.replace(tmp, final)
|
||||||
|
|||||||
+10
-6
@@ -13,20 +13,24 @@ _LEGACY_CONFIG_FILE = Path(".immich_config.json") # pre-v0.6: lived in process
|
|||||||
|
|
||||||
|
|
||||||
def _getenv_int(name: str, default: int) -> int:
|
def _getenv_int(name: str, default: int) -> int:
|
||||||
val = os.getenv(name, str(default))
|
raw = os.getenv(name)
|
||||||
|
if raw is None:
|
||||||
|
return default
|
||||||
try:
|
try:
|
||||||
return int(val)
|
return int(raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
logging.warning("%s=%r is not a valid integer — using default %s", name, val, default)
|
logging.warning("%s=%r is not a valid integer — using default %s", name, raw, default)
|
||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
def _getenv_float(name: str, default: float) -> float:
|
def _getenv_float(name: str, default: float) -> float:
|
||||||
val = os.getenv(name, str(default))
|
raw = os.getenv(name)
|
||||||
|
if raw is None:
|
||||||
|
return default
|
||||||
try:
|
try:
|
||||||
return float(val)
|
return float(raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
logging.warning("%s=%r is not a valid float — using default %s", name, val, default)
|
logging.warning("%s=%r is not a valid float — using default %s", name, raw, default)
|
||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+3
-5
@@ -101,18 +101,17 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
name = person["name"]
|
name = person["name"]
|
||||||
|
|
||||||
job_task = progress.add_task(f"Processing {name}...", total=len(assets))
|
job_task = progress.add_task(f"Processing {name}...", total=len(assets))
|
||||||
|
try:
|
||||||
try:
|
try:
|
||||||
person_dir = _safe_person_dir(Config.OUTPUT_DIR, name)
|
person_dir = _safe_person_dir(Config.OUTPUT_DIR, name)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
progress.remove_task(job_task)
|
|
||||||
continue
|
continue
|
||||||
# Face crops are transient (uploaded then discarded); wipe before each run.
|
# Face crops are transient (uploaded then discarded); wipe before each run.
|
||||||
# A symlink could appear here via a TOCTOU race after _safe_person_dir
|
# A symlink could appear here via a TOCTOU race after _safe_person_dir
|
||||||
# returned — writing through it would land crops outside output_dir.
|
# returned — writing through it would land crops outside output_dir.
|
||||||
if os.path.islink(person_dir):
|
if os.path.islink(person_dir):
|
||||||
logger.error("person_dir %s became a symlink after path check — skipping job", person_dir)
|
logger.error("person_dir %s became a symlink after path check — skipping job", person_dir)
|
||||||
progress.remove_task(job_task)
|
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
if os.path.isdir(person_dir):
|
if os.path.isdir(person_dir):
|
||||||
@@ -120,7 +119,6 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
os.makedirs(person_dir, exist_ok=True)
|
os.makedirs(person_dir, exist_ok=True)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
logger.error("Failed to prepare output dir for %s: %s", name, e)
|
logger.error("Failed to prepare output dir for %s: %s", name, e)
|
||||||
progress.remove_task(job_task)
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Track filename → asset_id, filename → confidence score, filename → crop dims
|
# Track filename → asset_id, filename → confidence score, filename → crop dims
|
||||||
@@ -209,11 +207,11 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
job["score_map"] = score_map
|
job["score_map"] = score_map
|
||||||
job["dims_map"] = dims_map
|
job["dims_map"] = dims_map
|
||||||
|
|
||||||
progress.remove_task(job_task)
|
|
||||||
|
|
||||||
# Log how many images were actually saved vs selected
|
# Log how many images were actually saved vs selected
|
||||||
if count < len(assets):
|
if count < len(assets):
|
||||||
logger.info("%s: saved %s/%s selected images", name, count, len(assets))
|
logger.info("%s: saved %s/%s selected images", name, count, len(assets))
|
||||||
|
finally:
|
||||||
|
progress.remove_task(job_task)
|
||||||
|
|
||||||
|
|
||||||
def upload_to_frigate(jobs: list[dict]) -> None:
|
def upload_to_frigate(jobs: list[dict]) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user