release: v0.5.13 — robustness fixes from full-project audit
- executor.py: wrap shutil.rmtree/os.makedirs in try/except OSError so a permission failure logs and skips the job rather than aborting the run - cache.py: write embeddings to a .tmp file and atomically rename into place via os.replace so a process kill can't leave a corrupted .npy cache slot - immich_api.py: guard fileCreatedAt with isinstance(str) check before calling .replace() so a non-string timestamp doesn't raise AttributeError and kill the entire filter_recent_assets pass - upload_tracker.py: raise SQLite busy timeout from 5 s to 30 s to handle concurrent cron+manual run overlap without dropping upload-tracking records
This commit is contained in:
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.5.13] - 2026-06-15
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **`execute_jobs` output-dir OSError now skips the job instead of aborting the run**: `shutil.rmtree` and `os.makedirs` were not wrapped in any error handler — an `OSError` or `PermissionError` (e.g. read-only filesystem, lingering lock) propagated out of the `for job in jobs` loop, abandoning `job_task` in the Rich progress display and silently dropping all remaining jobs. Both calls are now wrapped in `try/except OSError`; on failure the error is logged, the progress task is removed, and the loop continues to the next job.
|
||||||
|
|
||||||
|
- **Embedding cache writes are now atomic**: `cache.py` previously called `np.save(path, embedding)` directly to the final `.npy` path. A process kill or container stop mid-write left a truncated file that `np.load` would subsequently raise on. Because `get()` catches the exception and returns `None`, the slot appeared empty on every future run — the corrupted file was never cleaned up and the embedding was silently recomputed forever. The write now goes to a `.tmp` sibling and is renamed into place with `os.replace` (atomic on POSIX); the tmp file is removed on any write failure.
|
||||||
|
|
||||||
|
- **`filter_recent_assets` guards against non-string `fileCreatedAt`**: the previous `if not created_at_str` guard passed truthy non-string values (e.g. a Unix-epoch integer returned by some Immich API versions), after which `created_at_str.replace("Z", "+00:00")` raised `AttributeError`. That exception was not caught by the surrounding `except ValueError`, so a single non-string timestamp aborted the entire filtering pass for the person being processed. The guard is now `if not isinstance(created_at_str, str) or not created_at_str`.
|
||||||
|
|
||||||
|
- **SQLite connection timeout raised to 30 s**: `sqlite3.connect` defaulted to a 5-second busy timeout. Under concurrent access (scheduled and manual runs overlapping), 5 s was often insufficient, causing `OperationalError: database is locked` that propagated through `upload_to_frigate` and dropped upload-tracking records — assets would then be re-uploaded on the next run. The timeout is now 30 s, matching the typical upload cycle length.
|
||||||
|
|
||||||
## [0.5.12] - 2026-06-15
|
## [0.5.12] - 2026-06-15
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "winnow"
|
name = "winnow"
|
||||||
version = "0.5.12"
|
version = "0.5.13"
|
||||||
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
|
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|||||||
+8
-1
@@ -82,10 +82,17 @@ class EmbeddingCache:
|
|||||||
def put(self, asset_id: str, embedding: np.ndarray, model: str = "insightface") -> None:
|
def put(self, asset_id: str, embedding: np.ndarray, model: str = "insightface") -> None:
|
||||||
"""Store an embedding in the cache."""
|
"""Store an embedding in the cache."""
|
||||||
self._ensure_dir()
|
self._ensure_dir()
|
||||||
|
final = self._path(asset_id, model)
|
||||||
|
tmp = final + ".tmp"
|
||||||
try:
|
try:
|
||||||
np.save(self._path(asset_id, model), embedding)
|
np.save(tmp, embedding)
|
||||||
|
os.replace(tmp, final)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Cache write failed for %s: %s", asset_id, e)
|
logger.debug("Cache write failed for %s: %s", asset_id, e)
|
||||||
|
try:
|
||||||
|
os.remove(tmp)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
"""Delete all cached embeddings."""
|
"""Delete all cached embeddings."""
|
||||||
|
|||||||
+8
-3
@@ -114,9 +114,14 @@ def execute_jobs(jobs: list[dict]) -> None:
|
|||||||
logger.error("person_dir %s became a symlink after path check — skipping job", person_dir)
|
logger.error("person_dir %s became a symlink after path check — skipping job", person_dir)
|
||||||
progress.remove_task(job_task)
|
progress.remove_task(job_task)
|
||||||
continue
|
continue
|
||||||
if os.path.isdir(person_dir):
|
try:
|
||||||
shutil.rmtree(person_dir)
|
if os.path.isdir(person_dir):
|
||||||
os.makedirs(person_dir, exist_ok=True)
|
shutil.rmtree(person_dir)
|
||||||
|
os.makedirs(person_dir, exist_ok=True)
|
||||||
|
except OSError as e:
|
||||||
|
logger.error("Failed to prepare output dir for %s: %s", name, e)
|
||||||
|
progress.remove_task(job_task)
|
||||||
|
continue
|
||||||
|
|
||||||
# Track filename → asset_id, filename → confidence score, filename → crop dims
|
# Track filename → asset_id, filename → confidence score, filename → crop dims
|
||||||
asset_map: dict[str, str] = {}
|
asset_map: dict[str, str] = {}
|
||||||
|
|||||||
@@ -277,7 +277,7 @@ def filter_recent_assets(assets: list[dict], years: int | None = None) -> list[d
|
|||||||
recent, skipped = [], 0
|
recent, skipped = [], 0
|
||||||
for asset in assets:
|
for asset in assets:
|
||||||
created_at_str = asset.get("fileCreatedAt")
|
created_at_str = asset.get("fileCreatedAt")
|
||||||
if not created_at_str:
|
if not isinstance(created_at_str, str) or not created_at_str:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ def _get_conn() -> sqlite3.Connection:
|
|||||||
|
|
||||||
if _conn is None:
|
if _conn is None:
|
||||||
Path(data_dir).mkdir(parents=True, exist_ok=True)
|
Path(data_dir).mkdir(parents=True, exist_ok=True)
|
||||||
_conn = sqlite3.connect(db_path, check_same_thread=False)
|
_conn = sqlite3.connect(db_path, check_same_thread=False, timeout=30)
|
||||||
_conn.row_factory = sqlite3.Row
|
_conn.row_factory = sqlite3.Row
|
||||||
_conn.execute("PRAGMA journal_mode=WAL")
|
_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
_conn.execute("PRAGMA foreign_keys=ON")
|
_conn.execute("PRAGMA foreign_keys=ON")
|
||||||
|
|||||||
Reference in New Issue
Block a user