flan/winnow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: collapse Config proxy, migrate tracker to SQLite, split reconcile module

Browse Source 
- Config: remove _ConfigAccessor and ConfigManager; use __getattr__ for lazy
  loading on single _Config class; re-register self as _instance in __getattr__
  so reset() always clears the correct object (item 1)
- upload_tracker: replace hand-rolled JSON store with sqlite3; auto-migrates
  existing JSON on first run; remove dead record_frigate_file function;
  connection re-opens when CACHE_DIR changes for test isolation (items 2, 8)
- diversity: move ThreadPoolExecutor import to module level; inject optional
  fetch_fn parameter for testability (items 3, 6)
- pyproject: consolidate 4 variant files into extras (gpu/rocm/intel/cpu);
  update Dockerfile to use --extra flag; delete variant pyproject/lock files;
  uv.lock needs regen with `uv lock` after this change (item 4)
- jobs: extract _build_job helper to separate business logic from terminal I/O;
  auto_configure delegates dedup/selection to _build_job (item 5)
- logging: convert f-string log calls to % interpolation throughout all winnow/
  modules (item 7)
- reconcile: new module with reconcile_frigate_mappings and
  enrich_asset_with_face_data extracted from executor.py (item 9)
- scheduler: print next scheduled run time after startup and after each run;
  fix f-string logger.error call (item 10)
This commit is contained in:
flan
2026-06-14 19:59:17 +00:00
parent ad1fbd4c2a
commit e2a1924fb0
22 changed files with 754 additions and 6613 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+7 -8
View File
@@ -41,18 +41,17 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
WORKDIR /app
# Swap in the variant-specific pyproject and lockfile before syncing.
COPY pyproject.toml uv.lock pyproject-cpu.toml uv-cpu.lock \
pyproject-rocm.toml uv-rocm.lock pyproject-intel.toml uv-intel.lock ./
COPY pyproject.toml uv.lock ./
RUN if [ "$VARIANT" = "cpu" ]; then \
cp pyproject-cpu.toml pyproject.toml && cp uv-cpu.lock uv.lock; \
uv sync --frozen --no-dev --extra cpu; \
elif [ "$VARIANT" = "rocm" ]; then \
cp pyproject-rocm.toml pyproject.toml && cp uv-rocm.lock uv.lock; \
uv sync --frozen --no-dev --extra rocm; \
elif [ "$VARIANT" = "intel" ]; then \
cp pyproject-intel.toml pyproject.toml && cp uv-intel.lock uv.lock; \
uv sync --frozen --no-dev --extra intel; \
else \
uv sync --frozen --no-dev --extra gpu; \
fi && \
uv sync --frozen --no-dev \
&& uv cache clean
uv cache clean
COPY winnow/ winnow/
COPY entrypoint.sh scheduler.py ./
pyproject-cpu.toml
-74
View File
@@ -1,74 +0,0 @@
[project]
name = "winnow"
version = "0.4.11"
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
license = "AGPL-3.0-or-later"
requires-python = ">=3.13"
authors = [{ name = "Holden Salomon", email = "holden@arch.fyi" }]
keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering :: Image Recognition",
]
dependencies = [
"croniter>=5.0.2",
"insightface>=0.7.3",
"numpy>=2.2.6",
"onnxruntime>=1.23.2",
"opencv-python-headless>=4.12.0.88",
"pillow>=12.1.0",
"python-dotenv>=1.2.1",
"requests>=2.32.5",
"rich>=14.2.0",
]
[project.scripts]
winnow = "winnow.cli:main"
[project.urls]
Repository = "https://github.com/sudolulo/winnow"
Changelog = "https://github.com/sudolulo/winnow/blob/main/CHANGELOG.md"
[tool.uv]
required-environments = [
"sys_platform == 'linux' and platform_machine == 'x86_64'",
]
[dependency-groups]
dev = [
"pytest>=8.0",
"ruff>=0.15.17",
]
[tool.hatch.build.targets.wheel]
packages = ["winnow"]
[tool.ruff]
line-length = 120
target-version = "py313"
[tool.ruff.lint]
select = ["E", "F", "I"]
[tool.deptry]
pep621_dev_dependency_groups = ["dev"]
[tool.deptry.package_module_name_map]
pillow = "PIL"
opencv-python-headless = "cv2"
python-dotenv = "dotenv"
insightface = "insightface"
numpy = "numpy"
onnxruntime = "onnxruntime"
requests = "requests"
rich = "rich"
[tool.pytest.ini_options]
testpaths = ["tests"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
pyproject-intel.toml
-84
View File
@@ -1,84 +0,0 @@
[project]
name = "winnow"
version = "0.4.11"
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
license = "AGPL-3.0-or-later"
requires-python = ">=3.13"
authors = [{ name = "Holden Salomon", email = "holden@arch.fyi" }]
keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering :: Image Recognition",
]
dependencies = [
"croniter>=5.0.2",
"insightface>=0.7.3",
"numpy>=2.2.6",
"onnxruntime-openvino>=1.20.0",
"opencv-python-headless>=4.12.0.88",
"pillow>=12.1.0",
"python-dotenv>=1.2.1",
"requests>=2.32.5",
"rich>=14.2.0",
]
[project.scripts]
winnow = "winnow.cli:main"
[project.urls]
Repository = "https://github.com/sudolulo/winnow"
Changelog = "https://github.com/sudolulo/winnow/blob/main/CHANGELOG.md"
[tool.uv]
conflicts = [
[
{ package = "onnxruntime" },
{ package = "onnxruntime-gpu" },
{ package = "onnxruntime-openvino" },
],
]
required-environments = [
"sys_platform == 'linux' and platform_machine == 'x86_64'",
]
[dependency-groups]
dev = [
"pytest>=8.0",
"ruff>=0.15.17",
]
[tool.hatch.build.targets.wheel]
packages = ["winnow"]
[tool.ruff]
line-length = 120
target-version = "py313"
[tool.ruff.lint]
select = ["E", "F", "I"]
[tool.deptry]
pep621_dev_dependency_groups = ["dev"]
[tool.deptry.package_module_name_map]
pillow = "PIL"
opencv-python-headless = "cv2"
python-dotenv = "dotenv"
insightface = "insightface"
numpy = "numpy"
onnxruntime-openvino = "onnxruntime"
requests = "requests"
rich = "rich"
[tool.deptry.per_rule_ignores]
DEP002 = ["onnxruntime-openvino"]
[tool.pytest.ini_options]
testpaths = ["tests"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
pyproject-rocm.toml
-85
View File
@@ -1,85 +0,0 @@
[project]
name = "winnow"
version = "0.4.11"
description = "Selects diverse, high-quality photos from Immich as training data for Frigate face recognition."
license = "AGPL-3.0-or-later"
requires-python = ">=3.13"
authors = [{ name = "Holden Salomon", email = "holden@arch.fyi" }]
keywords = ["immich", "frigate", "face-recognition", "training-data", "arcface", "insightface"]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering :: Image Recognition",
]
dependencies = [
"croniter>=5.0.2",
"insightface>=0.7.3",
"numpy>=2.2.6",
"onnxruntime-rocm>=1.16.0",
"opencv-python-headless>=4.12.0.88",
"pillow>=12.1.0",
"python-dotenv>=1.2.1",
"requests>=2.32.5",
"rich>=14.2.0",
]
[project.scripts]
winnow = "winnow.cli:main"
[project.urls]
Repository = "https://github.com/sudolulo/winnow"
Changelog = "https://github.com/sudolulo/winnow/blob/main/CHANGELOG.md"
[tool.uv]
index-strategy = "unsafe-best-match"
conflicts = [
[
{ package = "onnxruntime" },
{ package = "onnxruntime-gpu" },
{ package = "onnxruntime-rocm" },
],
]
required-environments = [
"sys_platform == 'linux' and platform_machine == 'x86_64'",
]
[dependency-groups]
dev = [
"pytest>=8.0",
"ruff>=0.15.17",
]
[tool.hatch.build.targets.wheel]
packages = ["winnow"]
[tool.ruff]
line-length = 120
target-version = "py313"
[tool.ruff.lint]
select = ["E", "F", "I"]
[tool.deptry]
pep621_dev_dependency_groups = ["dev"]
[tool.deptry.package_module_name_map]
pillow = "PIL"
opencv-python-headless = "cv2"
python-dotenv = "dotenv"
insightface = "insightface"
numpy = "numpy"
onnxruntime-rocm = "onnxruntime"
requests = "requests"
rich = "rich"
[tool.deptry.per_rule_ignores]
DEP002 = ["onnxruntime-rocm"]
[tool.pytest.ini_options]
testpaths = ["tests"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
pyproject.toml
+15 -8
View File
@@ -17,11 +17,7 @@ classifiers = [
dependencies = [
"croniter>=5.0.2",
"insightface>=0.7.3",
"nvidia-cudnn-cu12>=9.0.0",
"numpy>=2.2.6",
"onnxruntime-gpu>=1.23.2; sys_platform == 'linux' and platform_machine == 'x86_64'",
"onnxruntime>=1.23.2; sys_platform == 'linux' and platform_machine != 'x86_64'",
"onnxruntime>=1.23.2; sys_platform != 'linux'",
"opencv-python-headless>=4.12.0.88",
"pillow>=12.1.0",
"python-dotenv>=1.2.1",
@@ -29,6 +25,12 @@ dependencies = [
"rich>=14.2.0",
]
[project.optional-dependencies]
gpu = ["onnxruntime-gpu>=1.23.2", "nvidia-cudnn-cu12>=9.0.0"]
rocm = ["onnxruntime-rocm>=1.16.0"]
intel = ["onnxruntime-openvino>=1.20.0"]
cpu = ["onnxruntime>=1.23.2"]
[project.scripts]
winnow = "winnow.cli:main"
@@ -38,10 +40,13 @@ Changelog = "https://github.com/sudolulo/winnow/blob/main/CHANGELOG.md"
Documentation = "https://github.com/sudolulo/winnow/wiki"
[tool.uv]
index-strategy = "unsafe-best-match"
conflicts = [
[
{ package = "onnxruntime" },
{ package = "onnxruntime-gpu" },
{ extra = "gpu" },
{ extra = "rocm" },
{ extra = "intel" },
{ extra = "cpu" },
],
]
required-environments = [
@@ -77,11 +82,14 @@ insightface = "insightface"
numpy = "numpy"
nvidia-cudnn-cu12 = "nvidia.cudnn"
onnxruntime-gpu = "onnxruntime"
onnxruntime-rocm = "onnxruntime"
onnxruntime-openvino = "onnxruntime"
onnxruntime = "onnxruntime"
requests = "requests"
rich = "rich"
[tool.deptry.per_rule_ignores]
DEP002 = ["onnxruntime-gpu", "nvidia-cudnn-cu12"]
DEP002 = ["onnxruntime-gpu", "nvidia-cudnn-cu12", "onnxruntime-rocm", "onnxruntime-openvino", "onnxruntime"]
[tool.pytest.ini_options]
testpaths = ["tests"]
@@ -89,4 +97,3 @@ testpaths = ["tests"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
scheduler.py
+3 -1
View File
@@ -30,6 +30,7 @@ def check_models() -> None:
NOW = time.time()
cron = croniter(SCHEDULE, NOW)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
while True:
now = time.time()
@@ -42,7 +43,8 @@ while True:
except KeyboardInterrupt:
raise
except Exception as e:
logger.error(f"winnow run failed: {e}", exc_info=True)
logger.error("winnow run failed: %s", e, exc_info=True)
print(f"winnow run failed: {e}", flush=True)
next_run = cron.get_next(float)
print(f"Next run: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(next_run))}", flush=True)
time.sleep(max(1, next_run - time.time()))
tests/test_upload_tracker.py
+45 -40
View File
@@ -10,7 +10,19 @@ def isolated_cache(monkeypatch, tmp_path):
monkeypatch.setenv("CACHE_DIR", str(tmp_path))
from winnow.config import _Config
_Config.reset()
# Also reset the SQLite connection so the next call opens the new path
import winnow.upload_tracker as ut
ut._conn = None
ut._conn_path = None
yield tmp_path
# Teardown
if ut._conn is not None:
try:
ut._conn.close()
except Exception:
pass
ut._conn = None
ut._conn_path = None
_Config.reset()
@@ -73,12 +85,12 @@ def test_duplicate_marks_are_idempotent():
# ── frigate_files mapping ─────────────────────────────────────────────────────
def test_record_and_remove_frigate_file():
from winnow.upload_tracker import get_person_summary, record_frigate_file, remove_frigate_file
record_frigate_file("Alice", "Alice-1000.webp", "asset-a1")
def test_record_and_remove_frigate_files_batch():
from winnow.upload_tracker import get_person_summary, record_frigate_files_batch, remove_frigate_file
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a1"})
assert "Alice-1000.webp" in get_person_summary()["Alice"]["frigate_files"]
remove_frigate_file("Alice", "Alice-1000.webp")
assert "Alice-1000.webp" not in get_person_summary()["Alice"]["frigate_files"]
assert "Alice-1000.webp" not in get_person_summary().get("Alice", {}).get("frigate_files", {})
def test_remove_nonexistent_frigate_file_is_safe():
@@ -92,11 +104,11 @@ def test_remove_frigate_file_does_not_unmark_asset():
from winnow.upload_tracker import (
filter_already_uploaded,
mark_uploaded,
record_frigate_file,
record_frigate_files_batch,
remove_frigate_file,
)
mark_uploaded("asset-a1", person_name="Alice")
record_frigate_file("Alice", "Alice-1000.webp", "asset-a1")
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a1"})
remove_frigate_file("Alice", "Alice-1000.webp")
# Asset must still be excluded — it was deliberately replaced, not lost
assert filter_already_uploaded(["asset-a1"]) == []
@@ -108,14 +120,14 @@ def test_get_tracked_frigate_file_count_zero_when_empty():
def test_get_tracked_frigate_file_count_counts_only_mapped():
"""Only files explicitly recorded via record_frigate_file count toward the cap."""
from winnow.upload_tracker import get_tracked_frigate_file_count, mark_uploaded, record_frigate_file
"""Only files explicitly recorded via record_frigate_files_batch count toward the cap."""
from winnow.upload_tracker import get_tracked_frigate_file_count, mark_uploaded, record_frigate_files_batch
mark_uploaded("asset-a", person_name="Alice")
mark_uploaded("asset-b", person_name="Alice")
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a"})
# asset-b is uploaded but not yet mapped — does not count
assert get_tracked_frigate_file_count("Alice") == 1
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
record_frigate_files_batch("Alice", {"Alice-1001.webp": "asset-b"})
assert get_tracked_frigate_file_count("Alice") == 2
@@ -128,12 +140,11 @@ def test_get_lowest_quality_mapped_file_returns_lowest():
from winnow.upload_tracker import (
get_lowest_quality_mapped_file,
mark_uploaded,
record_frigate_file,
record_frigate_files_batch,
)
mark_uploaded("asset-hi", person_name="Alice", score=0.95)
mark_uploaded("asset-lo", person_name="Alice", score=0.71)
record_frigate_file("Alice", "Alice-1000.webp", "asset-hi")
record_frigate_file("Alice", "Alice-1001.webp", "asset-lo")
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-hi", "Alice-1001.webp": "asset-lo"})
result = get_lowest_quality_mapped_file("Alice")
assert result is not None
frigate_filename, asset_id, score = result
@@ -147,12 +158,11 @@ def test_get_lowest_quality_mapped_file_skips_unscored():
from winnow.upload_tracker import (
get_lowest_quality_mapped_file,
mark_uploaded,
record_frigate_file,
record_frigate_files_batch,
)
mark_uploaded("asset-scored", person_name="Alice", score=0.85)
mark_uploaded("asset-noscr", person_name="Alice")
record_frigate_file("Alice", "Alice-1000.webp", "asset-scored")
record_frigate_file("Alice", "Alice-1001.webp", "asset-noscr")
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-scored", "Alice-1001.webp": "asset-noscr"})
result = get_lowest_quality_mapped_file("Alice")
assert result is not None
assert result[1] == "asset-scored" # only scored file is a candidate
@@ -166,28 +176,26 @@ def test_get_tracked_frigate_filenames_empty():
def test_get_tracked_frigate_filenames_returns_mapped():
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_file
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_files_batch
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a", "Alice-1001.webp": "asset-b"})
assert get_tracked_frigate_filenames("Alice") == {"Alice-1000.webp", "Alice-1001.webp"}
def test_get_tracked_frigate_filenames_excludes_removed():
from winnow.upload_tracker import (
get_tracked_frigate_filenames,
record_frigate_file,
record_frigate_files_batch,
remove_frigate_file,
)
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a", "Alice-1001.webp": "asset-b"})
remove_frigate_file("Alice", "Alice-1000.webp")
assert get_tracked_frigate_filenames("Alice") == {"Alice-1001.webp"}
def test_get_tracked_frigate_filenames_isolated_by_person():
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_file
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
record_frigate_file("Bob", "Bob-2000.webp", "asset-b")
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_files_batch
record_frigate_files_batch("Alice", {"Alice-1000.webp": "asset-a"})
record_frigate_files_batch("Bob", {"Bob-2000.webp": "asset-b"})
assert get_tracked_frigate_filenames("Alice") == {"Alice-1000.webp"}
assert get_tracked_frigate_filenames("Bob") == {"Bob-2000.webp"}
@@ -198,12 +206,11 @@ def test_get_lowest_quality_exclude_skips_specified_file():
from winnow.upload_tracker import (
get_lowest_quality_mapped_file,
mark_uploaded,
record_frigate_file,
record_frigate_files_batch,
)
mark_uploaded("asset-lo", person_name="Alice", score=0.10)
mark_uploaded("asset-hi", person_name="Alice", score=0.90)
record_frigate_file("Alice", "Alice-lo.webp", "asset-lo")
record_frigate_file("Alice", "Alice-hi.webp", "asset-hi")
record_frigate_files_batch("Alice", {"Alice-lo.webp": "asset-lo", "Alice-hi.webp": "asset-hi"})
result = get_lowest_quality_mapped_file("Alice", exclude={"Alice-lo.webp"})
assert result is not None
assert result[1] == "asset-hi" # lo was excluded; hi is returned
@@ -213,29 +220,28 @@ def test_get_lowest_quality_exclude_all_returns_none():
from winnow.upload_tracker import (
get_lowest_quality_mapped_file,
mark_uploaded,
record_frigate_file,
record_frigate_files_batch,
)
mark_uploaded("asset-a", person_name="Alice", score=0.50)
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
record_frigate_files_batch("Alice", {"Alice-a.webp": "asset-a"})
assert get_lowest_quality_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
# ── get_most_redundant_mapped_file ────────────────────────────────────────────
def test_get_most_redundant_none_when_no_frigate_scores():
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_files_batch
mark_uploaded("asset-a", person_name="Alice", score=0.80)
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
record_frigate_files_batch("Alice", {"Alice-a.webp": "asset-a"})
# blur score only, no frigate_score → no candidates
assert get_most_redundant_mapped_file("Alice") is None
def test_get_most_redundant_returns_highest_frigate_score():
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_files_batch
mark_uploaded("asset-novel", person_name="Alice", score=0.50, frigate_score=0.31)
mark_uploaded("asset-redundant", person_name="Alice", score=0.90, frigate_score=0.88)
record_frigate_file("Alice", "Alice-novel.webp", "asset-novel")
record_frigate_file("Alice", "Alice-redundant.webp", "asset-redundant")
record_frigate_files_batch("Alice", {"Alice-novel.webp": "asset-novel", "Alice-redundant.webp": "asset-redundant"})
result = get_most_redundant_mapped_file("Alice")
assert result is not None
frigate_filename, asset_id, score = result
@@ -245,18 +251,17 @@ def test_get_most_redundant_returns_highest_frigate_score():
def test_get_most_redundant_exclude_skips_file():
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_files_batch
mark_uploaded("asset-hi", person_name="Alice", score=0.9, frigate_score=0.85)
mark_uploaded("asset-lo", person_name="Alice", score=0.5, frigate_score=0.40)
record_frigate_file("Alice", "Alice-hi.webp", "asset-hi")
record_frigate_file("Alice", "Alice-lo.webp", "asset-lo")
record_frigate_files_batch("Alice", {"Alice-hi.webp": "asset-hi", "Alice-lo.webp": "asset-lo"})
result = get_most_redundant_mapped_file("Alice", exclude={"Alice-hi.webp"})
assert result is not None
assert result[1] == "asset-lo" # hi excluded; lo is next highest
def test_get_most_redundant_exclude_all_returns_none():
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_files_batch
mark_uploaded("asset-a", person_name="Alice", score=0.5, frigate_score=0.70)
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
record_frigate_files_batch("Alice", {"Alice-a.webp": "asset-a"})
assert get_most_redundant_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
uv-cpu.lock
-1884
View File
File diff suppressed because it is too large Load Diff
uv-intel.lock
-1941
View File
File diff suppressed because it is too large Load Diff
uv-rocm.lock
-1933
View File
File diff suppressed because it is too large Load Diff
winnow/cache.py
+2 -2
View File
@@ -85,7 +85,7 @@ class EmbeddingCache:
try:
np.save(self._path(asset_id, model), embedding)
except Exception as e:
logger.debug(f"Cache write failed for {asset_id}: {e}")
logger.debug("Cache write failed for %s: %s", asset_id, e)
def clear(self) -> None:
"""Delete all cached embeddings."""
@@ -96,7 +96,7 @@ class EmbeddingCache:
if f.endswith(".npy"):
os.remove(os.path.join(self.cache_dir, f))
count += 1
logger.info(f"Cleared {count} cached embeddings.")
logger.info("Cleared %s cached embeddings.", count)
# Singleton instance
winnow/cli.py
+2 -2
View File
@@ -7,7 +7,7 @@ import sys
from rich import print as rprint
from rich.prompt import Confirm
from .config import Config, ConfigManager
from .config import Config
from .executor import execute_jobs, upload_to_frigate
from .immich_api import get_immich_version, get_people, merge_people
from .jobs import _show_preview, auto_configure, interactive_configure
@@ -168,7 +168,7 @@ def main() -> None:
"Image quality issues caused by non-default values will not be investigated.[/dim]\n"
)
ConfigManager.get().interactive_setup()
Config.interactive_setup()
try:
Config.validate()
winnow/config.py
+55 -68
View File
@@ -15,43 +15,70 @@ CONFIG_FILE = Path(".immich_config.json")
class _Config:
"""Singleton configuration with uppercase attribute access for backward compatibility."""
"""Singleton configuration with lazy loading via __getattr__.
Class-level attributes are annotations only (no defaults), so attribute
access on an un-loaded instance falls through to __getattr__, which
triggers _load() exactly once.
"""
_instance: ClassVar["_Config | None"] = None
# Configuration values
IMMICH_URL: str | None = None
API_KEY: str | None = None
OUTPUT_DIR: str = "./frigate_train"
YEARS_FILTER: int = 10
# Annotations only — no class-level defaults so __getattr__ fires on first access
IMMICH_URL: str | None
API_KEY: str | None
OUTPUT_DIR: str
YEARS_FILTER: int
# Quality filtering
MIN_FACE_WIDTH: int = 90
BLUR_THRESHOLD: float = 120.0
MIN_CONFIDENCE: float = 0.7
MAX_AUTO_IMAGES: int = 20
QUALITY_REPLACEMENT: bool = True
FRIGATE_SCORE_CEILING: float | None = None
ENABLE_FRIGATE_SCORES: bool = True
MIN_FACE_WIDTH: int
BLUR_THRESHOLD: float
MIN_CONFIDENCE: float
MAX_AUTO_IMAGES: int
QUALITY_REPLACEMENT: bool
FRIGATE_SCORE_CEILING: float | None
ENABLE_FRIGATE_SCORES: bool
# People filtering
MIN_FACE_COUNT: int = 3
MERGE_DUPLICATE_PEOPLE: bool = False
MIN_FACE_COUNT: int
MERGE_DUPLICATE_PEOPLE: bool
# Output quality
FACE_MARGIN: float = 0.15
USE_FULL_RESOLUTION: bool = True
ENABLE_FACE_ALIGNMENT: bool = True
FACE_MARGIN: float
USE_FULL_RESOLUTION: bool
ENABLE_FACE_ALIGNMENT: bool
ENABLE_CACHE: bool = True
CACHE_DIR: str = ".if_cache"
ENABLE_CACHE: bool
CACHE_DIR: str
def __new__(cls) -> "_Config":
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._load()
# Do NOT call _load() here — keep __new__ I/O-free so that import
# time does not trigger env/file reads.
return cls._instance
def __getattr__(self, name: str):
"""Called only when the attribute is not found on the instance.
On first access to any config attribute, load all values from env/file
and return the requested one. Re-registers self as _instance so that
a subsequent reset() correctly finds and clears this object's attrs.
"""
if name.startswith("_"):
raise AttributeError(name)
self._load()
# Re-register self as the singleton so reset() can clear our __dict__.
# This handles the case where __getattr__ is called on the module-level
# Config object after a reset() set _instance to None.
_Config._instance = self
# _load() sets the attribute as an instance attr; retrieve it directly
# to avoid infinite recursion through __getattr__.
try:
return self.__dict__[name]
except KeyError:
raise AttributeError(f"_Config has no attribute {name!r}")
def _load(self) -> None:
"""Load configuration from environment and config file."""
# Load from environment (highest priority)
@@ -83,11 +110,13 @@ class _Config:
if not os.getenv("OUTPUT_DIR"):
self.OUTPUT_DIR = data.get("OUTPUT_DIR", self.OUTPUT_DIR)
except (json.JSONDecodeError, OSError) as e:
logging.warning(f"Failed to load config file: {e}")
logging.warning("Failed to load config file: %s", e)
@classmethod
def reset(cls) -> None:
"""Reset the singleton — mainly useful for testing or delayed env setup."""
if cls._instance is not None:
cls._instance.__dict__.clear()
cls._instance = None
def save(self) -> None:
@@ -106,9 +135,9 @@ class _Config:
indent=2,
)
)
logging.info(f"Configuration saved to {CONFIG_FILE}")
logging.info("Configuration saved to %s", CONFIG_FILE)
except OSError as e:
logging.error(f"Failed to save config: {e}")
logging.error("Failed to save config: %s", e)
def interactive_setup(self) -> None:
"""Prompt user for missing configuration."""
@@ -132,52 +161,10 @@ class _Config:
raise ValueError("Missing Immich URL or API Key.")
# Singleton instance — use a lazy property pattern to avoid import-time side effects
# when env vars aren't yet set. Call Config.instance() or just access attributes on
# the module-level `Config` (which delegates to the singleton).
class _ConfigAccessor:
"""Lazy accessor that defers singleton creation until first attribute access.
This avoids reading .env and config files at import time, so environment
variables set after importing the module are properly picked up.
"""
def __getattr__(self, name: str):
return getattr(_Config(), name)
def __setattr__(self, name: str, value):
if name.startswith("_"):
super().__setattr__(name, value)
else:
setattr(_Config(), name, value)
def reset(self) -> None:
"""Reset the underlying singleton."""
_Config.reset()
def interactive_setup(self) -> None:
"""Delegate to the singleton."""
_Config().interactive_setup()
def validate(self) -> None:
"""Delegate to the singleton."""
_Config().validate()
def save(self) -> None:
"""Delegate to the singleton."""
_Config().save()
Config = _ConfigAccessor()
class ConfigManager:
@staticmethod
def get() -> _Config:
return _Config()
# Module-level singleton — lazy: no I/O until first attribute access.
Config = _Config()
def get_headers() -> dict[str, str]:
"""Return HTTP headers for Immich API requests."""
return {"x-api-key": Config.API_KEY or "", "Accept": "application/json"}
winnow/diversity.py
+20 -16
View File
@@ -10,6 +10,7 @@ Selection pipeline:
"""
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
import numpy as np
@@ -30,6 +31,7 @@ def select_diverse_assets(
selection_mode: str = "smart",
person_id: str | None = None,
progress_callback=None,
fetch_fn=None,
) -> list:
"""
Select diverse assets using cluster-aware FPS or time spread.
@@ -40,6 +42,8 @@ def select_diverse_assets(
entity_name: Name of the person for logging
selection_mode: 'smart' (embedding-based) or 'time' (time spread)
progress_callback: Optional callback(current, total) for progress
fetch_fn: Optional callable(asset_id) -> Image | None; defaults to
_fetch_thumbnail. Injected for testability.
Returns:
List of selected assets
@@ -57,9 +61,9 @@ def select_diverse_assets(
return _select_time_spread(assets, limit)
try:
return _select_by_embedding(assets, limit, person_id, progress_callback)
return _select_by_embedding(assets, limit, person_id, progress_callback, fetch_fn=fetch_fn)
except Exception as e:
logger.error(f"Smart Diversity failed: {e}. Falling back to time spread.")
logger.error("Smart Diversity failed: %s. Falling back to time spread.", e)
return _select_time_spread(assets, limit)
@@ -178,6 +182,7 @@ def _select_by_embedding(
limit: int | str,
person_id: str | None = None,
progress_callback=None,
fetch_fn=None,
) -> list:
"""Select assets using embedding-based cluster-aware FPS.
@@ -210,8 +215,7 @@ def _select_by_embedding(
# representative in practice, but heavy JPEG compression on a preview could
# produce a subtly different embedding than the full-res version. For most
# libraries this is negligible; it matters if Immich preview quality is low.
from concurrent.futures import ThreadPoolExecutor, as_completed
_fetch = fetch_fn or _fetch_thumbnail
_BATCH = 32
embeddings, valid_candidates, confidence_scores = [], [], []
quality_filtered = 0
@@ -223,7 +227,7 @@ def _select_by_embedding(
# Download this batch concurrently
batch_images: dict[str, Image.Image] = {}
with ThreadPoolExecutor(max_workers=min(8, len(batch))) as pool:
futures = {pool.submit(_fetch_thumbnail, a["id"]): a for a in batch}
futures = {pool.submit(_fetch, a["id"]): a for a in batch}
for future in as_completed(futures):
asset = futures[future]
try:
@@ -231,7 +235,7 @@ def _select_by_embedding(
if img is not None:
batch_images[asset["id"]] = img
except Exception as e:
logger.debug(f"Failed to fetch thumbnail for {asset['id']}: {e}")
logger.debug("Failed to fetch thumbnail for %s: %s", asset["id"], e)
continue
# Process each image; batch_images goes out of scope after this loop,
@@ -257,7 +261,7 @@ def _select_by_embedding(
)
if not quality.passed:
quality_filtered += 1
logger.debug(f"Quality filtered {asset['id']}: {quality.reason}")
logger.debug("Quality filtered %s: %s", asset["id"], quality.reason)
continue
asset["quality_score"] = quality.blur_score
@@ -271,14 +275,14 @@ def _select_by_embedding(
confidence_scores.append(confidence)
if quality_filtered > 0:
logger.info(f"Quality filtering removed {quality_filtered} images.")
logger.info("Quality filtering removed %s images.", quality_filtered)
if not embeddings:
logger.warning("No valid embeddings found. Falling back to time spread.")
return _select_time_spread(assets, limit)
if limit != "auto" and len(valid_candidates) < limit:
logger.warning(f"Only {len(valid_candidates)} valid embeddings. Returning all.")
logger.warning("Only %s valid embeddings. Returning all.", len(valid_candidates))
return valid_candidates
# --- Phase 5: Near-duplicate removal ---
@@ -292,7 +296,7 @@ def _select_by_embedding(
# Re-check after dedup: pool may have shrunk below limit
if limit != "auto" and len(valid_candidates) < limit:
logger.warning(f"Only {len(valid_candidates)} embeddings after near-duplicate removal. Returning all.")
logger.warning("Only %s embeddings after near-duplicate removal. Returning all.", len(valid_candidates))
return valid_candidates
# --- Phase 6: Cluster-aware selection ---
@@ -352,7 +356,7 @@ def _dedup_embeddings(
dropped = len(embeddings) - len(kept_indices)
if dropped:
logger.info(f"Near-duplicate removal dropped {dropped} images (threshold {_DEDUP_THRESHOLD}).")
logger.info("Near-duplicate removal dropped %s images (threshold %s).", dropped, _DEDUP_THRESHOLD)
return (
[embeddings[i] for i in kept_indices],
@@ -447,7 +451,7 @@ def _compute_adaptive_threshold(emb_normed: np.ndarray) -> float:
median_dist = float(np.median(upper_tri))
threshold = max(0.05, median_dist * 0.20)
logger.debug(f"Adaptive threshold: {threshold:.4f} (median_dist={median_dist:.4f})")
logger.debug("Adaptive threshold: %.4f (median_dist=%.4f)", threshold, median_dist)
return threshold
@@ -485,7 +489,7 @@ def _cluster_aware_selection(
# --- Stage 1: K-Medoids clustering ---
k = min(max(5, target // 4), max(1, n // 3), n) # e.g., 1-20 clusters
logger.debug(f"Clustering {n} embeddings into {k} groups (K-Medoids)...")
logger.debug("Clustering %s embeddings into %s groups (K-Medoids)...", n, k)
# Compute full cosine distance matrix
dist_matrix = 1 - emb_normed @ emb_normed.T
@@ -494,7 +498,7 @@ def _cluster_aware_selection(
selected = list(medoid_indices)
selected_set = set(selected)
logger.debug(f"Selected {len(selected)} cluster medoids as initial picks.")
logger.debug("Selected %s cluster medoids as initial picks.", len(selected))
# --- Stage 2: FPS with hard example weighting ---
min_dists = np.full(n, np.inf)
@@ -534,7 +538,7 @@ def _cluster_aware_selection(
selected_conf = [conf_array[i] for i in selected if conf_array[i] < 1.0]
hard_count = sum(1 for c in selected_conf if c < 0.85)
logger.info(f"Selection complete: {len(selected)} images ({hard_count} hard examples with confidence < 0.85).")
logger.info("Selection complete: %s images (%s hard examples with confidence < 0.85).", len(selected), hard_count)
return [candidates[i] for i in selected]
@@ -549,7 +553,7 @@ def _select_time_spread(assets: list, limit: int | str) -> list:
if limit == "auto":
limit = 30
logger.info(f"Selecting {limit} images using time spread.")
logger.info("Selecting %s images using time spread.", limit)
if len(assets) <= limit:
return assets
winnow/embeddings.py
+9 -9
View File
@@ -66,7 +66,7 @@ def _preload_cuda_libs() -> None:
else:
logger.debug("onnxruntime.preload_dlls() not available (ORT < 1.21)")
except Exception as e:
logger.warning(f"Failed to preload CUDA/cuDNN DLLs: {e}")
logger.warning("Failed to preload CUDA/cuDNN DLLs: %s", e)
# =============================================================================
@@ -100,7 +100,7 @@ def get_insightface_app():
# Get providers, excluding TensorRT to avoid noisy errors
providers = [p for p in ort.get_available_providers() if p != "TensorrtExecutionProvider"]
logger.debug(f"ONNX providers available: {providers}")
logger.debug("ONNX providers available: %s", providers)
gpu_providers = {
"CUDAExecutionProvider",
@@ -121,7 +121,7 @@ def get_insightface_app():
if p == "OpenVINOExecutionProvider" else p
for p in providers
]
logger.debug(f"OpenVINO EP: device_type={openvino_device}")
logger.debug("OpenVINO EP: device_type=%s", openvino_device)
if not has_gpu_provider and not _is_force_cpu():
logger.warning(
@@ -136,21 +136,21 @@ def get_insightface_app():
device_str = f"OpenVINO ({os.getenv('OPENVINO_DEVICE', 'CPU')})"
else:
device_str = "GPU"
logger.info(f"InsightFace Buffalo_L: loading into memory on {device_str}...")
logger.info("InsightFace Buffalo_L: loading into memory on %s...", device_str)
t0 = time.time()
with _suppress_output():
_insightface_app = FaceAnalysis(name="buffalo_l", root=insightface_home, providers=providers)
_insightface_app.prepare(ctx_id=ctx_id, det_size=(640, 640))
logger.info(f"InsightFace Buffalo_L: ready on {device_str} ({time.time() - t0:.1f}s)")
logger.info("InsightFace Buffalo_L: ready on %s (%.1fs)", device_str, time.time() - t0)
return _insightface_app
except ImportError:
logger.error("InsightFace not installed!")
return None
except Exception as e:
logger.error(f"Failed to load InsightFace: {e}")
logger.error("Failed to load InsightFace: %s", e)
if ctx_id == 0:
logger.warning("InsightFace GPU load failed — retrying on CPU...")
try:
@@ -164,10 +164,10 @@ def get_insightface_app():
providers=["CPUExecutionProvider"],
)
_insightface_app.prepare(ctx_id=-1, det_size=(640, 640))
logger.info(f"InsightFace Buffalo_L: ready on CPU (fallback, {time.time() - t0:.1f}s)")
logger.info("InsightFace Buffalo_L: ready on CPU (fallback, %.1fs)", time.time() - t0)
return _insightface_app
except Exception as ex:
logger.error(f"InsightFace CPU fallback failed: {ex}")
logger.error("InsightFace CPU fallback failed: %s", ex)
return None
@@ -193,7 +193,7 @@ def get_face_embedding(img_pil: Image.Image) -> np.ndarray | None:
largest = max(faces, key=lambda f: (f.bbox[2] - f.bbox[0]) * (f.bbox[3] - f.bbox[1]))
return largest.embedding
except Exception as e:
logger.error(f"Error getting face embedding: {e}")
logger.error("Error getting face embedding: %s", e)
return None
winnow/executor.py
+10 -117
View File
@@ -3,7 +3,6 @@
import logging
import os
import shutil
import time
from io import BytesIO
from urllib.parse import quote
@@ -21,9 +20,10 @@ from .frigate_api import (
recognize_face,
)
from .image_processing import process_face_mode
from .immich_api import fetch_face_data, fetch_full_image
from .immich_api import fetch_full_image
from .log_config import console
from .quality import assess_quality
from .reconcile import enrich_asset_with_face_data, reconcile_frigate_mappings
from .upload_tracker import (
get_lowest_quality_mapped_file,
get_most_redundant_mapped_file,
@@ -32,7 +32,6 @@ from .upload_tracker import (
has_frigate_scores,
mark_rejected,
mark_uploaded,
record_frigate_files_batch,
remove_frigate_file,
)
@@ -55,112 +54,6 @@ def _safe_person_dir(output_dir: str, person_name: str) -> str:
return candidate
def _reconcile_frigate_mappings(
person_name: str,
known_files_before: set[str],
uploaded: list[tuple[str, str | None]],
) -> None:
"""Map Frigate filenames to asset IDs after a batch of uploads.
Polls until all expected new files appear in the Frigate API, then maps
them to asset IDs by filename timestamp order (Frigate processes the
upload queue in FIFO order, so earlier uploads get earlier timestamps).
KNOWN LIMITATION — race condition with external uploads:
If another client uploads a face file for this person concurrently, the
count of new files will exceed `len(uploaded)` and we bail out entirely
(the "> target" branch). That's safe — we never record a wrong mapping —
but those uploads become permanently unmapped (they won't be eligible for
quality replacement). The right fix is a Frigate API that returns the
filename in the upload response, removing the need for any post-upload
diffing. Until then, the external-upload guard keeps mappings correct at
the cost of occasionally missing them when another client is active.
"""
target = len(uploaded)
current_files: set[str] = set()
for delay in (1, 2, 4, 8):
time.sleep(delay)
fresh = get_frigate_person_files(person_name)
if fresh is None:
logger.warning(
f"{person_name}: Frigate API unreachable during mapping reconciliation"
" — quality replacement won't target these files"
)
return
current_files = set(fresh)
if len(current_files - known_files_before) >= target:
break
new_files = current_files - known_files_before
if len(new_files) == target:
def _ts(fname: str) -> float:
try:
return float(fname.rsplit("_", 1)[-1].replace(".webp", ""))
except (ValueError, IndexError):
return 0.0
mappings = {
frigate_file: asset_id
for (_, asset_id), frigate_file in zip(uploaded, sorted(new_files, key=_ts))
if asset_id
}
record_frigate_files_batch(person_name, mappings)
elif len(new_files) > target:
logger.info(
f"{person_name}: {len(new_files)} new Frigate files for {target} uploads"
" (external upload detected) — skipping file mapping"
)
else:
logger.warning(
f"{person_name}: only {len(new_files)} of {target} expected Frigate files"
" appeared after reconciliation — mapping skipped"
)
def _enrich_asset_with_face_data(asset: dict, person: dict) -> dict:
"""Enrich an asset dict with face bounding box data from the Immich faces API.
The search/metadata endpoint does not include face bounding box data,
so we fetch it from GET /api/faces?id={asset_id} and inject it into
the asset's "people" field so process_face_mode can find it.
Returns the enriched asset dict (modifies in place and returns it).
"""
person_id = person["id"]
face_data = fetch_face_data(asset["id"], person_id=person_id)
if face_data is None:
logger.debug(f"No face data returned for {person.get('name')} in asset {asset.get('id')}")
# Clean any None entries from the people list (can come from Immich API)
if "people" in asset:
asset["people"] = [p for p in asset["people"] if p is not None]
return asset
# Skip zero-area bounding boxes (face detection failed or no face found)
if face_data.bbox == (0, 0, 0, 0):
logger.debug(f"Zero-area bounding box for {person.get('name')} in asset {asset.get('id')}")
# Clean any None entries from the people list (can come from Immich API)
if "people" in asset:
asset["people"] = [p for p in asset["people"] if p is not None]
return asset
face_info = {
"boundingBoxX1": face_data.bbox[0],
"boundingBoxY1": face_data.bbox[1],
"boundingBoxX2": face_data.bbox[2],
"boundingBoxY2": face_data.bbox[3],
"imageWidth": face_data.image_width,
"imageHeight": face_data.image_height,
}
# Inject into asset so process_face_mode can find it via asset["people"]
asset["people"] = [{"id": person_id, "faces": [face_info]}]
asset["face_confidence"] = face_data.confidence
return asset
def execute_jobs(jobs: list[dict]) -> None:
"""Download and process images for all jobs.
@@ -184,7 +77,7 @@ def execute_jobs(jobs: list[dict]) -> None:
insightface_app = get_insightface_app()
except Exception as e:
logger.debug(f"InsightFace unavailable for crop alignment: {e}")
logger.debug("InsightFace unavailable for crop alignment: %s", e)
with Progress(
SpinnerColumn(),
@@ -221,7 +114,7 @@ def execute_jobs(jobs: list[dict]) -> None:
try:
# Enrich the asset with face bounding box data from the Immich
# faces API (not included in search/metadata results).
asset = _enrich_asset_with_face_data(asset, person)
asset = enrich_asset_with_face_data(asset, person)
# Skip download if detection confidence already disqualifies
# the asset — avoids fetching a large image we'll discard.
conf = asset.get("face_confidence")
@@ -271,7 +164,7 @@ def execute_jobs(jobs: list[dict]) -> None:
score_img.thumbnail((1440, 1440), Image.LANCZOS)
score_map[filename] = assess_quality(score_img).blur_score
except Exception as exc:
logger.debug(f"Quality score fallback for {asset['id']}: {exc}")
logger.debug("Quality score fallback for %s: %s", asset["id"], exc)
score_map[filename] = 0.0 # unknown quality — treat as lowest
count += 1
@@ -280,7 +173,7 @@ def execute_jobs(jobs: list[dict]) -> None:
f"[yellow]Skipped {asset['id']} (no usable face data)[/yellow]"
)
except Exception as e:
logger.error(f"Failed to process asset {asset['id']}: {e}")
logger.error("Failed to process asset %s: %s", asset["id"], e)
progress.advance(job_task)
progress.advance(overall_task)
@@ -294,7 +187,7 @@ def execute_jobs(jobs: list[dict]) -> None:
# Log how many images were actually saved vs selected
if count < len(assets):
logger.info(f"{name}: saved {count}/{len(assets)} selected images")
logger.info("%s: saved %s/%s selected images", name, count, len(assets))
def upload_to_frigate(jobs: list[dict]) -> None:
@@ -558,7 +451,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
effective_count -= 1
min_quality_score_for_slot = None if using_fscore else candidate_score
else:
logger.warning(f"Failed to delete {target_frigate_file} for {name}, skipping replacement")
logger.warning("Failed to delete %s for %s, skipping replacement", target_frigate_file, name)
failed_deletes.add(target_frigate_file)
progress.advance(upload_task)
continue
@@ -611,7 +504,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
if resp.status_code == 400:
progress.console.print(f" [dim]{error_detail}[/dim]")
else:
logger.debug(f"{fname} HTTP {resp.status_code}: {error_detail}")
logger.debug("%s HTTP %s: %s", fname, resp.status_code, error_detail)
if resp.status_code == 400 and "face" in full_body.lower():
asset_id = asset_map.get(fname)
if asset_id:
@@ -656,7 +549,7 @@ def upload_to_frigate(jobs: list[dict]) -> None:
# Batch-map Frigate filenames to asset IDs now that all uploads are done.
if actually_uploaded:
_reconcile_frigate_mappings(name, known_frigate_files_at_start, actually_uploaded)
reconcile_frigate_mappings(name, known_frigate_files_at_start, actually_uploaded)
# Per-person summary
if person_failed == 0:
winnow/frigate_api.py
+6 -6
View File
@@ -36,7 +36,7 @@ def _get_faces_data() -> dict | None:
resp.raise_for_status()
return resp.json()
except Exception as e:
logger.warning(f"Could not query Frigate faces API: {e}")
logger.warning("Could not query Frigate faces API: %s", e)
return None
@@ -125,7 +125,7 @@ def recognize_face(file_path: str) -> tuple[str | None, float] | None:
return (data.get("face_name"), round(float(data["score"]), 4))
return None
except Exception as e:
logger.debug(f"Frigate recognize failed for {file_path}: {e}")
logger.debug("Frigate recognize failed for %s: %s", file_path, e)
return None
@@ -147,15 +147,15 @@ def delete_frigate_person_files(person_name: str, filenames: list[str]) -> bool:
timeout=10,
)
if resp.ok:
logger.debug(f"Deleted {len(filenames)} Frigate file(s) for {person_name}")
logger.debug("Deleted %s Frigate file(s) for %s", len(filenames), person_name)
return True
if resp.status_code == 404:
# File already absent — stale tracker entry. Return True so the caller
# removes it from the tracker and frees the slot cleanly.
logger.warning(f"Frigate file(s) not found for {person_name} (stale tracker entry?): {filenames}")
logger.warning("Frigate file(s) not found for %s (stale tracker entry?): %s", person_name, filenames)
return True
logger.warning(f"Frigate delete returned {resp.status_code} for {person_name}")
logger.warning("Frigate delete returned %s for %s", resp.status_code, person_name)
return False
except Exception as e:
logger.warning(f"Failed to delete Frigate files for {person_name}: {e}")
logger.warning("Failed to delete Frigate files for %s: %s", person_name, e)
return False
winnow/image_processing.py
+5 -5
View File
@@ -37,7 +37,7 @@ def align_face(img: Image.Image, landmarks: list[list[float]] | np.ndarray) -> I
img_np = np.asarray(img)
lm = np.array(landmarks, dtype=np.float32)
if lm.shape != (5, 2):
logger.debug(f"Invalid landmark shape: {lm.shape}, expected (5, 2)")
logger.debug("Invalid landmark shape: %s, expected (5, 2)", lm.shape)
return None
aligned = norm_crop(img_np, lm)
return Image.fromarray(aligned)
@@ -45,7 +45,7 @@ def align_face(img: Image.Image, landmarks: list[list[float]] | np.ndarray) -> I
logger.debug("InsightFace not available for face alignment")
return None
except Exception as e:
logger.debug(f"Face alignment failed: {e}")
logger.debug("Face alignment failed: %s", e)
return None
@@ -79,7 +79,7 @@ def process_face_mode(
break
if not face_info:
logger.debug(f"No face info for {person.get('name')} in asset {asset.get('id')}")
logger.debug("No face info for %s in asset %s", person.get("name"), asset.get("id"))
return None
img_w, img_h = img.size
@@ -95,7 +95,7 @@ def process_face_mode(
face_w, face_h = x2 - x1, y2 - y1
if face_w < min_width or face_h < min_width:
logger.debug(f"Face too small ({face_w:.1f}x{face_h:.1f})")
logger.debug("Face too small (%.1fx%.1f)", face_w, face_h)
return None
# Re-detect face with InsightFace for landmark-based alignment.
@@ -131,7 +131,7 @@ def process_face_mode(
_save_jpeg(aligned, os.path.join(output_dir, f"{count}.jpg"))
return aligned.size
except Exception as e:
logger.debug(f"InsightFace re-detection failed for {asset.get('id')}: {e}")
logger.debug("InsightFace re-detection failed for %s: %s", asset.get("id"), e)
# Landmark alignment from Immich metadata (Immich does not currently
# expose landmarks, so this path is a future-proofing fallback)
winnow/immich_api.py
+14 -14
View File
@@ -59,7 +59,7 @@ def get_people() -> list[dict]:
resp.raise_for_status()
return resp.json().get("people", [])
except (requests.RequestException, ValueError) as e:
logger.error(f"Failed to fetch people from Immich: {e}")
logger.error("Failed to fetch people from Immich: %s", e)
return []
@@ -79,7 +79,7 @@ def merge_people(survivor_id: str, merge_ids: list[str]) -> bool:
resp.raise_for_status()
return True
except requests.RequestException as e:
logger.error(f"Failed to merge people into {survivor_id}: {e}")
logger.error("Failed to merge people into %s: %s", survivor_id, e)
return False
@@ -90,7 +90,7 @@ def fetch_all_assets(person: dict) -> list[dict]:
url = f"{Config.IMMICH_URL}/api/search/metadata"
page_size = 1000
logger.debug(f"Fetching assets for {name}...")
logger.debug("Fetching assets for %s...", name)
assets = []
for page in range(1, MAX_PAGES + 1):
@@ -103,7 +103,7 @@ def fetch_all_assets(person: dict) -> list[dict]:
)
if not resp.ok:
logger.error(f"Error fetching assets for {name} (page {page}): {resp.status_code}")
logger.error("Error fetching assets for %s (page %s): %s", name, page, resp.status_code)
break
page_assets = resp.json().get("assets", [])
@@ -114,13 +114,13 @@ def fetch_all_assets(person: dict) -> list[dict]:
break
assets.extend(a for a in page_assets if isinstance(a, dict))
logger.debug(f"Fetched page {page}, total: {len(assets)}")
logger.debug("Fetched page %s, total: %s", page, len(assets))
if len(page_assets) < page_size or len(assets) >= _MAX_ASSETS_PER_PERSON:
break
except (requests.RequestException, ValueError) as e:
logger.error(f"Exception fetching assets for {name}: {e}")
logger.error("Exception fetching assets for %s: %s", name, e)
break
return assets
@@ -148,7 +148,7 @@ def fetch_face_data(asset_id: str, person_id: str | None = None) -> FaceData | N
)
if not resp.ok:
logger.debug(f"Face data endpoint returned {resp.status_code} for {asset_id}")
logger.debug("Face data endpoint returned %s for %s", resp.status_code, asset_id)
return None
faces = resp.json()
@@ -181,10 +181,10 @@ def fetch_face_data(asset_id: str, person_id: str | None = None) -> FaceData | N
)
except requests.RequestException as e:
logger.debug(f"Failed to fetch face data for {asset_id}: {e}")
logger.debug("Failed to fetch face data for %s: %s", asset_id, e)
return None
except (AttributeError, KeyError, TypeError, ValueError) as e:
logger.debug(f"Failed to parse face data for {asset_id}: {e}")
logger.debug("Failed to parse face data for %s: %s", asset_id, e)
return None
@@ -205,9 +205,9 @@ def fetch_full_image(asset_id: str, timeout: int = 60) -> Image.Image | None:
try:
return ImageOps.exif_transpose(Image.open(BytesIO(resp.content)))
except Exception:
logger.debug(f"PIL can't open original for {asset_id}, falling back to preview")
logger.debug("PIL can't open original for %s, falling back to preview", asset_id)
except requests.RequestException:
logger.debug(f"Original request failed for {asset_id}, falling back to preview")
logger.debug("Original request failed for %s, falling back to preview", asset_id)
# Fall back to preview thumbnail (always JPEG)
try:
@@ -219,7 +219,7 @@ def fetch_full_image(asset_id: str, timeout: int = 60) -> Image.Image | None:
if resp.ok:
return ImageOps.exif_transpose(Image.open(BytesIO(resp.content)))
except Exception as e:
logger.error(f"Failed to fetch image {asset_id}: {e}")
logger.error("Failed to fetch image %s: %s", asset_id, e)
return None
@@ -229,7 +229,7 @@ def filter_recent_assets(assets: list[dict], years: int | None = None) -> list[d
years = years or Config.YEARS_FILTER
cutoff = datetime.now(timezone.utc) - timedelta(days=365 * years)
logger.debug(f"Filtering assets older than {years} years ({cutoff})")
logger.debug("Filtering assets older than %s years (%s)", years, cutoff)
recent, skipped = [], 0
for asset in assets:
@@ -247,6 +247,6 @@ def filter_recent_assets(assets: list[dict], years: int | None = None) -> list[d
except ValueError:
continue
logger.debug(f"Retained {len(recent)} assets (filtered {skipped} old assets).")
logger.debug("Retained %s assets (filtered %s old assets).", len(recent), skipped)
return recent
winnow/jobs.py
+68 -43
View File
@@ -120,13 +120,36 @@ def _perform_selection(
return selected
def _build_job(
person: dict,
recent_assets: list,
limit: int | str,
selection_mode: str,
retry_rejected: bool = False,
quality_replacement: bool = False,
) -> dict | None:
"""Build a job dict from pre-fetched assets and parameters. No I/O."""
name = person["name"]
new_asset_ids = set(filter_already_uploaded([a["id"] for a in recent_assets], retry_rejected=retry_rejected))
new_assets = [a for a in recent_assets if a["id"] in new_asset_ids]
if not new_assets:
return None
selected = _perform_selection(new_assets, limit, name, selection_mode, person_id=person["id"])
if not selected:
return None
return {
"person": person,
"assets": selected,
"limit": len(selected),
"config": {"name": name, "quality_replacement": quality_replacement},
}
def _configure_person(person: dict, people: list[dict]) -> dict | None:
"""Configure training for a single person. Returns job dict or None."""
name = person["name"]
console.print(f"\nSelected: [bold green]{name}[/bold green]")
config = {"name": name, "quality_replacement": Config.QUALITY_REPLACEMENT}
# Fetch and filter assets
years = IntPrompt.ask("Filter images older than (years)", default=Config.YEARS_FILTER)
@@ -137,22 +160,6 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None:
rprint(f" Found [bold]{len(all_assets)}[/bold] total, [bold]{len(recent_assets)}[/bold] in range ({years} years).")
# Filter out assets already uploaded to Frigate.
# In interactive mode, ask — use the env var only as the default so it can
# still be pre-set (e.g. RETRY_REJECTED=true) without forcing the answer.
retry_env = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
retry_rejected = Confirm.ask("Include previously rejected images?", default=retry_env)
before_dedup = len(recent_assets)
new_asset_ids = set(filter_already_uploaded([a["id"] for a in recent_assets], retry_rejected=retry_rejected))
recent_assets = [a for a in recent_assets if a["id"] in new_asset_ids]
skipped = before_dedup - len(recent_assets)
if skipped:
rprint(f" [dim]Skipped {skipped} assets already uploaded to Frigate.[/dim]")
if not recent_assets:
rprint(" [dim]Skipping (0 new images after dedup).[/dim]")
return None
# Strategy selection
has_embedding = is_embedding_available()
rprint(f"\n[bold cyan]Select Training Strategy for {name}:[/bold cyan]")
@@ -161,11 +168,30 @@ def _configure_person(person: dict, people: list[dict]) -> dict | None:
if selection_mode == "skip":
return None
# Perform selection
selected_assets = _perform_selection(recent_assets, limit, name, selection_mode, person_id=person["id"])
# In interactive mode, ask about retry_rejected — use env var as default
retry_env = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
retry_rejected = Confirm.ask("Include previously rejected images?", default=retry_env)
rprint(f" [green]Queued {len(selected_assets)} images for {name}.[/green]")
return {"person": person, "assets": selected_assets, "limit": len(selected_assets), "config": config}
job = _build_job(
person,
recent_assets,
limit,
selection_mode,
retry_rejected=retry_rejected,
quality_replacement=Config.QUALITY_REPLACEMENT,
)
if job is None:
rprint(" [dim]Skipping (0 new images after dedup or selection).[/dim]")
return None
# Show how many were skipped (dedup info)
new_asset_ids = set(filter_already_uploaded([a["id"] for a in recent_assets], retry_rejected=retry_rejected))
skipped = len(recent_assets) - len(new_asset_ids)
if skipped:
rprint(f" [dim]Skipped {skipped} assets already uploaded to Frigate.[/dim]")
rprint(f" [green]Queued {job['limit']} images for {name}.[/green]")
return job
def interactive_configure(people: list[dict]) -> list[dict]:
@@ -240,26 +266,12 @@ def auto_configure(people: list[dict]) -> list[dict]:
jobs = []
for person in valid_people:
name = person["name"]
config = {"name": name}
all_assets = fetch_all_assets(person)
recent_assets = filter_recent_assets(all_assets, years=Config.YEARS_FILTER)
rprint(f" {name}: {len(all_assets)} total, {len(recent_assets)} recent")
# Filter out assets already uploaded to Frigate
retry_rejected = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
before_dedup = len(recent_assets)
new_asset_ids = set(filter_already_uploaded([a["id"] for a in recent_assets], retry_rejected=retry_rejected))
recent_assets = [a for a in recent_assets if a["id"] in new_asset_ids]
skipped = before_dedup - len(recent_assets)
if skipped:
rprint(f" [dim]Skipped {skipped} assets already uploaded to Frigate.[/dim]")
if not recent_assets:
rprint(f" [dim]Skipping {name} (0 new images after dedup).[/dim]")
continue
# Enforce MAX_AUTO_IMAGES against the tracked file count only.
# Manually-added Frigate files are invisible to this cap so users can
# curate their own files without shrinking winnow's managed quota.
@@ -281,7 +293,7 @@ def auto_configure(people: list[dict]) -> list[dict]:
else:
quality_replacement_only = False
config["quality_replacement"] = quality_replacement_only or Config.QUALITY_REPLACEMENT
quality_replacement = quality_replacement_only or Config.QUALITY_REPLACEMENT
has_embedding = is_embedding_available()
limit, selection_mode = _resolve_strategy(strategy, has_embedding)
@@ -299,13 +311,26 @@ def auto_configure(people: list[dict]) -> list[dict]:
if selection_mode == "skip":
continue
selected_assets = _perform_selection(recent_assets, limit, name, selection_mode, person_id=person["id"])
if auto_cap is not None:
selected_assets = selected_assets[:auto_cap]
retry_rejected = os.environ.get("RETRY_REJECTED", "false").lower() in ("true", "1", "yes")
job = _build_job(
person,
recent_assets,
limit,
selection_mode,
retry_rejected=retry_rejected,
quality_replacement=quality_replacement,
)
if job is None:
rprint(f" [dim]Skipping {name} (0 new images after dedup or selection).[/dim]")
continue
if selected_assets:
rprint(f" [green]Queued {len(selected_assets)} images for {name}.[/green]")
jobs.append({"person": person, "assets": selected_assets, "limit": len(selected_assets), "config": config})
# Apply auto_cap post-selection if needed
if auto_cap is not None and len(job["assets"]) > auto_cap:
job["assets"] = job["assets"][:auto_cap]
job["limit"] = len(job["assets"])
rprint(f" [green]Queued {job['limit']} images for {name}.[/green]")
jobs.append(job)
return jobs
winnow/reconcile.py
+124
View File
@@ -0,0 +1,124 @@
"""Frigate upload post-processing: reconciliation and asset enrichment."""
import logging
from .frigate_api import get_frigate_person_files
from .immich_api import fetch_face_data
from .upload_tracker import record_frigate_files_batch
logger = logging.getLogger(__name__)
def reconcile_frigate_mappings(
person_name: str,
known_files_before: set[str],
uploaded: list[tuple[str, str | None]],
) -> None:
"""Map Frigate filenames to asset IDs after a batch of uploads.
Polls until all expected new files appear in the Frigate API, then maps
them to asset IDs by filename timestamp order (Frigate processes the
upload queue in FIFO order, so earlier uploads get earlier timestamps).
KNOWN LIMITATION — race condition with external uploads:
If another client uploads a face file for this person concurrently, the
count of new files will exceed `len(uploaded)` and we bail out entirely
(the "> target" branch). That's safe — we never record a wrong mapping —
but those uploads become permanently unmapped (they won't be eligible for
quality replacement). The right fix is a Frigate API that returns the
filename in the upload response, removing the need for any post-upload
diffing. Until then, the external-upload guard keeps mappings correct at
the cost of occasionally missing them when another client is active.
"""
import time
target = len(uploaded)
current_files: set[str] = set()
for delay in (1, 2, 4, 8):
time.sleep(delay)
fresh = get_frigate_person_files(person_name)
if fresh is None:
logger.warning(
"%s: Frigate API unreachable during mapping reconciliation"
" — quality replacement won't target these files",
person_name,
)
return
current_files = set(fresh)
if len(current_files - known_files_before) >= target:
break
new_files = current_files - known_files_before
if len(new_files) == target:
def _ts(fname: str) -> float:
try:
return float(fname.rsplit("_", 1)[-1].replace(".webp", ""))
except (ValueError, IndexError):
return 0.0
mappings = {
frigate_file: asset_id
for (_, asset_id), frigate_file in zip(uploaded, sorted(new_files, key=_ts))
if asset_id
}
record_frigate_files_batch(person_name, mappings)
elif len(new_files) > target:
logger.info(
"%s: %s new Frigate files for %s uploads"
" (external upload detected) — skipping file mapping",
person_name,
len(new_files),
target,
)
else:
logger.warning(
"%s: only %s of %s expected Frigate files"
" appeared after reconciliation — mapping skipped",
person_name,
len(new_files),
target,
)
def enrich_asset_with_face_data(asset: dict, person: dict) -> dict:
"""Enrich an asset dict with face bounding box data from the Immich faces API.
The search/metadata endpoint does not include face bounding box data,
so we fetch it from GET /api/faces?id={asset_id} and inject it into
the asset's "people" field so process_face_mode can find it.
Returns the enriched asset dict (modifies in place and returns it).
"""
person_id = person["id"]
face_data = fetch_face_data(asset["id"], person_id=person_id)
if face_data is None:
logger.debug("No face data returned for %s in asset %s", person.get("name"), asset.get("id"))
# Clean any None entries from the people list (can come from Immich API)
if "people" in asset:
asset["people"] = [p for p in asset["people"] if p is not None]
return asset
# Skip zero-area bounding boxes (face detection failed or no face found)
if face_data.bbox == (0, 0, 0, 0):
logger.debug("Zero-area bounding box for %s in asset %s", person.get("name"), asset.get("id"))
# Clean any None entries from the people list (can come from Immich API)
if "people" in asset:
asset["people"] = [p for p in asset["people"] if p is not None]
return asset
face_info = {
"boundingBoxX1": face_data.bbox[0],
"boundingBoxY1": face_data.bbox[1],
"boundingBoxX2": face_data.bbox[2],
"boundingBoxY2": face_data.bbox[3],
"imageWidth": face_data.image_width,
"imageHeight": face_data.image_height,
}
# Inject into asset so process_face_mode can find it via asset["people"]
asset["people"] = [{"id": person_id, "faces": [face_info]}]
asset["face_confidence"] = face_data.confidence
return asset
winnow/upload_tracker.py
+369 -273
View File
@@ -1,146 +1,211 @@
"""Persistent tracker for Immich asset IDs already uploaded/rejected by Frigate.
"""Persistent tracker for Immich asset IDs uploaded/rejected by Frigate.
Two separate JSON files in CACHE_DIR:
frigate_uploaded_ids.json — successfully uploaded assets
frigate_rejected_ids.json — assets Frigate rejected (e.g. no face detected)
Uses a local SQLite database (frigate_tracker.db) in CACHE_DIR.
Both are excluded from future candidate pools. To reset:
- All: delete both files
- One person: call reset_person("Name") or set RESET_PERSON=Name
- Rejects only: delete frigate_rejected_ids.json, or set RETRY_REJECTED=true
Schema
------
tracked_assets — one row per (asset_id, status) pair
frigate_files — Frigate filename → Immich asset_id mapping
person_metadata — last-known Frigate training image count per person
by_person schema (frigate_uploaded_ids.json):
{
"asset_ids": ["immich-id-1", ...], # all assets we attempted to upload
"scores": {"immich-id-1": 450.3}, # Laplacian blur variance at upload time
"frigate_scores": {"immich-id-1": 0.87}, # Frigate recognition confidence (0-1) pre-upload
"frigate_files": {"PersonName-123.webp": "immich-id-1"}, # Frigate filename → asset ID
"crop_dims": {"immich-id-1": [640, 480]}, # crop pixel dimensions at upload time
"frigate_count": 42 # last known Frigate training image count
}
frigate_scores stores pre-upload recognize scores (0-1 sigmoid-mapped cosine
similarity). High score = the existing training set already covers this face
condition well. Low score = a gap — novel/diverse for the training set.
frigate_files only contains files winnow uploaded — files added manually through
Frigate's UI are never mapped here and are never touched by quality replacement.
Migration
---------
On first open, if the old JSON files exist and the tables are empty, their
data is migrated automatically. The JSON files are then renamed to .json.bak.
"""
import json
import logging
import os
import sqlite3
from pathlib import Path
from .frigate_api import delete_frigate_person_files
logger = logging.getLogger(__name__)
UPLOAD_TRACKER_FILE = "frigate_uploaded_ids.json"
REJECT_TRACKER_FILE = "frigate_rejected_ids.json"
# Legacy JSON filenames (for migration)
_UPLOAD_JSON = "frigate_uploaded_ids.json"
_REJECT_JSON = "frigate_rejected_ids.json"
_DB_NAME = "frigate_tracker.db"
# Write-through in-memory cache keyed by the resolved file path.
# Reduces per-call JSON reads from O(calls) to O(1) after the first load.
# Keyed by full path so tests with isolated tmp dirs never share entries.
_cache: dict[str, dict] = {}
_DDL = """
CREATE TABLE IF NOT EXISTS tracked_assets (
asset_id TEXT NOT NULL,
person_name TEXT,
status TEXT NOT NULL CHECK(status IN ('uploaded', 'rejected')),
blur_score REAL,
crop_width INTEGER,
crop_height INTEGER,
frigate_score REAL,
PRIMARY KEY (asset_id, status)
);
CREATE TABLE IF NOT EXISTS frigate_files (
frigate_filename TEXT PRIMARY KEY,
person_name TEXT NOT NULL,
asset_id TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS person_metadata (
person_name TEXT PRIMARY KEY,
frigate_count INTEGER
);
"""
# Module-level connection state — re-opened when CACHE_DIR changes (test isolation)
_conn: sqlite3.Connection | None = None
_conn_path: str | None = None
def _tracker_path(filename: str) -> Path:
try:
from .config import Config
return Path(Config.CACHE_DIR) / filename
except (ImportError, AttributeError):
return Path(filename)
def _get_conn() -> sqlite3.Connection:
"""Return (or create) the module-level SQLite connection.
Re-opens the connection when Config.CACHE_DIR has changed — this provides
test isolation when the isolated_cache fixture sets a new tmp directory and
calls _Config.reset().
"""
global _conn, _conn_path
def _load(filename: str) -> dict:
path = _tracker_path(filename)
key = str(path)
if key in _cache:
return _cache[key]
data: dict = {}
if path.exists():
from .config import Config
cache_dir = Config.CACHE_DIR
db_path = str(Path(cache_dir) / _DB_NAME)
if _conn is not None and _conn_path != db_path:
try:
with open(path) as f:
data = json.load(f)
except (json.JSONDecodeError, OSError) as e:
logger.warning(f"Could not load tracker {filename}: {e}")
_cache[key] = data
return data
_conn.close()
except Exception:
pass
_conn = None
if _conn is None:
Path(cache_dir).mkdir(parents=True, exist_ok=True)
_conn = sqlite3.connect(db_path, check_same_thread=False)
_conn.row_factory = sqlite3.Row
_conn.execute("PRAGMA journal_mode=WAL")
_conn.execute("PRAGMA foreign_keys=ON")
_conn.executescript(_DDL)
_conn.commit()
_conn_path = db_path
_maybe_migrate(cache_dir, _conn)
return _conn
def _save(filename: str, data: dict) -> None:
path = _tracker_path(filename)
_cache[str(path)] = data # keep cache consistent with what we write
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f:
json.dump(data, f, indent=2)
# ---------------------------------------------------------------------------
# JSON → SQLite migration
# ---------------------------------------------------------------------------
def _maybe_migrate(cache_dir: str, conn: sqlite3.Connection) -> None:
"""If the old JSON files exist and DB is empty, migrate and rename them."""
base = Path(cache_dir)
upload_json = base / _UPLOAD_JSON
reject_json = base / _REJECT_JSON
if not upload_json.exists() and not reject_json.exists():
return
# Check if tables are already populated
row = conn.execute("SELECT COUNT(*) FROM tracked_assets").fetchone()
if row[0] > 0:
return # already migrated
logger.info("Migrating JSON tracker files to SQLite in %s", cache_dir)
with conn:
if upload_json.exists():
try:
data = json.loads(upload_json.read_text())
_migrate_json_data(conn, data, "uploaded")
upload_json.rename(upload_json.with_suffix(".json.bak"))
except Exception as exc:
logger.warning("Migration of %s failed: %s", upload_json, exc)
if reject_json.exists():
try:
data = json.loads(reject_json.read_text())
_migrate_json_data(conn, data, "rejected")
reject_json.rename(reject_json.with_suffix(".json.bak"))
except Exception as exc:
logger.warning("Migration of %s failed: %s", reject_json, exc)
logger.info("JSON → SQLite migration complete")
def _flat_key(filename: str) -> str:
return "uploaded_asset_ids" if "uploaded" in filename else "rejected_asset_ids"
def _migrate_json_data(conn: sqlite3.Connection, data: dict, status: str) -> None:
"""Insert one JSON tracker file's data into SQLite tables."""
flat_key = "uploaded_asset_ids" if status == "uploaded" else "rejected_asset_ids"
flat_ids: set[str] = set(data.get(flat_key, []))
person_covered: set[str] = set()
for person_name, raw_entry in data.get("by_person", {}).items():
if isinstance(raw_entry, list):
entry: dict = {"asset_ids": raw_entry, "scores": {}, "frigate_scores": {},
"frigate_files": {}, "crop_dims": {}}
else:
entry = {
"asset_ids": raw_entry.get("asset_ids", []),
"scores": raw_entry.get("scores", {}),
"frigate_scores": raw_entry.get("frigate_scores", {}),
"frigate_files": raw_entry.get("frigate_files", {}),
"crop_dims": raw_entry.get("crop_dims", {}),
"frigate_count": raw_entry.get("frigate_count"),
}
for asset_id in entry["asset_ids"]:
person_covered.add(asset_id)
dims = entry.get("crop_dims", {}).get(asset_id)
conn.execute(
"""INSERT OR IGNORE INTO tracked_assets
(asset_id, person_name, status, blur_score,
crop_width, crop_height, frigate_score)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
asset_id,
person_name,
status,
entry.get("scores", {}).get(asset_id),
dims[0] if dims else None,
dims[1] if dims else None,
entry.get("frigate_scores", {}).get(asset_id) if status == "uploaded" else None,
),
)
if status == "uploaded":
for ff, aid in entry.get("frigate_files", {}).items():
conn.execute(
"INSERT OR IGNORE INTO frigate_files (frigate_filename, person_name, asset_id) VALUES (?, ?, ?)",
(ff, person_name, aid),
)
fc = entry.get("frigate_count")
if fc is not None:
conn.execute(
"INSERT OR REPLACE INTO person_metadata (person_name, frigate_count) VALUES (?, ?)",
(person_name, fc),
)
# Flat IDs not covered by any by_person entry → insert with NULL person
for asset_id in flat_ids - person_covered:
conn.execute(
"INSERT OR IGNORE INTO tracked_assets (asset_id, person_name, status) VALUES (?, NULL, ?)",
(asset_id, status),
)
def _load_flat(filename: str) -> set[str]:
return set(_load(filename).get(_flat_key(filename), []))
def _get_ids(entry: list | dict) -> list[str]:
"""Extract asset_ids from either the old list format or the new dict format."""
if isinstance(entry, list):
return entry
return entry.get("asset_ids", [])
def _migrate_entry(entry: list | dict) -> dict:
"""Ensure by_person entry is in the current dict format."""
if isinstance(entry, list):
return {"asset_ids": sorted(entry), "scores": {}, "frigate_scores": {}, "frigate_files": {}, "crop_dims": {}}
entry.setdefault("asset_ids", [])
entry.setdefault("scores", {})
entry.setdefault("frigate_scores", {})
entry.setdefault("frigate_files", {})
entry.setdefault("crop_dims", {})
return entry
def _mark(
filename: str,
asset_id: str,
person_name: str | None,
score: float | None = None,
crop_dims: tuple[int, int] | None = None,
frigate_score: float | None = None,
) -> None:
data = _load(filename)
flat_key = _flat_key(filename)
flat = set(data.get(flat_key, []))
flat.add(asset_id)
data[flat_key] = sorted(flat)
if person_name:
by_person = data.setdefault("by_person", {})
entry = _migrate_entry(by_person.get(person_name, {}))
ids = set(entry["asset_ids"])
ids.add(asset_id)
entry["asset_ids"] = sorted(ids)
if score is not None:
entry["scores"][asset_id] = round(score, 4)
if crop_dims is not None:
entry["crop_dims"][asset_id] = [crop_dims[0], crop_dims[1]]
if frigate_score is not None:
entry["frigate_scores"][asset_id] = round(frigate_score, 4)
by_person[person_name] = entry
_save(filename, data)
# ── Public API ────────────────────────────────────────────────────────────────
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def load_uploaded_ids() -> set[str]:
return _load_flat(UPLOAD_TRACKER_FILE)
conn = _get_conn()
rows = conn.execute("SELECT asset_id FROM tracked_assets WHERE status='uploaded'").fetchall()
return {r[0] for r in rows}
def load_rejected_ids() -> set[str]:
return _load_flat(REJECT_TRACKER_FILE)
conn = _get_conn()
rows = conn.execute("SELECT asset_id FROM tracked_assets WHERE status='rejected'").fetchall()
return {r[0] for r in rows}
def mark_uploaded(
@@ -150,233 +215,264 @@ def mark_uploaded(
crop_dims: tuple[int, int] | None = None,
frigate_score: float | None = None,
) -> None:
_mark(UPLOAD_TRACKER_FILE, asset_id, person_name, score=score, crop_dims=crop_dims, frigate_score=frigate_score)
logger.debug(f"Marked {asset_id} as uploaded ({person_name})")
conn = _get_conn()
with conn:
conn.execute(
"""INSERT OR REPLACE INTO tracked_assets
(asset_id, person_name, status, blur_score, crop_width, crop_height, frigate_score)
VALUES (?, ?, 'uploaded', ?, ?, ?, ?)""",
(
asset_id,
person_name,
round(score, 4) if score is not None else None,
crop_dims[0] if crop_dims else None,
crop_dims[1] if crop_dims else None,
round(frigate_score, 4) if frigate_score is not None else None,
),
)
logger.debug("Marked %s as uploaded (%s)", asset_id, person_name)
def mark_rejected(asset_id: str, person_name: str | None = None) -> None:
_mark(REJECT_TRACKER_FILE, asset_id, person_name)
logger.debug(f"Marked {asset_id} as rejected ({person_name})")
def record_frigate_file(person_name: str, frigate_filename: str, asset_id: str) -> None:
"""Record the mapping from a Frigate training filename to an Immich asset ID."""
data = _load(UPLOAD_TRACKER_FILE)
by_person = data.setdefault("by_person", {})
entry = _migrate_entry(by_person.get(person_name, {}))
entry["frigate_files"][frigate_filename] = asset_id
by_person[person_name] = entry
_save(UPLOAD_TRACKER_FILE, data)
logger.debug(f"Mapped Frigate file {frigate_filename} → {asset_id} ({person_name})")
conn = _get_conn()
with conn:
conn.execute(
"INSERT OR IGNORE INTO tracked_assets (asset_id, person_name, status) VALUES (?, ?, 'rejected')",
(asset_id, person_name),
)
logger.debug("Marked %s as rejected (%s)", asset_id, person_name)
def record_frigate_files_batch(person_name: str, mappings: dict[str, str]) -> None:
"""Record multiple Frigate filename → asset_id mappings in a single load/save."""
"""Record multiple Frigate filename → asset_id mappings in a single transaction."""
if not mappings:
return
data = _load(UPLOAD_TRACKER_FILE)
by_person = data.setdefault("by_person", {})
entry = _migrate_entry(by_person.get(person_name, {}))
entry["frigate_files"].update(mappings)
by_person[person_name] = entry
_save(UPLOAD_TRACKER_FILE, data)
logger.debug(f"Batch-mapped {len(mappings)} Frigate file(s) for {person_name}")
conn = _get_conn()
with conn:
conn.executemany(
"INSERT OR REPLACE INTO frigate_files (frigate_filename, person_name, asset_id) VALUES (?, ?, ?)",
[(ff, person_name, aid) for ff, aid in mappings.items()],
)
logger.debug("Batch-mapped %s Frigate file(s) for %s", len(mappings), person_name)
def remove_frigate_file(person_name: str, frigate_filename: str) -> None:
"""Remove a Frigate filename from the mapping after it has been deleted.
"""Remove a Frigate filename mapping and clear its asset's frigate_score.
Does NOT unmark the source asset_id — the deletion was deliberate and
we don't want to re-upload the inferior image on the next run.
Does NOT unmark the source asset_id — the deletion was deliberate.
"""
data = _load(UPLOAD_TRACKER_FILE)
by_person = data.get("by_person", {})
entry = _migrate_entry(by_person.get(person_name, {}))
asset_id = entry["frigate_files"].pop(frigate_filename, None)
if asset_id:
entry["frigate_scores"].pop(asset_id, None)
by_person[person_name] = entry
_save(UPLOAD_TRACKER_FILE, data)
logger.debug(f"Removed Frigate file mapping {frigate_filename} ({person_name})")
conn = _get_conn()
with conn:
row = conn.execute(
"SELECT asset_id FROM frigate_files WHERE frigate_filename=? AND person_name=?",
(frigate_filename, person_name),
).fetchone()
conn.execute(
"DELETE FROM frigate_files WHERE frigate_filename=? AND person_name=?",
(frigate_filename, person_name),
)
if row:
conn.execute(
"UPDATE tracked_assets SET frigate_score=NULL WHERE asset_id=? AND person_name=?",
(row["asset_id"], person_name),
)
logger.debug("Removed Frigate file mapping %s (%s)", frigate_filename, person_name)
def get_tracked_frigate_file_count(person_name: str) -> int:
"""Return the number of Frigate training files winnow has mapped for this person.
Used as the cap baseline so that manually-added Frigate files do not
consume slots from winnow's managed quota.
"""
data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
return len(entry["frigate_files"])
"""Return the number of Frigate training files winnow has mapped for this person."""
conn = _get_conn()
row = conn.execute(
"SELECT COUNT(*) FROM frigate_files WHERE person_name=?", (person_name,)
).fetchone()
return row[0]
def get_tracked_frigate_filenames(person_name: str) -> set[str]:
"""Return the set of Frigate filenames currently mapped in the tracker for a person.
Used as a pre-upload baseline when the Frigate GET API is unreachable at
upload start, so reconciliation can still identify newly uploaded files.
"""
data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
return set(entry["frigate_files"].keys())
"""Return the set of Frigate filenames currently mapped for a person."""
conn = _get_conn()
rows = conn.execute(
"SELECT frigate_filename FROM frigate_files WHERE person_name=?", (person_name,)
).fetchall()
return {r[0] for r in rows}
def has_frigate_scores(person_name: str) -> bool:
"""Return True if any mapped file for this person has a stored Frigate recognition score."""
data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
frigate_files = entry.get("frigate_files", {})
frigate_scores = entry.get("frigate_scores", {})
return any(asset_id in frigate_scores for asset_id in frigate_files.values())
conn = _get_conn()
row = conn.execute(
"""SELECT COUNT(*) FROM frigate_files ff
JOIN tracked_assets ta ON ta.asset_id=ff.asset_id AND ta.person_name=ff.person_name
WHERE ff.person_name=? AND ta.frigate_score IS NOT NULL""",
(person_name,),
).fetchone()
return row[0] > 0
def _pick_mapped_file(
person_name: str, score_key: str, *, highest: bool, exclude: set[str] | None = None
person_name: str, score_col: str, *, highest: bool, exclude: set[str] | None = None
) -> tuple[str, str, float] | None:
data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
scores = entry.get(score_key, {})
candidates = [
(ff, asset_id, scores[asset_id])
for ff, asset_id in entry.get("frigate_files", {}).items()
if (exclude is None or ff not in exclude) and asset_id in scores
]
if not candidates:
return None
return max(candidates, key=lambda x: x[2]) if highest else min(candidates, key=lambda x: x[2])
conn = _get_conn()
order = "DESC" if highest else "ASC"
rows = conn.execute(
f"""SELECT ff.frigate_filename, ff.asset_id, ta.{score_col}
FROM frigate_files ff
JOIN tracked_assets ta ON ta.asset_id=ff.asset_id AND ta.person_name=ff.person_name
WHERE ff.person_name=? AND ta.{score_col} IS NOT NULL
ORDER BY ta.{score_col} {order}""",
(person_name,),
).fetchall()
for row in rows:
if exclude is None or row[0] not in exclude:
return (row[0], row[1], row[2])
return None
def get_lowest_quality_mapped_file(
person_name: str, exclude: set[str] | None = None
) -> tuple[str, str, float] | None:
"""Return (frigate_filename, asset_id, score) for the mapped file with the lowest
blur score, or None if no mapped files with known scores exist.
Used for quality replacement when no Frigate scores are available.
Pass `exclude` to skip files that failed to delete this run.
"""
return _pick_mapped_file(person_name, "scores", highest=False, exclude=exclude)
"""Return (frigate_filename, asset_id, score) for the mapped file with the lowest blur score."""
return _pick_mapped_file(person_name, "blur_score", highest=False, exclude=exclude)
def get_most_redundant_mapped_file(
person_name: str, exclude: set[str] | None = None
) -> tuple[str, str, float] | None:
"""Return (frigate_filename, asset_id, score) for the mapped file with the highest
Frigate recognition score, or None if no mapped files with Frigate scores exist.
High Frigate score = the training set already covers this face condition well
= the most redundant file and therefore the best replacement target.
Pass `exclude` to skip files that failed to delete this run.
"""
return _pick_mapped_file(person_name, "frigate_scores", highest=True, exclude=exclude)
"""Return (frigate_filename, asset_id, score) for the mapped file with the highest Frigate score."""
return _pick_mapped_file(person_name, "frigate_score", highest=True, exclude=exclude)
def get_frigate_filename_for_asset(person_name: str, asset_id: str) -> str | None:
"""Return the Frigate training filename mapped to this asset ID, or None."""
data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(data.get("by_person", {}).get(person_name, {}))
for frigate_filename, aid in entry["frigate_files"].items():
if aid == asset_id:
return frigate_filename
return None
conn = _get_conn()
row = conn.execute(
"SELECT frigate_filename FROM frigate_files WHERE person_name=? AND asset_id=?",
(person_name, asset_id),
).fetchone()
return row[0] if row else None
def find_by_crop_dimension(size: int) -> list[dict]:
"""Return all tracked crops whose width or height matches `size` pixels.
Returns a list of dicts: {person, asset_id, width, height, blur_score, frigate_filename}.
frigate_filename is None when the Frigate mapping was lost to a reconciliation race.
Returns a list of dicts: {person, asset_id, width, height, blur_score, frigate_score, frigate_filename}.
"""
data = _load(UPLOAD_TRACKER_FILE)
results = []
for person_name, raw_entry in data.get("by_person", {}).items():
entry = _migrate_entry(raw_entry)
scores = entry.get("scores", {})
frigate_files = entry.get("frigate_files", {})
asset_to_frigate = {v: k for k, v in frigate_files.items()}
frigate_scores = entry.get("frigate_scores", {})
for asset_id, dims in entry.get("crop_dims", {}).items():
w, h = dims[0], dims[1]
if w == size or h == size:
results.append({
"person": person_name,
"asset_id": asset_id,
"width": w,
"height": h,
"blur_score": scores.get(asset_id),
"frigate_score": frigate_scores.get(asset_id),
"frigate_filename": asset_to_frigate.get(asset_id),
})
return results
conn = _get_conn()
rows = conn.execute(
"""SELECT ta.person_name, ta.asset_id, ta.crop_width, ta.crop_height,
ta.blur_score, ta.frigate_score, ff.frigate_filename
FROM tracked_assets ta
LEFT JOIN frigate_files ff ON ff.asset_id=ta.asset_id AND ff.person_name=ta.person_name
WHERE ta.status='uploaded' AND (ta.crop_width=? OR ta.crop_height=?)""",
(size, size),
).fetchall()
return [
{
"person": r["person_name"],
"asset_id": r["asset_id"],
"width": r["crop_width"],
"height": r["crop_height"],
"blur_score": r["blur_score"],
"frigate_score": r["frigate_score"],
"frigate_filename": r["frigate_filename"],
}
for r in rows
]
def update_frigate_count(person_name: str, count: int) -> None:
"""Record Frigate's authoritative training image count for a person."""
data = _load(UPLOAD_TRACKER_FILE)
by_person = data.setdefault("by_person", {})
entry = _migrate_entry(by_person.get(person_name, {}))
entry["frigate_count"] = count
by_person[person_name] = entry
_save(UPLOAD_TRACKER_FILE, data)
conn = _get_conn()
with conn:
conn.execute(
"INSERT OR REPLACE INTO person_metadata (person_name, frigate_count) VALUES (?, ?)",
(person_name, count),
)
def reset_person(person_name: str) -> None:
"""Remove all uploaded and rejected records for a given person.
Also deletes winnow-managed Frigate training files so the next run starts
clean rather than uploading on top of orphaned files. Manually-added Frigate
files (not in frigate_files) are never touched. Proceeds with tracker reset
even if Frigate is unreachable.
clean rather than uploading on top of orphaned files.
"""
upload_data = _load(UPLOAD_TRACKER_FILE)
entry = _migrate_entry(upload_data.get("by_person", {}).get(person_name, {}))
frigate_filenames = list(entry.get("frigate_files", {}).keys())
conn = _get_conn()
# Collect Frigate filenames before deleting
frigate_filenames = list(get_tracked_frigate_filenames(person_name))
if frigate_filenames:
if not os.environ.get("FRIGATE_URL", "").strip():
logger.info(f"FRIGATE_URL not set — skipping Frigate file deletion for {person_name}")
logger.info("FRIGATE_URL not set — skipping Frigate file deletion for %s", person_name)
elif delete_frigate_person_files(person_name, frigate_filenames):
logger.info(f"Deleted {len(frigate_filenames)} Frigate file(s) for {person_name}")
logger.info("Deleted %s Frigate file(s) for %s", len(frigate_filenames), person_name)
else:
logger.warning(f"Could not delete Frigate files for {person_name} — tracker reset proceeding anyway")
logger.warning(
"Could not delete Frigate files for %s — tracker reset proceeding anyway", person_name
)
changed = False
tracker_files = ((UPLOAD_TRACKER_FILE, upload_data), (REJECT_TRACKER_FILE, _load(REJECT_TRACKER_FILE)))
for filename, data in tracker_files:
flat_key = _flat_key(filename)
by_person = data.get("by_person", {})
tracker_entry = by_person.pop(person_name, None)
if tracker_entry is not None:
person_ids = set(_get_ids(tracker_entry))
flat = set(data.get(flat_key, [])) - person_ids
data[flat_key] = sorted(flat)
data["by_person"] = by_person
_save(filename, data)
changed = True
if changed:
logger.info(f"Reset tracking data for {person_name}")
else:
logger.debug(f"reset_person: no tracking data found for {person_name}")
with conn:
conn.execute("DELETE FROM frigate_files WHERE person_name=?", (person_name,))
conn.execute("DELETE FROM tracked_assets WHERE person_name=?", (person_name,))
conn.execute("DELETE FROM person_metadata WHERE person_name=?", (person_name,))
logger.info("Reset tracking data for %s", person_name)
def get_person_summary() -> dict[str, dict]:
"""Return {person_name: {uploaded, rejected, frigate_count, scores, frigate_files}} for display/capacity."""
uploaded_data = _load(UPLOAD_TRACKER_FILE).get("by_person", {})
rejected_data = _load(REJECT_TRACKER_FILE).get("by_person", {})
names = set(uploaded_data) | set(rejected_data)
result = {}
for name in sorted(names):
u_entry = uploaded_data.get(name, {})
r_entry = rejected_data.get(name, {})
result[name] = {
"uploaded": len(_get_ids(u_entry)),
"rejected": len(_get_ids(r_entry)),
"frigate_count": u_entry.get("frigate_count") if isinstance(u_entry, dict) else None,
"scores": u_entry.get("scores", {}) if isinstance(u_entry, dict) else {},
"frigate_files": u_entry.get("frigate_files", {}) if isinstance(u_entry, dict) else {},
}
return result
conn = _get_conn()
# Counts per person per status
rows = conn.execute(
"""SELECT person_name, status, COUNT(*) AS cnt
FROM tracked_assets WHERE person_name IS NOT NULL
GROUP BY person_name, status"""
).fetchall()
summary: dict[str, dict] = {}
for r in rows:
name = r["person_name"]
if name not in summary:
summary[name] = {"uploaded": 0, "rejected": 0, "frigate_count": None,
"scores": {}, "frigate_files": {}}
summary[name][r["status"]] = r["cnt"]
# Scores for uploaded assets
score_rows = conn.execute(
"""SELECT person_name, asset_id, blur_score
FROM tracked_assets
WHERE status='uploaded' AND person_name IS NOT NULL AND blur_score IS NOT NULL"""
).fetchall()
for r in score_rows:
name = r["person_name"]
if name not in summary:
summary[name] = {"uploaded": 0, "rejected": 0, "frigate_count": None,
"scores": {}, "frigate_files": {}}
summary[name]["scores"][r["asset_id"]] = r["blur_score"]
# Frigate file mappings
ff_rows = conn.execute(
"SELECT person_name, frigate_filename, asset_id FROM frigate_files"
).fetchall()
for r in ff_rows:
name = r["person_name"]
if name not in summary:
summary[name] = {"uploaded": 0, "rejected": 0, "frigate_count": None,
"scores": {}, "frigate_files": {}}
summary[name]["frigate_files"][r["frigate_filename"]] = r["asset_id"]
# Frigate counts
meta_rows = conn.execute(
"SELECT person_name, frigate_count FROM person_metadata"
).fetchall()
for r in meta_rows:
name = r["person_name"]
if name not in summary:
summary[name] = {"uploaded": 0, "rejected": 0, "frigate_count": None,
"scores": {}, "frigate_files": {}}
summary[name]["frigate_count"] = r["frigate_count"]
return dict(sorted(summary.items()))
def filter_already_uploaded(
@@ -390,5 +486,5 @@ def filter_already_uploaded(
new_ids = [aid for aid in asset_ids if aid not in exclude]
skipped = len(asset_ids) - len(new_ids)
if skipped:
logger.info(f"Skipping {skipped} assets already uploaded or rejected by Frigate")
logger.info("Skipping %s assets already uploaded or rejected by Frigate", skipped)
return new_ids