flan
01fa307031
- frigate_api._get_faces_data now validates the /api/faces response is a dict before returning it, so a malformed body no longer crashes data.items() in get_all_frigate_person_files (and the equivalent data.get() in get_frigate_person_files). - upload_tracker now takes an exclusive flock on a DATA_DIR lock file for every tracker load-mutate-save cycle, so a scheduled run and a manual docker exec against the same DATA_DIR can no longer race a read-modify-write and silently drop the loser's marks. - begin_batch()/flush_batch() now flush to disk every 10 marks instead of deferring the whole per-person upload loop, bounding how many uploaded marks a crash mid-batch can lose.
384 lines
16 KiB
Python
384 lines
16 KiB
Python
"""Tests for upload tracker — mark, filter, reset, and summary logic."""
|
|
|
|
import fcntl
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolated_cache(monkeypatch, tmp_path):
|
|
"""Point tracker at a temp directory so tests don't touch real cache files."""
|
|
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
|
from winnow.config import _Config
|
|
_Config.reset()
|
|
yield tmp_path
|
|
_Config.reset()
|
|
|
|
|
|
def test_filter_returns_all_when_empty():
|
|
from winnow.upload_tracker import filter_already_uploaded
|
|
ids = ["a1", "b2", "c3"]
|
|
assert filter_already_uploaded(ids) == ids
|
|
|
|
|
|
def test_mark_uploaded_excludes_from_filter():
|
|
from winnow.upload_tracker import filter_already_uploaded, mark_uploaded
|
|
mark_uploaded("a1", person_name="Alice")
|
|
result = filter_already_uploaded(["a1", "b2"])
|
|
assert result == ["b2"]
|
|
|
|
|
|
def test_mark_rejected_excludes_from_filter():
|
|
from winnow.upload_tracker import filter_already_uploaded, mark_rejected
|
|
mark_rejected("x9", person_name="Bob")
|
|
result = filter_already_uploaded(["x9", "y8"])
|
|
assert result == ["y8"]
|
|
|
|
|
|
def test_retry_rejected_includes_rejected():
|
|
from winnow.upload_tracker import filter_already_uploaded, mark_rejected
|
|
mark_rejected("x9", person_name="Bob")
|
|
result = filter_already_uploaded(["x9", "y8"], retry_rejected=True)
|
|
assert "x9" in result
|
|
|
|
|
|
def test_reset_person_clears_records():
|
|
from winnow.upload_tracker import filter_already_uploaded, mark_uploaded, reset_person
|
|
mark_uploaded("a1", person_name="Alice")
|
|
mark_uploaded("a2", person_name="Alice")
|
|
mark_uploaded("b1", person_name="Bob")
|
|
reset_person("Alice")
|
|
assert filter_already_uploaded(["a1", "a2"]) == ["a1", "a2"]
|
|
assert filter_already_uploaded(["b1"]) == []
|
|
|
|
|
|
def test_get_person_summary():
|
|
from winnow.upload_tracker import get_person_summary, mark_rejected, mark_uploaded
|
|
mark_uploaded("a1", person_name="Alice")
|
|
mark_uploaded("a2", person_name="Alice")
|
|
mark_rejected("a3", person_name="Alice")
|
|
mark_uploaded("b1", person_name="Bob")
|
|
summary = get_person_summary()
|
|
assert summary["Alice"]["uploaded"] == 2
|
|
assert summary["Alice"]["rejected"] == 1
|
|
assert summary["Bob"]["uploaded"] == 1
|
|
assert summary["Bob"]["rejected"] == 0
|
|
|
|
|
|
def test_duplicate_marks_are_idempotent():
|
|
from winnow.upload_tracker import filter_already_uploaded, mark_uploaded
|
|
mark_uploaded("dup", person_name="Alice")
|
|
mark_uploaded("dup", person_name="Alice")
|
|
assert filter_already_uploaded(["dup", "new"]) == ["new"]
|
|
|
|
|
|
# ── frigate_files mapping ─────────────────────────────────────────────────────
|
|
|
|
def test_record_and_remove_frigate_file():
|
|
from winnow.upload_tracker import get_person_summary, record_frigate_file, remove_frigate_file
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a1")
|
|
assert "Alice-1000.webp" in get_person_summary()["Alice"]["frigate_files"]
|
|
remove_frigate_file("Alice", "Alice-1000.webp")
|
|
assert "Alice-1000.webp" not in get_person_summary()["Alice"]["frigate_files"]
|
|
|
|
|
|
def test_remove_nonexistent_frigate_file_is_safe():
|
|
from winnow.upload_tracker import remove_frigate_file
|
|
# Should not raise even if the file was never recorded
|
|
remove_frigate_file("Alice", "Alice-ghost.webp")
|
|
|
|
|
|
def test_remove_frigate_file_does_not_unmark_asset():
|
|
"""Deleting a Frigate file should not re-expose the source asset for upload."""
|
|
from winnow.upload_tracker import (
|
|
filter_already_uploaded,
|
|
mark_uploaded,
|
|
record_frigate_file,
|
|
remove_frigate_file,
|
|
)
|
|
mark_uploaded("asset-a1", person_name="Alice")
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a1")
|
|
remove_frigate_file("Alice", "Alice-1000.webp")
|
|
# Asset must still be excluded — it was deliberately replaced, not lost
|
|
assert filter_already_uploaded(["asset-a1"]) == []
|
|
|
|
|
|
def test_get_tracked_frigate_file_count_zero_when_empty():
|
|
from winnow.upload_tracker import get_tracked_frigate_file_count
|
|
assert get_tracked_frigate_file_count("Alice") == 0
|
|
|
|
|
|
def test_get_tracked_frigate_file_count_counts_only_mapped():
|
|
"""Only files explicitly recorded via record_frigate_file count toward the cap."""
|
|
from winnow.upload_tracker import get_tracked_frigate_file_count, mark_uploaded, record_frigate_file
|
|
mark_uploaded("asset-a", person_name="Alice")
|
|
mark_uploaded("asset-b", person_name="Alice")
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
|
|
# asset-b is uploaded but not yet mapped — does not count
|
|
assert get_tracked_frigate_file_count("Alice") == 1
|
|
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
|
|
assert get_tracked_frigate_file_count("Alice") == 2
|
|
|
|
|
|
def test_get_lowest_quality_mapped_file_none_when_empty():
|
|
from winnow.upload_tracker import get_lowest_quality_mapped_file
|
|
assert get_lowest_quality_mapped_file("Alice") is None
|
|
|
|
|
|
def test_get_lowest_quality_mapped_file_returns_lowest():
|
|
from winnow.upload_tracker import (
|
|
get_lowest_quality_mapped_file,
|
|
mark_uploaded,
|
|
record_frigate_file,
|
|
)
|
|
mark_uploaded("asset-hi", person_name="Alice", score=0.95)
|
|
mark_uploaded("asset-lo", person_name="Alice", score=0.71)
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-hi")
|
|
record_frigate_file("Alice", "Alice-1001.webp", "asset-lo")
|
|
result = get_lowest_quality_mapped_file("Alice")
|
|
assert result is not None
|
|
frigate_filename, asset_id, score = result
|
|
assert frigate_filename == "Alice-1001.webp"
|
|
assert asset_id == "asset-lo"
|
|
assert score == pytest.approx(0.71, abs=0.001)
|
|
|
|
|
|
def test_get_lowest_quality_mapped_file_skips_unscored():
|
|
"""Files mapped without a score should not be returned as candidates."""
|
|
from winnow.upload_tracker import (
|
|
get_lowest_quality_mapped_file,
|
|
mark_uploaded,
|
|
record_frigate_file,
|
|
)
|
|
mark_uploaded("asset-scored", person_name="Alice", score=0.85)
|
|
mark_uploaded("asset-noscr", person_name="Alice")
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-scored")
|
|
record_frigate_file("Alice", "Alice-1001.webp", "asset-noscr")
|
|
result = get_lowest_quality_mapped_file("Alice")
|
|
assert result is not None
|
|
assert result[1] == "asset-scored" # only scored file is a candidate
|
|
|
|
|
|
# ── get_tracked_frigate_filenames ─────────────────────────────────────────────
|
|
|
|
def test_get_tracked_frigate_filenames_empty():
|
|
from winnow.upload_tracker import get_tracked_frigate_filenames
|
|
assert get_tracked_frigate_filenames("Alice") == set()
|
|
|
|
|
|
def test_get_tracked_frigate_filenames_returns_mapped():
|
|
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_file
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
|
|
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
|
|
assert get_tracked_frigate_filenames("Alice") == {"Alice-1000.webp", "Alice-1001.webp"}
|
|
|
|
|
|
def test_get_tracked_frigate_filenames_excludes_removed():
|
|
from winnow.upload_tracker import (
|
|
get_tracked_frigate_filenames,
|
|
record_frigate_file,
|
|
remove_frigate_file,
|
|
)
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
|
|
record_frigate_file("Alice", "Alice-1001.webp", "asset-b")
|
|
remove_frigate_file("Alice", "Alice-1000.webp")
|
|
assert get_tracked_frigate_filenames("Alice") == {"Alice-1001.webp"}
|
|
|
|
|
|
def test_get_tracked_frigate_filenames_isolated_by_person():
|
|
from winnow.upload_tracker import get_tracked_frigate_filenames, record_frigate_file
|
|
record_frigate_file("Alice", "Alice-1000.webp", "asset-a")
|
|
record_frigate_file("Bob", "Bob-2000.webp", "asset-b")
|
|
assert get_tracked_frigate_filenames("Alice") == {"Alice-1000.webp"}
|
|
assert get_tracked_frigate_filenames("Bob") == {"Bob-2000.webp"}
|
|
|
|
|
|
# ── get_lowest_quality_mapped_file with exclude ───────────────────────────────
|
|
|
|
def test_get_lowest_quality_exclude_skips_specified_file():
|
|
from winnow.upload_tracker import (
|
|
get_lowest_quality_mapped_file,
|
|
mark_uploaded,
|
|
record_frigate_file,
|
|
)
|
|
mark_uploaded("asset-lo", person_name="Alice", score=0.10)
|
|
mark_uploaded("asset-hi", person_name="Alice", score=0.90)
|
|
record_frigate_file("Alice", "Alice-lo.webp", "asset-lo")
|
|
record_frigate_file("Alice", "Alice-hi.webp", "asset-hi")
|
|
result = get_lowest_quality_mapped_file("Alice", exclude={"Alice-lo.webp"})
|
|
assert result is not None
|
|
assert result[1] == "asset-hi" # lo was excluded; hi is returned
|
|
|
|
|
|
def test_get_lowest_quality_exclude_all_returns_none():
|
|
from winnow.upload_tracker import (
|
|
get_lowest_quality_mapped_file,
|
|
mark_uploaded,
|
|
record_frigate_file,
|
|
)
|
|
mark_uploaded("asset-a", person_name="Alice", score=0.50)
|
|
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
|
assert get_lowest_quality_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
|
|
|
|
|
|
# ── get_most_redundant_mapped_file ────────────────────────────────────────────
|
|
|
|
def test_get_most_redundant_none_when_no_frigate_scores():
|
|
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
|
mark_uploaded("asset-a", person_name="Alice", score=0.80)
|
|
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
|
# blur score only, no frigate_score → no candidates
|
|
assert get_most_redundant_mapped_file("Alice") is None
|
|
|
|
|
|
def test_get_most_redundant_returns_highest_frigate_score():
|
|
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
|
mark_uploaded("asset-novel", person_name="Alice", score=0.50, frigate_score=0.31)
|
|
mark_uploaded("asset-redundant", person_name="Alice", score=0.90, frigate_score=0.88)
|
|
record_frigate_file("Alice", "Alice-novel.webp", "asset-novel")
|
|
record_frigate_file("Alice", "Alice-redundant.webp", "asset-redundant")
|
|
result = get_most_redundant_mapped_file("Alice")
|
|
assert result is not None
|
|
frigate_filename, asset_id, score = result
|
|
assert frigate_filename == "Alice-redundant.webp"
|
|
assert asset_id == "asset-redundant"
|
|
assert score == pytest.approx(0.88, abs=0.001)
|
|
|
|
|
|
def test_get_most_redundant_exclude_skips_file():
|
|
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
|
mark_uploaded("asset-hi", person_name="Alice", score=0.9, frigate_score=0.85)
|
|
mark_uploaded("asset-lo", person_name="Alice", score=0.5, frigate_score=0.40)
|
|
record_frigate_file("Alice", "Alice-hi.webp", "asset-hi")
|
|
record_frigate_file("Alice", "Alice-lo.webp", "asset-lo")
|
|
result = get_most_redundant_mapped_file("Alice", exclude={"Alice-hi.webp"})
|
|
assert result is not None
|
|
assert result[1] == "asset-lo" # hi excluded; lo is next highest
|
|
|
|
|
|
def test_get_most_redundant_exclude_all_returns_none():
|
|
from winnow.upload_tracker import get_most_redundant_mapped_file, mark_uploaded, record_frigate_file
|
|
mark_uploaded("asset-a", person_name="Alice", score=0.5, frigate_score=0.70)
|
|
record_frigate_file("Alice", "Alice-a.webp", "asset-a")
|
|
assert get_most_redundant_mapped_file("Alice", exclude={"Alice-a.webp"}) is None
|
|
|
|
|
|
# ── cross-process locking ──────────────────────────────────────────────────────
|
|
|
|
def test_tracker_lock_blocks_concurrent_exclusive_access():
|
|
"""While upload_tracker holds the tracker lock, a second (non-blocking) attempt
|
|
to exclusively lock the same file must fail — proving the lock is real and
|
|
guards a second winnow process from racing a read-modify-write."""
|
|
from winnow import upload_tracker as ut
|
|
|
|
ut._acquire_lock()
|
|
try:
|
|
lock_path = ut._lock_path()
|
|
assert lock_path.exists()
|
|
fd = os.open(lock_path, os.O_RDWR)
|
|
try:
|
|
with pytest.raises(OSError):
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
finally:
|
|
os.close(fd)
|
|
finally:
|
|
ut._release_lock()
|
|
|
|
# Released — a second exclusive, non-blocking lock now succeeds immediately.
|
|
fd = os.open(ut._lock_path(), os.O_RDWR)
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def test_tracker_lock_reentrant_within_process():
|
|
"""Nested acquisitions (e.g. begin_batch() for both tracker files, or a tracker
|
|
call made while a batch is open) must not deadlock the process that holds them."""
|
|
from winnow import upload_tracker as ut
|
|
|
|
assert ut._lock_depth == 0
|
|
ut._acquire_lock()
|
|
ut._acquire_lock()
|
|
assert ut._lock_depth == 2
|
|
ut._release_lock()
|
|
assert ut._lock_depth == 1
|
|
ut._release_lock()
|
|
assert ut._lock_depth == 0
|
|
|
|
|
|
def test_begin_flush_batch_releases_lock_for_next_caller():
|
|
"""begin_batch()/flush_batch() (including a nested mark_uploaded call inside the
|
|
batch) must fully release the lock so it doesn't stay held for the rest of the run."""
|
|
from winnow import upload_tracker as ut
|
|
from winnow.upload_tracker import (
|
|
REJECT_TRACKER_FILE,
|
|
UPLOAD_TRACKER_FILE,
|
|
begin_batch,
|
|
flush_batch,
|
|
mark_uploaded,
|
|
)
|
|
|
|
begin_batch(UPLOAD_TRACKER_FILE)
|
|
begin_batch(REJECT_TRACKER_FILE)
|
|
mark_uploaded("a1", person_name="Alice")
|
|
flush_batch(UPLOAD_TRACKER_FILE)
|
|
flush_batch(REJECT_TRACKER_FILE)
|
|
|
|
assert ut._lock_depth == 0
|
|
fd = os.open(ut._lock_path(), os.O_RDWR)
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
# ── batched writes: incremental flush bounds crash loss ───────────────────────
|
|
|
|
def test_batch_defers_writes_below_flush_threshold(isolated_cache):
|
|
"""Below the flush threshold, writes stay in memory — batching still avoids
|
|
a disk write per mark."""
|
|
from winnow.upload_tracker import UPLOAD_TRACKER_FILE, begin_batch, flush_batch, mark_uploaded
|
|
|
|
tracker_path = isolated_cache / UPLOAD_TRACKER_FILE
|
|
begin_batch(UPLOAD_TRACKER_FILE)
|
|
try:
|
|
mark_uploaded("asset-0", person_name="Alice")
|
|
assert not tracker_path.exists()
|
|
finally:
|
|
flush_batch(UPLOAD_TRACKER_FILE)
|
|
assert tracker_path.exists()
|
|
|
|
|
|
def test_batch_flushes_incrementally_bounding_crash_loss(isolated_cache):
|
|
"""A crash mid-batch (SIGKILL/OOM/host crash) must lose at most a bounded number
|
|
of marks, not the whole per-person batch — verified by reading the file straight
|
|
off disk before flush_batch() is ever called."""
|
|
from winnow.upload_tracker import _BATCH_FLUSH_EVERY, UPLOAD_TRACKER_FILE, begin_batch, flush_batch, mark_uploaded
|
|
|
|
tracker_path = isolated_cache / UPLOAD_TRACKER_FILE
|
|
begin_batch(UPLOAD_TRACKER_FILE)
|
|
try:
|
|
for i in range(_BATCH_FLUSH_EVERY):
|
|
mark_uploaded(f"asset-{i}", person_name="Alice")
|
|
# Threshold reached: disk already reflects all marks so far, even though
|
|
# flush_batch() has not run yet — simulates surviving a crash here.
|
|
on_disk = json.loads(tracker_path.read_text())
|
|
ids = on_disk["by_person"]["Alice"]["asset_ids"]
|
|
assert len(ids) == _BATCH_FLUSH_EVERY
|
|
|
|
# One more mark past the threshold stays deferred again until the next
|
|
# periodic flush or flush_batch().
|
|
mark_uploaded("asset-extra", person_name="Alice")
|
|
on_disk = json.loads(tracker_path.read_text())
|
|
assert len(on_disk["by_person"]["Alice"]["asset_ids"]) == _BATCH_FLUSH_EVERY
|
|
finally:
|
|
flush_batch(UPLOAD_TRACKER_FILE)
|
|
|
|
on_disk = json.loads(tracker_path.read_text())
|
|
assert len(on_disk["by_person"]["Alice"]["asset_ids"]) == _BATCH_FLUSH_EVERY + 1
|