From 0a8a0c16ddc8af812bb24970e3190d8cf61c5e5b Mon Sep 17 00:00:00 2001
From: Holden <holden@arch.fyi>
Date: Sun, 14 Jun 2026 01:26:41 +0000
Subject: [PATCH] Deduplicate near-identical embeddings before diversity
 selection

Burst shots produce embeddings that differ slightly (~0.01-0.05 cosine
distance) due to JPEG noise and minor lighting variation, so FPS does not
filter them. Add a greedy dedup pass after embedding collection: sort
candidates by quality score descending, then drop any candidate within
0.10 cosine distance of an already-kept image. The best frame from each
near-identical group survives; the rest are dropped before clustering.
---
 winnow/diversity.py | 64 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/winnow/diversity.py b/winnow/diversity.py
index 8976d0d..2d026b9 100644
--- a/winnow/diversity.py
+++ b/winnow/diversity.py
@@ -281,7 +281,16 @@ def _select_by_embedding(
         logger.warning(f"Only {len(valid_candidates)} valid embeddings. Returning all.")
         return valid_candidates
 
-    # --- Phase 5: Cluster-aware selection ---
+    # --- Phase 5: Near-duplicate removal ---
+    # Burst shots and repeated near-identical photos produce embeddings that are
+    # close but not identical, so FPS doesn't filter them out on its own.
+    # Greedily drop any candidate within DEDUP_THRESHOLD cosine distance of a
+    # higher-quality image already in the kept set.
+    embeddings, valid_candidates, confidence_scores = _dedup_embeddings(
+        embeddings, valid_candidates, confidence_scores
+    )
+
+    # --- Phase 6: Cluster-aware selection ---
     return _cluster_aware_selection(
         embeddings,
         valid_candidates,
@@ -291,6 +300,59 @@ def _select_by_embedding(
     )
 
 
+# =============================================================================
+# Near-Duplicate Removal
+# =============================================================================
+
+_DEDUP_THRESHOLD = 0.10  # cosine distance — burst shots are ~0.01-0.05 apart
+
+
+def _dedup_embeddings(
+    embeddings: list,
+    candidates: list,
+    confidence_scores: list,
+) -> tuple[list, list, list]:
+    """Greedy near-duplicate removal before clustering.
+
+    Sorts by quality score descending (best first), then for each candidate
+    drops it if any already-kept embedding is within _DEDUP_THRESHOLD cosine
+    distance. This eliminates burst-shot near-duplicates while preserving the
+    highest-quality representative from each near-identical group.
+    """
+    if len(embeddings) < 2:
+        return embeddings, candidates, confidence_scores
+
+    emb_matrix = np.vstack(embeddings)
+    norms = np.linalg.norm(emb_matrix, axis=1, keepdims=True)
+    emb_normed = emb_matrix / np.maximum(norms, 1e-8)
+
+    # Sort by quality descending so the best image in each near-duplicate group wins
+    quality_scores = [c.get("quality_score") or 0.0 for c in candidates]
+    order = sorted(range(len(candidates)), key=lambda i: quality_scores[i], reverse=True)
+
+    kept_indices = []
+    kept_normed = []
+
+    for i in order:
+        if kept_normed:
+            kept_stack = np.vstack(kept_normed)
+            sims = emb_normed[i] @ kept_stack.T
+            if np.any(sims > 1 - _DEDUP_THRESHOLD):
+                continue
+        kept_indices.append(i)
+        kept_normed.append(emb_normed[i])
+
+    dropped = len(embeddings) - len(kept_indices)
+    if dropped:
+        logger.info(f"Near-duplicate removal dropped {dropped} images (threshold {_DEDUP_THRESHOLD}).")
+
+    return (
+        [embeddings[i] for i in kept_indices],
+        [candidates[i] for i in kept_indices],
+        [confidence_scores[i] for i in kept_indices],
+    )
+
+
 # =============================================================================
 # K-Medoids (Lightweight Implementation)
 # =============================================================================