modified to deduplicate and overfetch

3 additions only:

New deduplicate_results() function (lines 30-72)
Modified search() function (lines 77-94) - now over-fetches and deduplicates
Updated UI text - shows "unique results"

Everything else stays the same.
This will now automatically filter out duplicates before returning results!RetryIM

app.py

@@ -25,7 +25,64 @@ print("🧠 Loading MedCPT model...")
 model = SentenceTransformer('ncbi/MedCPT-Query-Encoder')
 print("✅ Model ready")
 
-# Search function
+# ============================================================================
+# NEW: Deduplication function
+# ============================================================================
+def deduplicate_results(results, target_count):
+    """
+    Remove duplicate questions based on:
+    1. High text similarity (>0.92) - catches near-exact duplicates
+    2. Same answer + moderate similarity (>0.85) - catches conceptual duplicates
+    """
+    if not results['documents'][0]:
+        return results
+    
+    documents = results['documents'][0]
+    metadatas = results['metadatas'][0]
+    distances = results['distances'][0]
+    
+    selected_indices = []
+    
+    for i in range(len(documents)):
+        is_duplicate = False
+        current_answer = metadatas[i].get('answer', '')
+        
+        # Compare to already-selected results
+        for j in selected_indices:
+            selected_answer = metadatas[j].get('answer', '')
+            
+            # Calculate similarity between questions
+            # Lower distance = higher similarity
+            dist_diff = abs(distances[i] - distances[j])
+            
+            # Rule 1: Very similar questions (likely exact/near-exact duplicates)
+            if dist_diff < 0.08:  # Roughly equivalent to >0.92 similarity
+                is_duplicate = True
+                break
+            
+            # Rule 2: Same answer + similar question (conceptual duplicates)
+            if current_answer == selected_answer and dist_diff < 0.15:  # ~0.85 similarity
+                is_duplicate = True
+                break
+        
+        if not is_duplicate:
+            selected_indices.append(i)
+            
+            # Stop when we have enough unique results
+            if len(selected_indices) >= target_count:
+                break
+    
+    # Return filtered results in same format
+    return {
+        'documents': [[documents[i] for i in selected_indices]],
+        'metadatas': [[metadatas[i] for i in selected_indices]],
+        'distances': [[distances[i] for i in selected_indices]],
+        'ids': [[results['ids'][0][i] for i in selected_indices]] if 'ids' in results else None
+    }
+
+# ============================================================================
+# MODIFIED: Search function with deduplication
+# ============================================================================
 def search(query, num_results=3, source_filter=None):
     emb = model.encode(query).tolist()
     
@@ -34,11 +91,17 @@ def search(query, num_results=3, source_filter=None):
     if source_filter and source_filter != "all":
         where_clause = {"source": source_filter}
     
-    return collection.query(
+    # Over-fetch to ensure we get enough unique results
+    fetch_count = min(num_results * 4, 50)  # Fetch 4x but cap at 50
+    
+    results = collection.query(
         query_embeddings=[emb], 
-        n_results=int(num_results),
+        n_results=fetch_count,
         where=where_clause
     )
+    
+    # Deduplicate and return only requested number
+    return deduplicate_results(results, num_results)
 
 # Enhanced Gradio UI
 def ui_search(query, num_results=3, source_filter="all"):
@@ -51,7 +114,7 @@ def ui_search(query, num_results=3, source_filter="all"):
         if not r['documents'][0]:
             return "❌ No results found"
         
-        out = f"🔍 Found {len(r['documents'][0])} results\n\n"
+        out = f"🔍 Found {len(r['documents'][0])} unique results\n\n"
         
         for i in range(len(r['documents'][0])):
             source = r['metadatas'][0][i].get('source', 'unknown')
@@ -98,6 +161,8 @@ with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
     
     Search across **Med-Gemini** (expert explanations) and **MedQA** (USMLE questions) databases.
     Uses medical-specific embeddings (MedCPT) for accurate retrieval.
+    
+    ✨ **New**: Automatic deduplication removes similar/duplicate questions
     """)
     
     with gr.Row():