update

Implement Solution A - Modify your HuggingFace app.py to add the /batch_search_medqa endpoint. This gives you:

✅ True batch processing (1 API call for 60 objectives)
✅ Automatic deduplication on the server
✅ Pre-organized results
✅ No user interaction needed
✅ Scales to hundreds of objectives

app.py

@@ -7,7 +7,9 @@ from sentence_transformers import SentenceTransformer
 import gradio as gr
 from fastapi import FastAPI
 from pydantic import BaseModel
+from typing import List, Optional
 import re
+import time
 
 # Extract and load database
 DB_PATH = "./medqa_db"
@@ -17,7 +19,7 @@ if not os.path.exists(DB_PATH) and os.path.exists("./medqa_db.zip"):
         z.extractall(".")
     print("✅ Database extracted")
 
-print("🔌 Loading ChromaDB...")
+print("📌 Loading ChromaDB...")
 client = chromadb.PersistentClient(path=DB_PATH)
 collection = client.get_collection("medqa")
 print(f"✅ Loaded {collection.count()} questions")
@@ -191,7 +193,7 @@ def ui_search(query, num_results=3, source_filter="all"):
 # Create Gradio interface
 with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
     gr.Markdown("""
-    # 🏥 MedQA Semantic Search
+    # 🥼 MedQA Semantic Search
     
     Search across **Med-Gemini** (expert explanations) and **MedQA** (USMLE questions) databases.
     Uses medical-specific embeddings (MedCPT) for accurate retrieval.
@@ -271,6 +273,11 @@ class SearchRequest(BaseModel):
     num_results: int = 3
     source_filter: str = None
 
+class BatchSearchRequest(BaseModel):
+    queries: List[str]
+    num_results_per_query: int = 10
+    source_filter: Optional[str] = None
+
 @app.post("/search_medqa")
 def api_search(req: SearchRequest):
     """
@@ -311,6 +318,147 @@ def api_search(req: SearchRequest):
     
     return {"results": results}
 
+@app.post("/batch_search_medqa")
+def batch_api_search(req: BatchSearchRequest):
+    """
+    NEW: Batch search for multiple learning objectives.
+    Processes all queries, tracks duplicates, and returns organized results.
+    
+    Returns:
+    - results_by_objective: List of results organized by each objective
+    - unique_questions: Deduplicated list of all questions
+    - statistics: Coverage and quality metrics
+    """
+    
+    start_time = time.time()
+    
+    # Track all questions and their objective mappings
+    all_questions = {}  # key: question_text, value: question data + objectives list
+    results_by_objective = []
+    
+    for obj_idx, query in enumerate(req.queries):
+        objective_id = obj_idx + 1
+        
+        # Search for this objective
+        r = search(query, req.num_results_per_query, req.source_filter)
+        
+        objective_results = []
+        similarities = []
+        
+        if r['documents'][0]:
+            for i in range(len(r['documents'][0])):
+                doc_text = r['documents'][0][i]
+                metadata = r['metadatas'][0][i]
+                similarity = round(1 - r['distances'][0][i], 3)
+                similarities.append(similarity)
+                
+                # Parse the document
+                parsed = parse_question_document(doc_text, metadata)
+                
+                # Create unique key for deduplication
+                question_key = parsed['question'][:200]  # Use first 200 chars as key
+                
+                # Build result object
+                result = {
+                    "question": parsed['question'],
+                    "choices": parsed['choices'],
+                    "correct_answer": parsed['correct_answer_letter'],
+                    "correct_answer_text": parsed['correct_answer_text'],
+                    "explanation": metadata.get('explanation', ''),
+                    "has_explanation": bool(metadata.get('explanation', '').strip()),
+                    "source": metadata.get('source', 'unknown'),
+                    "similarity": similarity
+                }
+                
+                # Track for global deduplication
+                if question_key in all_questions:
+                    # This question already exists - add this objective to its list
+                    all_questions[question_key]['matches_objectives'].append(objective_id)
+                    # Update similarity if higher
+                    if similarity > all_questions[question_key]['max_similarity']:
+                        all_questions[question_key]['max_similarity'] = similarity
+                else:
+                    # First time seeing this question
+                    all_questions[question_key] = {
+                        **result,
+                        'matches_objectives': [objective_id],
+                        'max_similarity': similarity,
+                        'first_seen_at': objective_id
+                    }
+                
+                objective_results.append(result)
+        
+        # Store results for this objective
+        results_by_objective.append({
+            "objective_id": objective_id,
+            "objective_text": query,
+            "num_results": len(objective_results),
+            "avg_similarity": round(sum(similarities) / len(similarities), 3) if similarities else 0,
+            "results": objective_results
+        })
+    
+    # Prepare unique questions list
+    unique_questions = list(all_questions.values())
+    
+    # Calculate statistics
+    execution_time = round(time.time() - start_time, 2)
+    total_retrieved = sum(obj['num_results'] for obj in results_by_objective)
+    
+    # Coverage analysis
+    coverage = {
+        "excellent": [obj for obj in results_by_objective if obj['num_results'] >= 5],
+        "moderate": [obj for obj in results_by_objective if 2 <= obj['num_results'] < 5],
+        "limited": [obj for obj in results_by_objective if obj['num_results'] == 1],
+        "none": [obj for obj in results_by_objective if obj['num_results'] == 0]
+    }
+    
+    # Multi-objective questions
+    multi_objective_questions = [q for q in unique_questions if len(q['matches_objectives']) > 1]
+    
+    # Source distribution
+    sources = {}
+    for q in unique_questions:
+        source = q['source']
+        sources[source] = sources.get(source, 0) + 1
+    
+    # Similarity distribution
+    all_similarities = [q['max_similarity'] for q in unique_questions]
+    high_sim = len([s for s in all_similarities if s > 0.8])
+    med_sim = len([s for s in all_similarities if 0.7 <= s <= 0.8])
+    low_sim = len([s for s in all_similarities if s < 0.7])
+    
+    statistics = {
+        "total_objectives": len(req.queries),
+        "total_retrieved": total_retrieved,
+        "unique_questions": len(unique_questions),
+        "deduplication_rate": round((total_retrieved - len(unique_questions)) / total_retrieved * 100, 1) if total_retrieved > 0 else 0,
+        "execution_time_seconds": execution_time,
+        "coverage": {
+            "excellent_coverage_count": len(coverage["excellent"]),
+            "moderate_coverage_count": len(coverage["moderate"]),
+            "limited_coverage_count": len(coverage["limited"]),
+            "no_coverage_count": len(coverage["none"]),
+            "no_coverage_objectives": [obj['objective_id'] for obj in coverage["none"]]
+        },
+        "cross_objective": {
+            "multi_objective_questions": len(multi_objective_questions),
+            "multi_objective_percentage": round(len(multi_objective_questions) / len(unique_questions) * 100, 1) if unique_questions else 0
+        },
+        "sources": sources,
+        "similarity_distribution": {
+            "high_similarity_count": high_sim,
+            "medium_similarity_count": med_sim,
+            "low_similarity_count": low_sim,
+            "average_similarity": round(sum(all_similarities) / len(all_similarities), 3) if all_similarities else 0
+        }
+    }
+    
+    return {
+        "results_by_objective": results_by_objective,
+        "unique_questions": unique_questions,
+        "statistics": statistics
+    }
+
 app = gr.mount_gradio_app(app, demo, path="/")
 
 if __name__ == "__main__":