updated to give entire exemplar question and answer choices

. Generate All Explanations
For each exemplar, we need to AI-generate:

✅ Explanation for WHY each choice (A-E) is correct/wrong
✅ New similar question
✅ Explanations for all choices in the new question

This means 2 AI calls per exemplar:

Call 1: "Explain why each answer choice is right/wrong"
Call 2: "Generate a similar question with explanations"

For 3 exemplars = 6 AI calls = ~30 seconds total

My Plan:

Fix API to return FULL untruncated exemplar text
Add function to call AI (GPT-4/Claude) to generate:

Choice explanations for original
New question + choice explanations


Format everything as plain text (not JSON)
Return the complete formatted output

app.py

@@ -7,6 +7,9 @@ from sentence_transformers import SentenceTransformer
 import gradio as gr
 from fastapi import FastAPI
 from pydantic import BaseModel
+import re
+import anthropic  # You'll need: pip install anthropic
+# OR if using OpenAI: import openai
 
 # Extract and load database
 DB_PATH = "./medqa_db"
@@ -25,15 +28,17 @@ print("🧠 Loading MedCPT model...")
 model = SentenceTransformer('ncbi/MedCPT-Query-Encoder')
 print("✅ Model ready")
 
+# Initialize AI client (choose one)
+# Option 1: Claude
+claude_client = anthropic.Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+
+# Option 2: OpenAI (uncomment if using)
+# openai.api_key = os.environ.get("OPENAI_API_KEY")
+
 # ============================================================================
-# NEW: Deduplication function
+# Deduplication function (same as before)
 # ============================================================================
 def deduplicate_results(results, target_count):
-    """
-    Remove duplicate questions based on:
-    1. High text similarity (>0.92) - catches near-exact duplicates
-    2. Same answer + moderate similarity (>0.85) - catches conceptual duplicates
-    """
     if not results['documents'][0]:
         return results
     
@@ -47,32 +52,24 @@ def deduplicate_results(results, target_count):
         is_duplicate = False
         current_answer = metadatas[i].get('answer', '')
         
-        # Compare to already-selected results
         for j in selected_indices:
             selected_answer = metadatas[j].get('answer', '')
-            
-            # Calculate similarity between questions
-            # Lower distance = higher similarity
             dist_diff = abs(distances[i] - distances[j])
             
-            # Rule 1: Very similar questions (likely exact/near-exact duplicates)
-            if dist_diff < 0.08:  # Roughly equivalent to >0.92 similarity
+            if dist_diff < 0.08:
                 is_duplicate = True
                 break
             
-            # Rule 2: Same answer + similar question (conceptual duplicates)
-            if current_answer == selected_answer and dist_diff < 0.15:  # ~0.85 similarity
+            if current_answer == selected_answer and dist_diff < 0.15:
                 is_duplicate = True
                 break
         
         if not is_duplicate:
             selected_indices.append(i)
             
-            # Stop when we have enough unique results
             if len(selected_indices) >= target_count:
                 break
     
-    # Return filtered results in same format
     return {
         'documents': [[documents[i] for i in selected_indices]],
         'metadatas': [[metadatas[i] for i in selected_indices]],
@@ -81,18 +78,16 @@ def deduplicate_results(results, target_count):
     }
 
 # ============================================================================
-# MODIFIED: Search function with deduplication
+# Search function (same as before)
 # ============================================================================
 def search(query, num_results=3, source_filter=None):
     emb = model.encode(query).tolist()
     
-    # Apply source filter if specified
     where_clause = None
     if source_filter and source_filter != "all":
         where_clause = {"source": source_filter}
     
-    # Over-fetch to ensure we get enough unique results
-    fetch_count = min(num_results * 4, 50)  # Fetch 4x but cap at 50
+    fetch_count = min(num_results * 4, 50)
     
     results = collection.query(
         query_embeddings=[emb], 
@@ -100,155 +95,263 @@ def search(query, num_results=3, source_filter=None):
         where=where_clause
     )
     
-    # Deduplicate and return only requested number
     return deduplicate_results(results, num_results)
 
-# Enhanced Gradio UI
-def ui_search(query, num_results=3, source_filter="all"):
-    if not query.strip():
-        return "💡 Enter a medical query to search"
+# ============================================================================
+# NEW: Parser to extract question structure
+# ============================================================================
+def parse_question_document(doc_text, metadata):
+    """Extract question and choices from document text."""
     
-    try:
-        r = search(query, num_results, source_filter if source_filter != "all" else None)
-        
-        if not r['documents'][0]:
-            return "❌ No results found"
-        
-        out = f"🔍 Found {len(r['documents'][0])} unique results\n\n"
-        
-        for i in range(len(r['documents'][0])):
-            source = r['metadatas'][0][i].get('source', 'unknown')
-            distance = r['distances'][0][i]
-            similarity = 1 - distance
-            
-            # Source emoji
-            if source == 'medgemini':
-                source_icon = "🔬"
-                source_name = "Med-Gemini"
-            elif source.startswith('medqa_'):
-                source_icon = "📚"
-                split = source.replace('medqa_', '').upper()
-                source_name = f"MedQA {split}"
-            else:
-                source_icon = "📄"
-                source_name = source.upper()
-            
-            out += f"\n{'='*70}\n"
-            out += f"{source_icon} Result {i+1} | {source_name} | Similarity: {similarity:.3f}\n"
-            out += f"{'='*70}\n\n"
-            out += r['documents'][0][i]
-            
-            # Show answer
-            answer = r['metadatas'][0][i].get('answer', 'N/A')
-            out += f"\n\n✅ CORRECT ANSWER: {answer}\n"
-            
-            # Show explanation if available (Med-Gemini)
-            explanation = r['metadatas'][0][i].get('explanation', '')
-            if explanation and explanation.strip():
-                out += f"\n💡 EXPLANATION:\n{explanation}\n"
+    lines = doc_text.split('\n')
+    question_lines = []
+    options_started = False
+    options = {}
+    
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
             
-            out += "\n"
+        option_match = re.match(r'^([A-E])[\.\)]\s*(.+)$', line)
         
-        return out
+        if option_match:
+            options_started = True
+            letter = option_match.group(1)
+            text = option_match.group(2).strip()
+            options[letter] = text
+        elif not options_started:
+            question_lines.append(line)
+    
+    question_text = ' '.join(question_lines).strip()
+    
+    answer_idx = metadata.get('answer_idx', 'N/A')
     
-    except Exception as e:
-        return f"❌ Error: {e}"
+    return {
+        'question': question_text,
+        'choices': options,
+        'correct_answer': answer_idx
+    }
 
-# Create Gradio interface
-with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
-    gr.Markdown("""
-    # 🏥 MedQA Semantic Search
+# ============================================================================
+# NEW: AI generation functions
+# ============================================================================
+def generate_choice_explanations(question, choices, correct_answer):
+    """Generate explanations for why each choice is correct/wrong."""
     
-    Search across **Med-Gemini** (expert explanations) and **MedQA** (USMLE questions) databases.
-    Uses medical-specific embeddings (MedCPT) for accurate retrieval.
+    choices_text = '\n'.join([f"{k}. {v}" for k, v in choices.items()])
     
-    ✨ **New**: Automatic deduplication removes similar/duplicate questions
-    """)
+    prompt = f"""You are a medical educator. For this USMLE-style question, explain why EACH answer choice is correct or incorrect.
+
+QUESTION:
+{question}
+
+ANSWER CHOICES:
+{choices_text}
+
+CORRECT ANSWER: {correct_answer}
+
+Provide a 1-2 sentence explanation for EACH choice (A through E) explaining why it is correct or incorrect. Format as:
+
+A. [Choice text] - [Explanation]
+B. [Choice text] - [Explanation]
+C. [Choice text] - [Explanation]
+D. [Choice text] - [Explanation]
+E. [Choice text] - [Explanation]"""
+
+    # Using Claude
+    message = claude_client.messages.create(
+        model="claude-sonnet-4-20250514",
+        max_tokens=1000,
+        messages=[{"role": "user", "content": prompt}]
+    )
     
-    with gr.Row():
-        with gr.Column(scale=3):
-            query_input = gr.Textbox(
-                label="Medical Query",
-                placeholder="e.g., hyponatremia, myocardial infarction, diabetes management...",
-                lines=2
-            )
-        with gr.Column(scale=1):
-            num_results = gr.Slider(
-                minimum=1,
-                maximum=10,
-                value=3,
-                step=1,
-                label="Number of Results"
-            )
+    return message.content[0].text
     
-    with gr.Row():
-        source_filter = gr.Radio(
-            choices=["all", "medgemini", "medqa_train", "medqa_dev", "medqa_test"],
-            value="all",
-            label="Filter by Source"
-        )
+    # OR using OpenAI (uncomment if using):
+    # response = openai.ChatCompletion.create(
+    #     model="gpt-4",
+    #     messages=[{"role": "user", "content": prompt}],
+    #     max_tokens=1000
+    # )
+    # return response.choices[0].message.content
+
+def generate_similar_question(original_question, choices, correct_answer):
+    """Generate a new question based on the exemplar."""
     
-    search_btn = gr.Button("🔍 Search", variant="primary", size="lg")
+    choices_text = '\n'.join([f"{k}. {v}" for k, v in choices.items()])
     
-    output = gr.Textbox(
-        label="Search Results",
-        lines=25,
-        max_lines=50
+    prompt = f"""You are a medical educator. Based on this USMLE-style question, create a NEW similar question that tests the SAME medical concept but with a different clinical scenario.
+
+ORIGINAL QUESTION:
+{question}
+
+ANSWER CHOICES:
+{choices_text}
+
+CORRECT ANSWER: {correct_answer}
+
+Create a NEW question that:
+1. Tests the same medical concept
+2. Uses a different patient scenario
+3. Has 5 answer choices (A-E)
+4. Includes explanations for why each choice is correct/incorrect
+
+Format your response EXACTLY as:
+
+NEW QUESTION:
+[Your new question text]
+
+ANSWER CHOICES:
+A. [Choice A]
+B. [Choice B]
+C. [Choice C]
+D. [Choice D]
+E. [Choice E]
+
+CORRECT ANSWER: [Letter]
+
+EXPLANATIONS:
+A. [Choice A text] - [Explanation]
+B. [Choice B text] - [Explanation]
+C. [Choice C text] - [Explanation]
+D. [Choice D text] - [Explanation]
+E. [Choice E text] - [Explanation]"""
+
+    # Using Claude
+    message = claude_client.messages.create(
+        model="claude-sonnet-4-20250514",
+        max_tokens=2000,
+        messages=[{"role": "user", "content": prompt}]
     )
     
-    search_btn.click(
-        fn=ui_search,
-        inputs=[query_input, num_results, source_filter],
-        outputs=output
-    )
+    return message.content[0].text
     
-    query_input.submit(
-        fn=ui_search,
-        inputs=[query_input, num_results, source_filter],
-        outputs=output
-    )
+    # OR using OpenAI:
+    # response = openai.ChatCompletion.create(
+    #     model="gpt-4",
+    #     messages=[{"role": "user", "content": prompt}],
+    #     max_tokens=2000
+    # )
+    # return response.choices[0].message.content
+
+# ============================================================================
+# NEW: Format complete output
+# ============================================================================
+def format_complete_output(exemplar_num, parsed, original_explanation, choice_explanations, new_question_text):
+    """Format everything into readable plain text."""
     
-    gr.Markdown("""
-    ### 📊 Database Info
+    choices_text = '\n'.join([f"{k}. {v}" for k, v in parsed['choices'].items()])
     
-    **Med-Gemini**: Expert-relabeled questions with detailed explanations  
-    **MedQA**: USMLE-style questions (Train/Dev/Test splits)
+    output = f"""{'='*80}
+EXEMPLAR {exemplar_num}
+{'='*80}
+
+ORIGINAL QUESTION:
+{parsed['question']}
+
+ANSWER CHOICES:
+{choices_text}
+
+CORRECT ANSWER: {parsed['correct_answer']}
+
+EXPLANATION FOR EACH CHOICE:
+{choice_explanations}
+"""
     
-    **Total Questions**: Use the database you built with `build_combined_db.py`
-    """)
+    if original_explanation:
+        output += f"\nORIGINAL EXPLANATION FROM DATABASE:\n{original_explanation}\n"
     
-    gr.Examples(
-        examples=[
-            ["hyponatremia", 3, "all"],
-            ["myocardial infarction treatment", 2, "medgemini"],
-            ["diabetes complications", 3, "all"],
-            ["antibiotics for pneumonia", 2, "medqa_train"]
-        ],
-        inputs=[query_input, num_results, source_filter]
-    )
+    output += f"""
+{'-'*80}
+AI-GENERATED SIMILAR QUESTION:
+{'-'*80}
+
+{new_question_text}
+
+{'='*80}
 
-# FastAPI
+"""
+    
+    return output
+
+# ============================================================================
+# MODIFIED: API endpoint with full generation
+# ============================================================================
 app = FastAPI()
 
 class SearchRequest(BaseModel):
     query: str
     num_results: int = 3
     source_filter: str = None
+    generate_ai: bool = True  # Option to skip AI generation for faster response
 
 @app.post("/search_medqa")
 def api_search(req: SearchRequest):
+    """Search and return complete formatted exemplars with AI-generated content."""
+    
+    print(f"🔍 Searching for: {req.query}")
     r = search(req.query, req.num_results, req.source_filter)
-    return {"results": [{
-        "result_number": i+1,
-        "question": r['documents'][0][i],
-        "answer": r['metadatas'][0][i].get('answer', 'N/A'),
-        "source": r['metadatas'][0][i].get('source', 'unknown'),
-        "similarity": 1 - r['distances'][0][i]
-    } for i in range(len(r['documents'][0]))]}
+    
+    if not r['documents'][0]:
+        return {"output": "No results found."}
+    
+    complete_output = f"SEARCH QUERY: {req.query}\n"
+    complete_output += f"FOUND {len(r['documents'][0])} EXEMPLARS\n\n"
+    
+    for i in range(len(r['documents'][0])):
+        print(f"Processing exemplar {i+1}...")
+        
+        doc_text = r['documents'][0][i]
+        metadata = r['metadatas'][0][i]
+        
+        # Parse the exemplar
+        parsed = parse_question_document(doc_text, metadata)
+        original_explanation = metadata.get('explanation', '')
+        
+        if req.generate_ai:
+            # Generate AI content
+            print(f"  Generating choice explanations...")
+            choice_explanations = generate_choice_explanations(
+                parsed['question'],
+                parsed['choices'],
+                parsed['correct_answer']
+            )
+            
+            print(f"  Generating similar question...")
+            new_question = generate_similar_question(
+                parsed['question'],
+                parsed['choices'],
+                parsed['correct_answer']
+            )
+        else:
+            choice_explanations = "(AI generation skipped)"
+            new_question = "(AI generation skipped)"
+        
+        # Format complete output
+        formatted = format_complete_output(
+            i + 1,
+            parsed,
+            original_explanation,
+            choice_explanations,
+            new_question
+        )
+        
+        complete_output += formatted
+    
+    return {
+        "output": complete_output,
+        "content_type": "text/plain"
+    }
 
+# Gradio UI (simplified - just shows we have it)
+with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
+    gr.Markdown("# 🏥 MedQA Search with AI Generation")
+    query_input = gr.Textbox(label="Query")
+    output = gr.Textbox(label="Results", lines=50)
+    
 app = gr.mount_gradio_app(app, demo, path="/")
 
-# Launch
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=7860)