updated

made change to give complete exemplars, removed AI answer explanaiton creation as done in GPT BOT post vector databse retreival
app.py: Remove OpenAI, just return structured exemplar data (I'll code this separately)
knowledge.json: Add explanation_generation section (above)

app.py

@@ -8,8 +8,6 @@ import gradio as gr
 from fastapi import FastAPI
 from pydantic import BaseModel
 import re
-import anthropic  # You'll need: pip install anthropic
-# OR if using OpenAI: import openai
 
 # Extract and load database
 DB_PATH = "./medqa_db"
@@ -28,17 +26,15 @@ print("🧠 Loading MedCPT model...")
 model = SentenceTransformer('ncbi/MedCPT-Query-Encoder')
 print("✅ Model ready")
 
-# Initialize AI client (choose one)
-# Option 1: Claude
-claude_client = anthropic.Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
-
-# Option 2: OpenAI (uncomment if using)
-# openai.api_key = os.environ.get("OPENAI_API_KEY")
-
 # ============================================================================
-# Deduplication function (same as before)
+# Deduplication function
 # ============================================================================
 def deduplicate_results(results, target_count):
+    """
+    Remove duplicate questions based on:
+    1. High text similarity (>0.92) - catches near-exact duplicates
+    2. Same answer + moderate similarity (>0.85) - catches conceptual duplicates
+    """
     if not results['documents'][0]:
         return results
     
@@ -78,7 +74,7 @@ def deduplicate_results(results, target_count):
     }
 
 # ============================================================================
-# Search function (same as before)
+# Search function with deduplication
 # ============================================================================
 def search(query, num_results=3, source_filter=None):
     emb = model.encode(query).tolist()
@@ -98,10 +94,10 @@ def search(query, num_results=3, source_filter=None):
     return deduplicate_results(results, num_results)
 
 # ============================================================================
-# NEW: Parser to extract question structure
+# Parser to extract question structure
 # ============================================================================
 def parse_question_document(doc_text, metadata):
-    """Extract question and choices from document text."""
+    """Extract question and choices from document text - NO TRUNCATION."""
     
     lines = doc_text.split('\n')
     question_lines = []
@@ -113,6 +109,7 @@ def parse_question_document(doc_text, metadata):
         if not line:
             continue
             
+        # Check if this is an option line (A., B., C., etc.)
         option_match = re.match(r'^([A-E])[\.\)]\s*(.+)$', line)
         
         if option_match:
@@ -123,160 +120,149 @@ def parse_question_document(doc_text, metadata):
         elif not options_started:
             question_lines.append(line)
     
+    # Reconstruct FULL question text - no truncation
     question_text = ' '.join(question_lines).strip()
     
     answer_idx = metadata.get('answer_idx', 'N/A')
+    answer_text = metadata.get('answer', 'N/A')
+    
+    # If answer_text is just the letter, map it to the actual option text
+    if answer_text in options:
+        answer_text = options[answer_text]
     
     return {
         'question': question_text,
         'choices': options,
-        'correct_answer': answer_idx
+        'correct_answer_letter': answer_idx,
+        'correct_answer_text': answer_text
     }
 
 # ============================================================================
-# NEW: AI generation functions
+# Enhanced Gradio UI
 # ============================================================================
-def generate_choice_explanations(question, choices, correct_answer):
-    """Generate explanations for why each choice is correct/wrong."""
+def ui_search(query, num_results=3, source_filter="all"):
+    if not query.strip():
+        return "💡 Enter a medical query to search"
     
-    choices_text = '\n'.join([f"{k}. {v}" for k, v in choices.items()])
+    try:
+        r = search(query, num_results, source_filter if source_filter != "all" else None)
+        
+        if not r['documents'][0]:
+            return "❌ No results found"
+        
+        out = f"🔍 Found {len(r['documents'][0])} unique results\n\n"
+        
+        for i in range(len(r['documents'][0])):
+            source = r['metadatas'][0][i].get('source', 'unknown')
+            distance = r['distances'][0][i]
+            similarity = 1 - distance
+            
+            # Source emoji
+            if source == 'medgemini':
+                source_icon = "🔬"
+                source_name = "Med-Gemini"
+            elif source.startswith('medqa_'):
+                source_icon = "📚"
+                split = source.replace('medqa_', '').upper()
+                source_name = f"MedQA {split}"
+            else:
+                source_icon = "📄"
+                source_name = source.upper()
+            
+            out += f"\n{'='*70}\n"
+            out += f"{source_icon} Result {i+1} | {source_name} | Similarity: {similarity:.3f}\n"
+            out += f"{'='*70}\n\n"
+            out += r['documents'][0][i]
+            
+            answer = r['metadatas'][0][i].get('answer', 'N/A')
+            out += f"\n\n✅ CORRECT ANSWER: {answer}\n"
+            
+            explanation = r['metadatas'][0][i].get('explanation', '')
+            if explanation and explanation.strip():
+                out += f"\n💡 EXPLANATION:\n{explanation}\n"
+            
+            out += "\n"
+        
+        return out
     
-    prompt = f"""You are a medical educator. For this USMLE-style question, explain why EACH answer choice is correct or incorrect.
-
-QUESTION:
-{question}
-
-ANSWER CHOICES:
-{choices_text}
-
-CORRECT ANSWER: {correct_answer}
+    except Exception as e:
+        return f"❌ Error: {e}"
 
-Provide a 1-2 sentence explanation for EACH choice (A through E) explaining why it is correct or incorrect. Format as:
-
-A. [Choice text] - [Explanation]
-B. [Choice text] - [Explanation]
-C. [Choice text] - [Explanation]
-D. [Choice text] - [Explanation]
-E. [Choice text] - [Explanation]"""
-
-    # Using Claude
-    message = claude_client.messages.create(
-        model="claude-sonnet-4-20250514",
-        max_tokens=1000,
-        messages=[{"role": "user", "content": prompt}]
-    )
+# Create Gradio interface
+with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
+    gr.Markdown("""
+    # 🏥 MedQA Semantic Search
     
-    return message.content[0].text
+    Search across **Med-Gemini** (expert explanations) and **MedQA** (USMLE questions) databases.
+    Uses medical-specific embeddings (MedCPT) for accurate retrieval.
     
-    # OR using OpenAI (uncomment if using):
-    # response = openai.ChatCompletion.create(
-    #     model="gpt-4",
-    #     messages=[{"role": "user", "content": prompt}],
-    #     max_tokens=1000
-    # )
-    # return response.choices[0].message.content
-
-def generate_similar_question(original_question, choices, correct_answer):
-    """Generate a new question based on the exemplar."""
+    ✨ **Features**: Automatic deduplication, structured output for AI integration
+    """)
     
-    choices_text = '\n'.join([f"{k}. {v}" for k, v in choices.items()])
+    with gr.Row():
+        with gr.Column(scale=3):
+            query_input = gr.Textbox(
+                label="Medical Query",
+                placeholder="e.g., hyponatremia, myocardial infarction, diabetes management...",
+                lines=2
+            )
+        with gr.Column(scale=1):
+            num_results = gr.Slider(
+                minimum=1,
+                maximum=10,
+                value=3,
+                step=1,
+                label="Number of Results"
+            )
     
-    prompt = f"""You are a medical educator. Based on this USMLE-style question, create a NEW similar question that tests the SAME medical concept but with a different clinical scenario.
-
-ORIGINAL QUESTION:
-{question}
-
-ANSWER CHOICES:
-{choices_text}
-
-CORRECT ANSWER: {correct_answer}
-
-Create a NEW question that:
-1. Tests the same medical concept
-2. Uses a different patient scenario
-3. Has 5 answer choices (A-E)
-4. Includes explanations for why each choice is correct/incorrect
-
-Format your response EXACTLY as:
-
-NEW QUESTION:
-[Your new question text]
-
-ANSWER CHOICES:
-A. [Choice A]
-B. [Choice B]
-C. [Choice C]
-D. [Choice D]
-E. [Choice E]
-
-CORRECT ANSWER: [Letter]
-
-EXPLANATIONS:
-A. [Choice A text] - [Explanation]
-B. [Choice B text] - [Explanation]
-C. [Choice C text] - [Explanation]
-D. [Choice D text] - [Explanation]
-E. [Choice E text] - [Explanation]"""
-
-    # Using Claude
-    message = claude_client.messages.create(
-        model="claude-sonnet-4-20250514",
-        max_tokens=2000,
-        messages=[{"role": "user", "content": prompt}]
-    )
+    with gr.Row():
+        source_filter = gr.Radio(
+            choices=["all", "medgemini", "medqa_train", "medqa_dev", "medqa_test"],
+            value="all",
+            label="Filter by Source"
+        )
     
-    return message.content[0].text
+    search_btn = gr.Button("🔍 Search", variant="primary", size="lg")
     
-    # OR using OpenAI:
-    # response = openai.ChatCompletion.create(
-    #     model="gpt-4",
-    #     messages=[{"role": "user", "content": prompt}],
-    #     max_tokens=2000
-    # )
-    # return response.choices[0].message.content
-
-# ============================================================================
-# NEW: Format complete output
-# ============================================================================
-def format_complete_output(exemplar_num, parsed, original_explanation, choice_explanations, new_question_text):
-    """Format everything into readable plain text."""
+    output = gr.Textbox(
+        label="Search Results",
+        lines=25,
+        max_lines=50
+    )
     
-    choices_text = '\n'.join([f"{k}. {v}" for k, v in parsed['choices'].items()])
+    search_btn.click(
+        fn=ui_search,
+        inputs=[query_input, num_results, source_filter],
+        outputs=output
+    )
     
-    output = f"""{'='*80}
-EXEMPLAR {exemplar_num}
-{'='*80}
-
-ORIGINAL QUESTION:
-{parsed['question']}
-
-ANSWER CHOICES:
-{choices_text}
-
-CORRECT ANSWER: {parsed['correct_answer']}
-
-EXPLANATION FOR EACH CHOICE:
-{choice_explanations}
-"""
+    query_input.submit(
+        fn=ui_search,
+        inputs=[query_input, num_results, source_filter],
+        outputs=output
+    )
     
-    if original_explanation:
-        output += f"\nORIGINAL EXPLANATION FROM DATABASE:\n{original_explanation}\n"
+    gr.Markdown("""
+    ### 📊 Database Info
     
-    output += f"""
-{'-'*80}
-AI-GENERATED SIMILAR QUESTION:
-{'-'*80}
-
-{new_question_text}
-
-{'='*80}
-
-"""
+    **Med-Gemini**: Expert-relabeled questions with detailed explanations  
+    **MedQA**: USMLE-style questions (Train/Dev/Test splits)
     
-    return output
+    **Total Questions**: ~10,000+ USMLE-style questions
+    """)
+    
+    gr.Examples(
+        examples=[
+            ["hyponatremia", 3, "all"],
+            ["myocardial infarction treatment", 2, "medgemini"],
+            ["diabetes complications", 3, "all"],
+            ["antibiotics for pneumonia", 2, "medqa_train"]
+        ],
+        inputs=[query_input, num_results, source_filter]
+    )
 
 # ============================================================================
-# MODIFIED: API endpoint with full generation
+# FastAPI with structured JSON output (for OpenAI integration)
 # ============================================================================
 app = FastAPI()
 
@@ -284,72 +270,47 @@ class SearchRequest(BaseModel):
     query: str
     num_results: int = 3
     source_filter: str = None
-    generate_ai: bool = True  # Option to skip AI generation for faster response
 
 @app.post("/search_medqa")
 def api_search(req: SearchRequest):
-    """Search and return complete formatted exemplars with AI-generated content."""
+    """
+    Search MedQA and return structured exemplars.
+    Returns COMPLETE question text with no truncation.
+    """
     
-    print(f"🔍 Searching for: {req.query}")
     r = search(req.query, req.num_results, req.source_filter)
     
     if not r['documents'][0]:
-        return {"output": "No results found."}
-    
-    complete_output = f"SEARCH QUERY: {req.query}\n"
-    complete_output += f"FOUND {len(r['documents'][0])} EXEMPLARS\n\n"
+        return {"results": []}
     
+    results = []
     for i in range(len(r['documents'][0])):
-        print(f"Processing exemplar {i+1}...")
-        
         doc_text = r['documents'][0][i]
         metadata = r['metadatas'][0][i]
         
-        # Parse the exemplar
+        # Parse the document into structured format
         parsed = parse_question_document(doc_text, metadata)
-        original_explanation = metadata.get('explanation', '')
         
-        if req.generate_ai:
-            # Generate AI content
-            print(f"  Generating choice explanations...")
-            choice_explanations = generate_choice_explanations(
-                parsed['question'],
-                parsed['choices'],
-                parsed['correct_answer']
-            )
-            
-            print(f"  Generating similar question...")
-            new_question = generate_similar_question(
-                parsed['question'],
-                parsed['choices'],
-                parsed['correct_answer']
-            )
-        else:
-            choice_explanations = "(AI generation skipped)"
-            new_question = "(AI generation skipped)"
-        
-        # Format complete output
-        formatted = format_complete_output(
-            i + 1,
-            parsed,
-            original_explanation,
-            choice_explanations,
-            new_question
-        )
+        # Build complete result object
+        result = {
+            "result_number": i + 1,
+            "question": parsed['question'],  # FULL question text
+            "choices": parsed['choices'],
+            "correct_answer": parsed['correct_answer_letter'],
+            "correct_answer_text": parsed['correct_answer_text'],
+            "explanation": metadata.get('explanation', ''),
+            "has_explanation": bool(metadata.get('explanation', '').strip()),
+            "source": metadata.get('source', 'unknown'),
+            "exam_type": metadata.get('exam_type', 'unknown'),
+            "split": metadata.get('split', 'unknown'),
+            "similarity": round(1 - r['distances'][0][i], 3),
+            "metamap_phrases": metadata.get('metamap_phrases', '')
+        }
         
-        complete_output += formatted
+        results.append(result)
     
-    return {
-        "output": complete_output,
-        "content_type": "text/plain"
-    }
+    return {"results": results}
 
-# Gradio UI (simplified - just shows we have it)
-with gr.Blocks(theme=gr.themes.Soft(), title="MedQA Search") as demo:
-    gr.Markdown("# 🏥 MedQA Search with AI Generation")
-    query_input = gr.Textbox(label="Query")
-    output = gr.Textbox(label="Results", lines=50)
-    
 app = gr.mount_gradio_app(app, demo, path="/")
 
 if __name__ == "__main__":