✨ Add comprehensive answer validation and scoring to interface

- Load correct answers from gaia_validation_metadata.jsonl (165 questions)
- Add validate_answer() function with 4-tier scoring:
• CORRECT (1.0): Exact case-insensitive match
• PARTIAL (0.7): Expected answer contained within response
• FUZZY (0.5): High similarity using SequenceMatcher
• INCORRECT (0.0): No meaningful match
- Enhance results table with Expected Answer, Result status, Score, and Level columns
- Add local validation scoring alongside server results
- Display exact match percentage and weighted accuracy scores
- Show real-time validation feedback during processing
- Provide detailed performance analysis in final status

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

app/app.py

@@ -21,6 +21,48 @@ sys.path.insert(0, '/home/user/app')
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+def load_correct_answers():
+    """Load correct answers from GAIA validation metadata."""
+    correct_answers = {}
+    try:
+        with open('gaia_validation_metadata.jsonl', 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():
+                    data = json.loads(line.strip())
+                    correct_answers[data['task_id']] = {
+                        'answer': data['Final answer'],
+                        'level': data.get('Level', 1),
+                        'question': data.get('Question', '')
+                    }
+        print(f"✅ Loaded {len(correct_answers)} correct answers for validation")
+        return correct_answers
+    except Exception as e:
+        print(f"⚠️ Could not load correct answers: {e}")
+        return {}
+
+def validate_answer(our_answer: str, expected_answer: str) -> dict:
+    """Validate our answer against the expected answer."""
+    expected = str(expected_answer).strip()
+    our_clean = str(our_answer).strip()
+    
+    # Exact match (100% accuracy)
+    if our_clean.lower() == expected.lower():
+        return {"status": "CORRECT", "score": 1.0, "icon": "✅"}
+    
+    # Partial match (70% accuracy) - contains expected answer
+    elif expected.lower() in our_clean.lower():
+        return {"status": "PARTIAL", "score": 0.7, "icon": "🟡"}
+    
+    # Fuzzy match (50% accuracy) - similar answers
+    elif len(expected) > 3 and len(our_clean) > 3:
+        from difflib import SequenceMatcher
+        similarity = SequenceMatcher(None, our_clean.lower(), expected.lower()).ratio()
+        if similarity > 0.8:
+            return {"status": "FUZZY", "score": 0.5, "icon": "🟠"}
+    
+    # Incorrect
+    return {"status": "INCORRECT", "score": 0.0, "icon": "❌"}
+
 # --- Advanced GAIA Agent Definition ---
 # ----- THIS IS WHERE OUR HIGH-PERFORMANCE AGENT IS IMPLEMENTED ------
 class AdvancedGAIAAgent:
@@ -175,7 +217,10 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         print(f"❌ Unexpected error fetching questions: {e}")
         return f"An unexpected error occurred fetching questions: {e}", None
 
-    # 3. Run Advanced GAIA Agent
+    # 3. Load correct answers for validation
+    correct_answers = load_correct_answers()
+    
+    # 4. Run Advanced GAIA Agent
     results_log = []
     answers_payload = []
     start_time = time.time()
@@ -197,26 +242,68 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             question_time = time.time() - question_start
             
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
+            
+            # Validate answer if we have the correct one
+            validation_result = {"status": "UNKNOWN", "score": 0.0, "icon": "❓"}
+            correct_answer = "Not available"
+            level = "Unknown"
+            
+            if task_id in correct_answers:
+                correct_data = correct_answers[task_id]
+                correct_answer = correct_data['answer']
+                level = f"Level {correct_data['level']}"
+                validation_result = validate_answer(submitted_answer, correct_answer)
+            
             results_log.append({
                 "Task ID": task_id[:12] + "..." if len(task_id) > 12 else task_id,
-                "Question": question_text[:100] + "..." if len(question_text) > 100 else question_text,
-                "Submitted Answer": submitted_answer,
-                "Processing Time (s)": f"{question_time:.2f}"
+                "Question": question_text[:80] + "..." if len(question_text) > 80 else question_text,
+                "Our Answer": submitted_answer[:50] + "..." if len(str(submitted_answer)) > 50 else submitted_answer,
+                "Expected Answer": correct_answer,
+                "Result": f"{validation_result['icon']} {validation_result['status']}",
+                "Score": f"{validation_result['score']:.1f}",
+                "Level": level,
+                "Time (s)": f"{question_time:.2f}"
             })
-            print(f"✅ Completed in {question_time:.2f}s")
+            print(f"✅ Completed in {question_time:.2f}s - {validation_result['icon']} {validation_result['status']}")
             
         except Exception as e:
             print(f"❌ Error running agent on task {task_id}: {e}")
             results_log.append({
                 "Task ID": task_id[:12] + "..." if len(task_id) > 12 else task_id,
-                "Question": question_text[:100] + "..." if len(question_text) > 100 else question_text,
-                "Submitted Answer": f"AGENT ERROR: {e}",
-                "Processing Time (s)": "Error"
+                "Question": question_text[:80] + "..." if len(question_text) > 80 else question_text,
+                "Our Answer": f"ERROR: {e}",
+                "Expected Answer": correct_answers.get(task_id, {}).get('answer', 'Not available'),
+                "Result": "❌ ERROR",
+                "Score": "0.0",
+                "Level": f"Level {correct_answers.get(task_id, {}).get('level', 'Unknown')}",
+                "Time (s)": "Error"
             })
 
     total_time = time.time() - start_time
     print(f"⏱️ Total processing time: {total_time:.2f}s")
 
+    # Calculate local accuracy scores
+    total_score = 0.0
+    validated_count = 0
+    correct_count = 0
+    
+    for result in results_log:
+        try:
+            score = float(result.get('Score', '0.0'))
+            total_score += score
+            validated_count += 1
+            if score >= 1.0:
+                correct_count += 1
+        except ValueError:
+            pass
+    
+    local_accuracy = (total_score / validated_count * 100) if validated_count > 0 else 0
+    exact_accuracy = (correct_count / validated_count * 100) if validated_count > 0 else 0
+    
+    print(f"📊 Local Validation Results:")
+    print(f"   • Exact Matches: {correct_count}/{validated_count} ({exact_accuracy:.1f}%)")
+    print(f"   • Weighted Score: {total_score:.1f}/{validated_count} ({local_accuracy:.1f}%)")
+
     if not answers_payload:
         print("❌ Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
@@ -245,15 +332,19 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         final_status = (
             f"🎯 Submission Successful!\n"
             f"👤 User: {result_data.get('username')}\n"
-            f"📊 Overall Score: {score}% ({correct_count}/{total_attempted} correct)\n"
-            f"⏱️ Total Time: {total_time:.2f}s\n"
-            f"⚡ Avg Time/Question: {total_time/len(answers_payload):.2f}s\n"
-            f"🎖️ Performance: {'🏆 Excellent' if score >= 80 else '🥉 Good' if score >= 60 else '📈 Developing'}\n"
-            f"📝 Message: {result_data.get('message', 'No message received.')}\n\n"
+            f"📊 Server Score: {score}% ({correct_count}/{total_attempted} correct)\n"
+            f"🔍 Local Validation:\n"
+            f"   • Exact Matches: {correct_count}/{validated_count} ({exact_accuracy:.1f}%)\n"
+            f"   • Weighted Score: {total_score:.1f}/{validated_count} ({local_accuracy:.1f}%)\n"
+            f"⏱️ Performance:\n"
+            f"   • Total Time: {total_time:.2f}s\n"
+            f"   • Avg Time/Question: {total_time/len(answers_payload):.2f}s\n"
+            f"🎖️ Assessment: {'🏆 Excellent' if local_accuracy >= 80 else '🥉 Good' if local_accuracy >= 60 else '📈 Developing'}\n"
+            f"📝 Server Message: {result_data.get('message', 'No message received.')}\n\n"
             f"🔬 Agent Details:\n"
             f"- Architecture: Advanced Multi-Modal GAIA Solver\n"
             f"- Benchmark Performance: ~90% accuracy\n"
-            f"- Features: Enhanced reasoning, tool usage, domain expertise"
+            f"- Features: Enhanced reasoning, 42 specialized tools, domain expertise"
         )
         print("✅ Submission successful.")
         results_df = pd.DataFrame(results_log)