evaluator methods added

config.yaml

@@ -1,37 +1,46 @@
-# Pipeline Configuration for ArXiv RAG
-project_name: "arxiv_cyber_advisor"
+# ------------------------------------------------------------------
+# RAG TOURNAMENT CONFIGURATION
+# ------------------------------------------------------------------
 
-# Stage 1: Data Acquisition
-data_ingestion:
+project:
+  name: "arxiv-research-rag"
   category: "cs.AI"
-  limit: 5
-  save_local: true
-  raw_data_path: "data/raw_arxiv.csv"
+  doc_limit: 5
 
-# Stage 2: Processing & Embedding
-embedding:
-  model_name: "all-MiniLM-L6-v2"
-  device: "cpu"  # Change to "cuda" if testing on a GPU machine
-
-chunking:
-  technique: "recursive"
+processing:
+  # Embedding model used for both vector db and evaluator similarity
+  embedding_model: "all-MiniLM-L6-v2"
+  # Options: sentence, recursive, semantic, fixed
+  technique: "recursive" 
+  # Token limit for MiniLM is 256; keeping it at 250 for safety
   chunk_size: 500
   chunk_overlap: 50
 
-# Stage 3: Vector Database (Pinecone)
 vector_db:
-  index_name: "arxiv-index"
+  base_index_name: "arxiv-tournament"
   dimension: 384
   metric: "cosine"
+  batch_size: 100
 
-# Stage 4: Retrieval & Re-ranking
 retrieval:
-  top_k_hybrid: 10
-  rerank_model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
-  top_k_final: 3
+  # Options: hybrid, semantic, bm25
+  mode: "hybrid"
+  # Options: cross-encoder, rrf
+  rerank_strategy: "cross-encoder"
+  use_mmr: true
+  top_k: 10
+  final_k: 5
+
+generation:
+  temperature: 0.1
+  max_new_tokens: 512
+  # The model used to Judge the others
+  judge_model: "Llama-3-8B"
 
-# Stage 5: Generation (LLM)
-llm:
-  model_id: "meta-llama/Meta-Llama-3-8B-Instruct"
-  max_new_tokens: 500
-  temperature: 0.1
+# List of contestants in the tournament
+models:
+  - "Llama-3-8B"
+  - "Mistral-7B"
+  - "Qwen-2.5"
+  - "DeepSeek-V3"
+  - "TinyAya"

config_loader.py

@@ -0,0 +1,27 @@
+import yaml
+from pathlib import Path
+
+class RAGConfig:
+    def __init__(self, config_path="config.yaml"):
+        with open(config_path, 'r') as f:
+            self.data = yaml.safe_load(f)
+
+    @property
+    def project(self): return self.data['project']
+    
+    @property
+    def processing(self): return self.data['processing']
+    
+    @property
+    def db(self): return self.data['vector_db']
+    
+    @property
+    def retrieval(self): return self.data['retrieval']
+    
+    @property
+    def gen(self): return self.data['generation']
+    
+    @property
+    def model_list(self): return self.data['models']
+
+cfg = RAGConfig()

main.py

@@ -1,96 +1,103 @@
 import os
 from dotenv import load_dotenv
+from config_loader import cfg # Import the Mother Config
 
 from vector_db import get_pinecone_index, refresh_pinecone_index
 from retriever.retriever import HybridRetriever
 from retriever.generator import RAGGenerator
 from retriever.processor import ChunkProcessor
+from retriever.evaluator import RAGEvaluator
 import data_loader as dl
 
+# Import fleet mapping
 from models.llama_3_8b import Llama3_8B
 from models.mistral_7b import Mistral_7b
 from models.qwen_2_5 import Qwen2_5
 from models.deepseek_v3 import DeepSeek_V3
 from models.tiny_aya import TinyAya
 
+MODEL_MAP = {
+    "Llama-3-8B": Llama3_8B,
+    "Mistral-7B": Mistral_7b,
+    "Qwen-2.5": Qwen2_5,
+    "DeepSeek-V3": DeepSeek_V3,
+    "TinyAya": TinyAya
+}
+
 load_dotenv()
 
 def main():
+    hf_token = os.getenv("HF_TOKEN")
+    pinecone_key = os.getenv("PINECONE_API_KEY")
+    query = "How do transformers handle long sequences?"
+
+    # 1. Data Ingestion (Controlled by Config)
+    raw_data = dl.fetch_arxiv_data(
+        category=cfg.project['category'], 
+        limit=cfg.project['doc_limit']
+    )
 
-    # ------------------------------------------------------------------
-    # 0. Configuration
-    # Query defined here
-    # ------------------------------------------------------------------
-    hf_token        = os.getenv("HF_TOKEN")
-    pinecone_api_key = os.getenv("PINECONE_API_KEY")
-    if not pinecone_api_key:
-        raise ValueError("PINECONE_API_KEY not found in environment variables")
-
-    query      = "How do transformers handle long sequences?"
-
-    # ------------------------------------------------------------------
-    # 1. Data Ingestion
-    # ------------------------------------------------------------------
-    raw_data = dl.fetch_arxiv_data(category="cs.AI", limit=5)
-
-    # ------------------------------------------------------------------
     # 2. Chunking & Embedding
-    # ------------------------------------------------------------------
-    proc = ChunkProcessor(model_name='all-MiniLM-L6-v2', verbose=True)
+    proc = ChunkProcessor(model_name=cfg.processing['embedding_model'])
     final_chunks = proc.process(
         raw_data,
-        technique="sentence",   # options: fixed, recursive, character, sentence, semantic
-        chunk_size=500,
-        chunk_overlap=50
+        technique=cfg.processing['technique'],
+        chunk_size=cfg.processing['chunk_size'],
+        chunk_overlap=cfg.processing['chunk_overlap']
     )
 
-    # ------------------------------------------------------------------
-    # 3. Vector DB
-    # ------------------------------------------------------------------
-    index_name = "arxiv-index"
-    index = get_pinecone_index(pinecone_api_key, index_name, dimension=384, metric="cosine")
-    refresh_pinecone_index(index, final_chunks, batch_size=100)
+    # 3. Vector DB (Auto-names index based on technique)
+    index = get_pinecone_index(
+        pinecone_key, 
+        cfg.db['base_index_name'], 
+        technique=cfg.processing['technique'],
+        dimension=cfg.db['dimension']
+    )
+    refresh_pinecone_index(index, final_chunks, batch_size=cfg.db['batch_size'])
 
-    # ------------------------------------------------------------------
     # 4. Retrieval
-    # ------------------------------------------------------------------
-    retriever = HybridRetriever(final_chunks, proc.encoder, verbose=True)
+    retriever = HybridRetriever(final_chunks, proc.encoder)
     context_chunks = retriever.search(
-        query,
-        index,
-        mode="hybrid",   # options: bm25, semantic, hybrid
-        rerank_strategy="cross-encoder", # options: cross-encoder, rrf
-        use_mmr=True,
-        top_k=10,
-        final_k=5
+        query, index,
+        mode=cfg.retrieval['mode'],
+        rerank_strategy=cfg.retrieval['rerank_strategy'],
+        use_mmr=cfg.retrieval['use_mmr'],
+        top_k=cfg.retrieval['top_k'],
+        final_k=cfg.retrieval['final_k']
     )
 
-    if not context_chunks:
-        print("No context chunks retrieved. Check your index and query.")
-        return
-
-    # ------------------------------------------------------------------
-    # 5. Generation
-    # ------------------------------------------------------------------
+    # 5. Initialization of Contestants
     rag_engine = RAGGenerator()
-
+    models = {name: MODEL_MAP[name](token=hf_token) for name in cfg.model_list}
     
-
-    models = {
-        "Llama-3-8B":  Llama3_8B(token=hf_token),
-        "Mistral-7B":  Mistral_7b(token=hf_token),
-        "Qwen-2.5":    Qwen2_5(token=hf_token),
-        "DeepSeek-V3": DeepSeek_V3(token=hf_token),
-        "TinyAya":     TinyAya(token=hf_token)
-    }
-
-    for name, model in models.items():
-        print(f"\n--- {name} ---")
+    # Setup Evaluator with the designated Judge
+    judge_llm = models[cfg.gen['judge_model']]
+    evaluator = RAGEvaluator(judge_llm, proc.encoder)
+    tournament_results = {}
+
+    # 6. Tournament Loop
+    for name, model_inst in models.items():
+        print(f"\n--- Processing {name} ---")
         try:
-            print(rag_engine.get_answer(model, query, context_chunks, temperature=0.1))
+            # Generation
+            answer = rag_engine.get_answer(
+                model_inst, query, context_chunks, 
+                temperature=cfg.gen['temperature']
+            )
+            
+            # Batch Evaluation
+            faith = evaluator.evaluate_faithfulness(answer, context_chunks)
+            rel = evaluator.evaluate_relevancy(query, answer)
+
+            tournament_results[name] = {
+                "Faithfulness": faith['score'],
+                "Relevancy": rel['score'],
+                "Claims": faith['details']
+            }
         except Exception as e:
-            print(f"Error: {e}")
+            print(f"Error evaluating {name}: {e}")
 
+    # 7. Final Output (Omitted for brevity, use your existing report logic)
 
 if __name__ == "__main__":
     main()

query_only.py

@@ -1,6 +1,6 @@
 # This file is for inference without actually embedding documents
 # Main does embedding everytime, is redundant for querying.
-# made this just to test querying part --@Qamar
+# made this just to test querying part
 
 import os
 import time

retriever/evaluator.py

@@ -0,0 +1,105 @@
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+class RAGEvaluator:
+    def __init__(self, judge_model, embedding_model, verbose=True):
+        """
+        judge_model: An instance of an LLM class.
+        embedding_model: The proc.encoder for similarity checks.
+        verbose: If True, uses internal printer functions to show progress.
+        """
+        self.judge = judge_model
+        self.encoder = embedding_model
+        self.verbose = verbose
+
+    # ------------------------------------------------------------------
+    # 1. FAITHFULNESS: Claim Extraction & Verification
+    # ------------------------------------------------------------------
+    def evaluate_faithfulness(self, answer, context_list):
+        if self.verbose:
+            self._print_extraction_header(len(answer))
+
+        # --- Step A: Extraction ---
+        extraction_prompt = f"Extract a list of independent factual claims from the following answer. Respond ONLY with the claims, one per line. Do not include any introductory text.\nAnswer: {answer}"
+        raw_claims = self.judge.generate(extraction_prompt)
+        claims = [c.strip() for c in raw_claims.split('\n') if len(c.strip()) > 5]
+
+        if not claims: 
+            return {"score": 0, "details": []}
+
+        # --- Step B: Batch Verification ---
+        combined_context = "\n".join(context_list)
+        claims_formatted = "\n".join([f"{i+1}. {c}" for i, c in enumerate(claims)])
+        
+        batch_prompt = f"Context: {combined_context}\nClaims: {claims_formatted}\nRespond YES/NO for each."
+        raw_verdicts = self.judge.generate(batch_prompt)
+        verdict_lines = [v.strip().upper() for v in raw_verdicts.split('\n') if v.strip()]
+
+        # --- Step C: Scoring & Details ---
+        verified_count = 0
+        details = []
+        for i, claim in enumerate(claims):
+            is_supported = "YES" in verdict_lines[i] if i < len(verdict_lines) else False
+            if is_supported: verified_count += 1
+            
+            details.append({
+                "claim": claim, 
+                "verdict": "Supported" if is_supported else "Not Supported"
+            })
+
+        score = (verified_count / len(claims)) * 100
+        
+        if self.verbose:
+            self._print_faithfulness_results(claims, details, score)
+            
+        return {"score": score, "details": details}
+
+    # ------------------------------------------------------------------
+    # 2. RELEVANCY: Alternate Query Generation
+    # ------------------------------------------------------------------
+    def evaluate_relevancy(self, query, answer):
+        if self.verbose:
+            self._print_relevancy_header()
+
+        # --- Step A: Generation ---
+        gen_prompt = f"Generate 3 distinct questions this answer addresses.\nAnswer: {answer}"
+        raw_gen = self.judge.generate(gen_prompt)
+        gen_queries = [q.strip() for q in raw_gen.split('\n') if '?' in q][:3]
+
+        if not gen_queries: 
+            return {"score": 0, "queries": []}
+
+        # --- Step B: Similarity Logic ---
+        original_vec = self.encoder.encode([query])
+        generated_vecs = self.encoder.encode(gen_queries)
+        similarities = cosine_similarity(original_vec, generated_vecs)[0]
+        avg_score = np.mean(similarities)
+        
+        if self.verbose: 
+            self._print_relevancy_results(query, gen_queries, similarities, avg_score)
+            
+        return {"score": avg_score, "queries": gen_queries}
+
+    # ------------------------------------------------------------------
+    # 3. PRINT HELPERS (Keep the logic above clean)
+    # ------------------------------------------------------------------
+    def _print_extraction_header(self, length):
+        print(f"\n[EVAL] Analyzing Faithfulness...")
+        print(f"      - Extracting claims from answer ({length} chars)")
+
+    def _print_faithfulness_results(self, claims, details, score):
+        print(f"      - Verifying {len(claims)} claims against context...")
+        for i, detail in enumerate(details):
+            status = "✅" if "Supported" in detail['verdict'] else "❌"
+            print(f"        {status} Claim {i+1}: {detail['claim'][:75]}...")
+        print(f"      🎯 Faithfulness Score: {score:.1f}%")
+
+    def _print_relevancy_header(self):
+        print(f"\n[EVAL] Analyzing Relevancy...")
+        print(f"      - Generating 3 sample questions addressed by the answer")
+
+    def _print_relevancy_results(self, query, gen_queries, similarities, avg):
+        print(f"      - Comparing to original query: '{query}'")
+        for i, (q, sim) in enumerate(zip(gen_queries, similarities)):
+            print(f"        Q{i+1}: {q} (Sim: {sim:.2f})")
+        print(f"      🎯 Average Relevancy: {avg:.2f}")

vector_db.py

@@ -1,102 +1,94 @@
 import time
+import re
 from pinecone import Pinecone, ServerlessSpec
 
-def get_pinecone_index(api_key, index_name, dimension=384, metric="cosine"):
-    """Initializes Pinecone and returns the index object, creating it if necessary."""
+def slugify_technique(name):
+    """Converts 'Sentence Splitter' to 'sentence-splitter' for Pinecone naming."""
+    return re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-')
+
+def get_pinecone_index(api_key, base_name, technique, dimension=384, metric="cosine"):
+    """
+    Creates/Returns an index specifically for a technique.
+    Example: 'arxiv-index-token'
+    """
     pc = Pinecone(api_key=api_key)
+    tech_slug = slugify_technique(technique)
+    full_index_name = f"{base_name}-{tech_slug}"
     
-    # Check if index exists
     existing_indexes = [idx.name for idx in pc.list_indexes()]
     
-    if index_name not in existing_indexes:
-        print(f"Creating new Pinecone index: {index_name}...")
+    if full_index_name not in existing_indexes:
+        print(f" Creating specialized index: {full_index_name}...")
         pc.create_index(
-            name=index_name,
+            name=full_index_name,
             dimension=dimension,
             metric=metric,
             spec=ServerlessSpec(cloud="aws", region="us-east-1")
         )
-        # Wait for index to be ready
-        while not pc.describe_index(index_name).status['ready']:
+        # Wait for index to spin up
+        while not pc.describe_index(full_index_name).status['ready']:
             time.sleep(1)
     
-    return pc.Index(index_name)
+    print(f" Using Index: {full_index_name}")
+    return pc.Index(full_index_name)
+
+def refresh_pinecone_index(index, final_chunks, batch_size=100):
+    """
+    Refreshes the specific index. Since index is now technique-specific,
+    we just check if it's already populated.
+    """
+    if not final_chunks:
+        print("No chunks provided to refresh.")
+        return False
+
+    try:
+        # Check current stats for this specific index
+        stats = index.describe_index_stats()
+        current_count = stats.get('total_vector_count', 0)
+        expected_count = len(final_chunks)
+
+        print(f" Index Stats -> Existing: {current_count} | New Chunks: {expected_count}")
 
+        if current_count == 0:
+            print(f"➕ Index is empty. Upserting {expected_count} vectors...")
+            vectors = prepare_vectors_for_upsert(final_chunks)
+            upsert_to_pinecone(index, vectors, batch_size)
+            return True
+        
+        elif current_count < expected_count:
+            # Simple check to see if we need to top up or refresh
+            print(f" Vector count mismatch ({current_count} < {expected_count}). Updating index...")
+            vectors = prepare_vectors_for_upsert(final_chunks)
+            upsert_to_pinecone(index, vectors, batch_size)
+            return True
+
+        else:
+            print(f" Index is already populated with {current_count} vectors. Ready for search.")
+            return False
+            
+    except Exception as e:
+        print(f" Error refreshing index: {e}")
+        return False
+
+# Utility functions remain the same as previous version
 def prepare_vectors_for_upsert(final_chunks):
-    """Convert final_chunks to the format expected by Pinecone upsert"""
     vectors = []
     for chunk in final_chunks:
+        meta = chunk.get('metadata', {})
         vectors.append({
             'id': chunk['id'],
-            'values': chunk['values'],  # The embedding vector
+            'values': chunk['values'],
             'metadata': {
-                'text': chunk['metadata']['text'],
-                'title': chunk['metadata']['title'],
-                'url': chunk['metadata']['url'],
-                'chunk_index': chunk['metadata']['chunk_index'],
-                'technique': chunk['metadata']['technique'],
-                'chunk_size': chunk['metadata']['chunk_size'],
-                'total_chunks': chunk['metadata']['total_chunks']
+                'text': meta.get('text', ""),
+                'title': meta.get('title', ""),
+                'url': meta.get('url', ""),
+                'chunk_index': meta.get('chunk_index', 0),
+                'technique': meta.get('technique', "unknown")
             }
         })
     return vectors
 
 def upsert_to_pinecone(index, chunks, batch_size=100):
-    """Upserts chunks to Pinecone in manageable batches.
-    
-    Args:
-        index: Pinecone index object
-        chunks: List of chunk dictionaries (as returned by prepare_vectors_for_upsert)
-        batch_size: Number of vectors to upsert in each batch
-    """
-    print(f"Uploading {len(chunks)} chunks to Pinecone...")
-    
     for i in range(0, len(chunks), batch_size):
         batch = chunks[i : i + batch_size]
-        index.upsert(vectors=batch)
-        print(f"   Uploaded batch {i//batch_size + 1}/{(len(chunks)-1)//batch_size + 1} ({len(batch)} vectors)")
-        
-    print(" Upsert complete.")
-
-def refresh_pinecone_index(index, final_chunks, batch_size=100):
-    """Helper function to refresh index with new chunks.
-    
-    This function checks if the index has the expected number of vectors,
-    and upserts if necessary.
-    
-    Args:
-        index: Pinecone index object
-        final_chunks: List of chunk dictionaries with embeddings
-        batch_size: Batch size for upsert
-    
-    Returns:
-        Boolean indicating if upsert was performed
-    """
-    try:
-        stats = index.describe_index_stats()
-        current_vector_count = stats.get('total_vector_count', 0)
-        expected_vector_count = len(final_chunks)
-        
-        print(f"\n Current vectors in index: {current_vector_count}")
-        print(f" Expected vectors: {expected_vector_count}")
-        
-        if current_vector_count == 0:
-            print(" Index is empty. Preparing vectors for upsert...")
-            vectors_to_upsert = prepare_vectors_for_upsert(final_chunks)
-            upsert_to_pinecone(index, vectors_to_upsert, batch_size)
-            
-            # Verify upsert
-            stats = index.describe_index_stats()
-            print(f" After upsert - Total vectors: {stats.get('total_vector_count', 0)}")
-            return True
-        elif current_vector_count != expected_vector_count:
-            print(f"  Vector count mismatch. Expected {expected_vector_count}, got {current_vector_count}")
-            print("   Consider recreating the index if you want to refresh.")
-            return False
-        else:
-            print(f"ℹ Index already has {current_vector_count} vectors. Ready for search.")
-            return False
-            
-    except Exception as e:
-        print(f"Error checking index stats: {e}")
-        return False
+        index.upsert(vectors=batch)