swap all-mpnet-base-v2 for MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli

deploy/main/claim_verifier.py

@@ -11,7 +11,9 @@ import string
 from deploy.utils.general_utils import TRUSTED_DOMAINS, SUSPICIOUS_DOMAINS
 from deploy.utils.content_extractor import extract_content
 from deploy.utils.url_filter import _is_corrupted_pdf_content, _is_pdf_or_download_url
-from semantic_similarity import calculate_semantic_similarity
+from nli_checks import advanced_claim_verifier_native
+
+# from semantic_similarity import calculate_semantic_similarity
 
 warnings.filterwarnings("ignore")
 
@@ -211,7 +213,15 @@ class ClaimVerifier:
     def _semantic_similarity_with_sentences(self, claim: str, sentences: str) -> float:
         """Calculate entailment scores and return the best one."""
         try:
-            score = calculate_semantic_similarity(claim, sentences)
+            # score = calculate_semantic_similarity(claim, sentences)
+            result = advanced_claim_verifier_native(claim, sentences)
+            print(f"Support Score: {result['support_score']:.4f}")
+            print(f"Overall Assessment: {result['prediction']} ✅")
+            print("-" * 30)
+            print("Most Relevant Evidence Found:")
+            print(f"-> \"{result['relevant_chunk']}\"")
+            print("\n" + "=" * 50 + "\n")
+            score = result["support_score"]
         except Exception as e:
             logging.error(f"Error analyzing sentence: {e}")
         return score

nli_checks.py

@@ -0,0 +1,116 @@
+import os
+import torch
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+model_name = "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForSequenceClassification.from_pretrained(model_name).to(device)
+model.eval()
+
+
+def advanced_claim_verifier_native(claim: str, evidence: str) -> dict:
+    """
+    Verifies a claim against potentially long evidence using chunking and a powerful NLI model
+    with the native transformers library.
+
+    Args:
+        claim (str): The claim to be verified (hypothesis).
+        evidence (str): The evidence text, which can be long (premise).
+
+    Returns:
+        dict: A dictionary containing the final support score, the predicted label,
+              and the most relevant evidence chunk.
+    """
+    if not evidence or not claim:
+        return {
+            "support_score": 0.5, # Neutral default
+            "prediction": "Neutral",
+            "relevant_chunk": "N/A"
+        }
+
+    # CHUNKING STRATEGY
+    chunks = evidence.split('\n\n')
+    chunks = [chunk.strip() for chunk in chunks if chunk.strip()]
+    
+    if not chunks:
+        return {
+            "support_score": 0.5,
+            "prediction": "Neutral",
+            "relevant_chunk": "N/A"
+        }
+
+
+    with torch.no_grad():
+        inputs = tokenizer(chunks, [claim] * len(chunks), truncation=True, padding=True, return_tensors="pt").to(device)
+
+        outputs = model(**inputs)
+
+        # Convert logits to probabilities using softmax
+        probabilities = torch.softmax(outputs.logits, dim=-1)
+        
+        # Convert to a NumPy array for easier handling
+        scores = probabilities.cpu().numpy()
+    
+    entailment_scores = scores[:, 0]
+    
+    # Find the chunk with the highest entailment score
+    best_chunk_idx = np.argmax(entailment_scores)
+    
+    # The final support score is the highest probability of entailment found
+    final_support_score = entailment_scores[best_chunk_idx]
+    
+    # Determine the final label based on the highest probability for that best chunk
+    final_prediction_idx = np.argmax(scores[best_chunk_idx])
+    
+    label_map = ["Supported", "Neutral", "Contradicted"]
+    final_prediction_label = label_map[final_prediction_idx]
+    
+    most_relevant_chunk = chunks[best_chunk_idx]
+
+    return {
+        "support_score": float(final_support_score),
+        "prediction": final_prediction_label,
+        "relevant_chunk": most_relevant_chunk
+    }
+
+
+if __name__ == "__main__":
+    claim_to_verify = "The company's new 'QuantumLeap' chip is expected to double processing speeds."
+
+    long_evidence = """
+    A press release today from Innovate Corp announced the 'QuantumLeap' processor. CEO Jane Doe stated, "We are thrilled to unveil this technology. Our internal benchmarks show that the QuantumLeap chip doubles the processing speed of our previous generation, a major milestone for the industry."
+
+    The announcement was met with cautious optimism. Analyst John Smith from TechAdvisory noted, "While the claims are impressive, we've seen bold promises before. Real-world performance will be the true test." He also pointed out that the chip's power consumption remains a concern for mobile applications, which was not addressed in the release.
+
+    The new processor will first be available in Innovate Corp's high-end desktop line, slated for a Q4 release. Broader availability has not yet been announced.
+    """
+
+    print(f"\nVERIFYING CLAIM: '{claim_to_verify}'\n")
+    
+    result = advanced_claim_verifier_native(claim_to_verify, long_evidence)
+
+    print(f"Final Support Score: {result['support_score']:.4f}")
+    print(f"Overall Assessment: {result['prediction']} ✅")
+    print("-" * 30)
+    print("Most Relevant Evidence Found:")
+    print(f"-> \"{result['relevant_chunk']}\"")
+
+    print("\n" + "="*50 + "\n")
+    
+    # Example for contradiction
+    claim_contradicted = "Implementing a four-day workweek increases employee productivity"
+    evidence_contradicted = "A trial of the four-day workweek led to employees completing fewer tasks overall, with managers noting a decline in project delivery speed and more frequent missed deadlines"
+    
+    print(f"VERIFYING CLAIM: '{claim_contradicted}'\n")
+    result_contra = advanced_claim_verifier_native(claim_contradicted, evidence_contradicted)
+    
+    print(f"Final Support Score: {result_contra['support_score']:.4f}")
+    print(f"Overall Assessment: {result_contra['prediction']} ❌")
+    print("-" * 30)
+    print("Most Relevant Evidence Found:")
+    print(f"-> \"{result_contra['relevant_chunk']}\"")