show metrics in graido app

app.py

@@ -9,6 +9,7 @@ from transformers import (
 import os
 from threading import Thread
 import subprocess
+from app_modules.utils import calc_bleu_rouge_scores, detect_repetitions
 
 from dotenv import find_dotenv, load_dotenv
 
@@ -93,10 +94,11 @@ def chat(message, history, temperature, repetition_penalty, do_sample, max_token
         if item[1] is not None:
             chat.append({"role": "assistant", "content": item[1]})
 
+    index = -1
     if [message] in examples:
         index = examples.index([message])
         message = f"{qa_system_prompt}\n\n{questions[index]['context']}\n\nQuestion: {message}"
-        print(message)
+        print("RAG prompt:", message)
 
     chat.append({"role": "user", "content": message})
 
@@ -105,6 +107,10 @@ def chat(message, history, temperature, repetition_penalty, do_sample, max_token
     streamer = TextIteratorStreamer(
         tok, timeout=200.0, skip_prompt=True, skip_special_tokens=True
     )
+
+    if temperature == 0:
+        temperature = 0.01
+
     generate_kwargs = dict(
         model_inputs,
         streamer=streamer,
@@ -114,9 +120,6 @@ def chat(message, history, temperature, repetition_penalty, do_sample, max_token
         eos_token_id=terminators,
     )
 
-    if temperature == 0:
-        generate_kwargs["do_sample"] = False
-
     t = Thread(target=model.generate, kwargs=generate_kwargs)
     t.start()
 
@@ -125,6 +128,22 @@ def chat(message, history, temperature, repetition_penalty, do_sample, max_token
         partial_text += new_text
         yield partial_text
 
+    answer = partial_text
+    (newline_score, repetition_score, total_repetitions) = detect_repetitions(answer)
+    partial_text += "\n\nRepetition Metrics:\n"
+    partial_text += f"1. Newline Score: {newline_score:.3f}\n"
+    partial_text += f"1. Repetition Score: {repetition_score:.3f}\n"
+    partial_text += f"1. Total Repetitions: {total_repetitions:.3f}\n"
+
+    if index >= 0:  # RAG
+        scores = calc_bleu_rouge_scores(
+            [answer], [questions[index]["wellFormedAnswers"]], debug=True
+        )
+
+        partial_text += "\n\n Performance Metrics:\n"
+        partial_text += f'1. BLEU: {scores["bleu_scores"]["bleu"]:.3f}\n'
+        partial_text += f'1. RougeL: {scores["rouge_scores"]["rougeL"]:.3f}\n'
+
     yield partial_text
 
 

app_modules/utils.py

@@ -191,15 +191,28 @@ bleu = evaluate.load("bleu")
 rouge = evaluate.load("rouge")
 
 
-def calc_metrics(df):
-    predictions = [df["answer"][i] for i in range(len(df))]
-    references = [df["ground_truth"][i] for i in range(len(df))]
+def calc_bleu_rouge_scores(predictions, references, debug=False):
+    if debug:
+        print("predictions:", predictions)
+        print("references:", references)
 
     bleu_scores = bleu.compute(
         predictions=predictions, references=references, max_order=1
     )
     rouge_scores = rouge.compute(predictions=predictions, references=references)
-    return {"bleu_scores": bleu_scores, "rouge_scores": rouge_scores}
+    result = {"bleu_scores": bleu_scores, "rouge_scores": rouge_scores}
+
+    if debug:
+        print("result:", result)
+
+    return result
+
+
+def calc_metrics(df):
+    predictions = [df["answer"][i] for i in range(len(df))]
+    references = [df["ground_truth"][i] for i in range(len(df))]
+
+    return calc_bleu_rouge_scores(predictions, references)
 
 
 pattern_abnormal_newlines = re.compile(r"\n{5,}")