some updates for scores.

Former-commit-id: 5d6b5d325b183e15512755c8f6accc4909b7f8a1

evaluation/scores/LLM_eval.py

@@ -13,35 +13,46 @@ from langchain.chat_models import ChatOpenAI
 # Load the evaluator
 
 def init_evaluator(source_lang="en", target_lang="zh", domain="startcraft2", model="gpt-4-0613"):
-    llm = ChatOpenAI(temperature=0, model=model)
 
-    lang_str = f"You are an expert {source_lang} to {target_lang} translator specialized in {domain}."
+    # map the language code to the language name
+    language_map = {
+        "en": "English",
+        "zh": "Chinese",
+    }
+
+    llm = ChatOpenAI(temperature=0, model=model)
 
+    # Completeness is the percentage of the input that is translated
+    # Accuracy is the percentage of the translation that is correct
     fstring = """
-            You are grading the following question:
+            You are grading the translation based on following input:
             {input}
-            Here is the real answer:
+            if the input is "", that means there is no input sentence.
+            you should grade the translation based on the reference translation:
+            Here is the real answer(reference):
             {reference}
-            You are grading the following predicted answer:
+            You are grading the following translation:
             {output}
             based on the following criteria: 
             {criteria}
-            Give one grades, accuracy and rate them from a scale of 0 to 100, where 0 is the lowest (very low accuracy) and 100 is the highest (very high accuracy)? 
+            Give two grades, accuracy and completeness rate them from a scale of 0 to 100, where 0 is the lowest (very low accuracy/completeness) and 100 is the highest (very high accuracy/completeness)? 
             Give explanations for every single one and if the answer if partially correct that is acceptable. However punish the scores for answers that are 
             numerically incorrect this also includes values that have the $ in front
             Please give the completeness score first followed by the accuracy score. 
-            For example: Accuracy: 40. Explanation here
+            For example: 
+            Accuracy: 40. Explanation here
+            Completeness: 80. Explanation here
             Do not differ from the format ever
             """
-    prompt = PromptTemplate.from_template(lang_str+fstring, template_format="f-string")
-
+    
+    if source_lang in language_map and target_lang in language_map:
+        lang_str = f"You are an expert {language_map[source_lang]} to {language_map[target_lang]} translator specialized in {domain}."
+        prompt = PromptTemplate.from_template(lang_str+fstring, template_format="f-string")
+    
+    else:
+        print("The language code is not supported, please check the language code.")
+        prompt = PromptTemplate.from_template(fstring, template_format="f-string")
 
-# Give two grades, one for completness and another for accuracy and rate them from a scale of 0 to 100, where 0 is the lowest (very low completeness/accuracy) and 100 is the highest (very high completness/accuracy)? 
-#             Do not base the two scores off each other give them the scores independently. Give explanations for every single one and if the answer if partially correct that is acceptable. However punish the scores for answers that are 
-#             numerically incorrect this also includes values that have the $ in front
-#             Please give the completeness score first followed by the accuracy score. 
-#             For example: Completeness: 70. Accuracy: 40. Explanation here
-#             Do not differ from the format ever
     return load_evaluator("labeled_criteria", llm=llm, prompt=prompt, criteria="correctness")
 
 # prase the output of the evaluation
@@ -55,18 +66,41 @@ def init_evaluator(source_lang="en", target_lang="zh", domain="startcraft2", mod
 #     explanation = ".".join(value[1:])
 #     return int(value[0]), explanation
 
-def parse_eval_result(eval_result):
-    # Extract the 'Accuracy' score using a regular expression from the 'reasoning' key
-    accuracy_match = re.search(r'Accuracy: (\d+)', eval_result['reasoning'])
-    if accuracy_match:
-        accuracy = int(accuracy_match.group(1))
-    else:
-        accuracy = 0
+# def parse_eval_result(eval_result):
+#     # Extract the 'Accuracy' score using a regular expression from the 'reasoning' key
+#     accuracy_match = re.search(r'Accuracy: (\d+)', eval_result['value'])
+#     print(accuracy_match)
+#     if accuracy_match:
+#         accuracy = int(accuracy_match.group(1))
+#     else:
+#         # try to get the accuracy from the 'value' key
+#         accuracy = 0
+
+#     # Directly get the 'Explanation' value from the 'value' key
+#     explanation = eval_result['value']
+
+#     return accuracy, explanation
+
+def parse_eval_result(data):
+    # Extract the value string
+    value_str = data.get('value', '')
+    reasoning_str = data.get('reasoning', '')
+
+    # Use regex to extract accuracy value and explanation
+    accuracy_match = re.search(r'Accuracy: (\d+)', value_str)
+    acc_explanation_match = re.search(r'Accuracy: \d+\. (.+)', value_str)
+
+    # Use regex to extract completeness value and explanation
+    completeness_match = re.search(r'Completeness: (\d+)', reasoning_str)
+    completeness_explanation_match = re.search(r'Completeness: \d+\. (.+)', reasoning_str)
 
-    # Directly get the 'Explanation' value from the 'value' key
-    explanation = eval_result['value']
+    # Extract the matched groups
+    completeness = int(completeness_match.group(1)) if completeness_match else None
+    completeness_explanation = completeness_explanation_match.group(1) if completeness_explanation_match else None
+    accuracy = int(accuracy_match.group(1)) if accuracy_match else None
+    acc_explanation = acc_explanation_match.group(1) if acc_explanation_match else None
 
-    return accuracy, explanation
+    return (accuracy, acc_explanation), (completeness, completeness_explanation)
 
 def evaluate_prediction(input, reference, prediction, evaluator):
     eval_result = evaluator.evaluate_strings(
@@ -74,11 +108,14 @@ def evaluate_prediction(input, reference, prediction, evaluator):
         input=input,
         reference=reference,
     )
+    # print(eval_result)
     return parse_eval_result(eval_result)
 
 if __name__ == "__main__":
     evaluator = init_evaluator()
     # For no input english sentence, just put "" in the input
-    accuracy, explanation = evaluate_prediction("this is an test sentences", "这不是一个测试语句。", "这是一个测试句子。", evaluator)
-    print("Accuracy:", accuracy)
-    print("Explanation:", explanation)
+    accuracy, completeness = evaluate_prediction("this is an test sentences", "这不是一个测试语句。", "这是一个测试句子。", evaluator)
+    print("Accuracy:", accuracy[0])
+    print("Acc_Explanation:", accuracy[1])
+    print("Completeness:", completeness[0])
+    print("Comp_Explanation:", completeness[1])

evaluation/scores/multi_scores.py

@@ -1,26 +1,46 @@
 from comet import download_model, load_from_checkpoint
 from sacrebleu.metrics import BLEU, CHRF, TER
-from scores import LLM_eval
+# from scores import LLM_eval
+import LLM_eval
 
 class multi_scores:
-    def __init__(self, source_lang="English", target_lang="Chinese", domain="starcraft 2") -> None:
+    def __init__(self, source_lang="en", target_lang="zh", domain="starcraft 2") -> None:
         self.comet_model = load_from_checkpoint(download_model("Unbabel/wmt22-comet-da"))
-        self.bleu_model = BLEU(tokenize="zh")
+        self.bleu_model = BLEU(tokenize=target_lang)
         self.LLM_model = LLM_eval.init_evaluator(source_lang=source_lang, target_lang=target_lang, domain=domain)
+        # self.score = {}
 
     # The function to get the scores
     # src: orginal sentence
     # mt: machine translation
     # ref: reference translation
+    def calculate_comet_llm(self, src:str, mt:str, ref:str) -> dict:
+        comet_score = self.comet_model.predict([{"src":src, "mt":mt, "ref":ref}], batch_size=8, gpus=0).scores[0]
+        # bleu_score = self.bleu_model.corpus_score([mt], [ref]).score
+        llm_acc, llm_completeness = LLM_eval.evaluate_prediction(src, ref, mt, self.LLM_model)
+        return {'comet_score':comet_score, 'llm_score':llm_acc[0], 'llm_explanation': llm_acc[1]}
+        # self.score['bleu_score'] = bleu_score
+        # self.score['comet_score'] = comet_score
+        # self.score['llm_score'] = llm_score
+        # self.score['llm_explanation'] = llm_explanation
+    
+    def calculate_bleu(self, mts:list, refs:list) -> dict:
+        # mt and ref are list of sentences
+        bleu_score = self.bleu_model.corpus_score(mts, refs).score
+        return {'bleu_score':bleu_score}
+    
     def get_scores(self, src:str, mt:str, ref:str) -> dict:
         comet_score = self.comet_model.predict([{"src":src, "mt":mt, "ref":ref}], batch_size=8, gpus=0).scores[0]
         bleu_score = self.bleu_model.corpus_score([mt], [ref]).score
-        llm_score, llm_explanation = LLM_eval.evaluate_prediction(src, ref, mt, self.LLM_model)
-        return {'bleu_score':bleu_score, 'comet_score':comet_score, 'llm_score':llm_score, 'llm_explanation': llm_explanation}
+        llm_acc, llm_completeness = LLM_eval.evaluate_prediction(src, ref, mt, self.LLM_model)
+        return {'bleu_score':bleu_score ,'comet_score':comet_score, 'llm_score':llm_acc[0], 'llm_explanation': llm_acc[1]}
+
     
 if __name__ == "__main__":
     src = "this is an test sentences"
     mt = "这是一个测试句子。"
     ref = "这不是一个测试语句。"
     print(multi_scores().get_scores(src, mt, ref))
+    # print(multi_scores().calculate_comet_llm(src, mt, ref))
+    # print(multi_scores().calculate_bleu([mt], [ref]))