Evaluation structure

Former-commit-id: 956e1085199af0c47b3fc01f395f1c3be195ece4

evaluation/readme.md

@@ -0,0 +1,24 @@
+Evaluation:  
+BLEU (https://github.com/mjpost/sacrebleu)  
+COMET (https://github.com/Unbabel/COMET)  
+LLM eval  
+Eval time stamp  
+
+Sep 18 - Sep 25
+Proj-t
+src
+evaluation
+- scores
+- LLM_eval.py (jiaen)
+- scores.py (wizard)
+- comet
+- sacrebleu
+	- alignment.py (david)
+	- evaluation.py (not assigned)
+- results
+- mmddyy-HMS-results.csv
+- logs
+
+entry: 
+Python3 evaluation/evaluation.py –pred path/to/pred –gt path/to/gt
+

evaluation/scores/LLM_eval.py

@@ -0,0 +1,76 @@
+# This script is used to evaluate the performance of Pigeon AI Video Translation system by using Large Language Model.
+
+# Written by Jiaen LIU, 2023/09/18
+
+# Import the necessary packages
+from langchain.evaluation import load_evaluator, EvaluatorType
+from langchain.prompts import PromptTemplate
+from langchain.chat_models import ChatAnthropic
+
+import re
+from tqdm import tqdm
+import pandas as pd
+import numpy as np
+import time
+import os
+import argparse
+from pathlib import Path 
+from src.srt_util.srt import SrtScript
+
+class PiegonLLMEvaluator():
+    """
+    input : 
+        - predicted sentences
+        - ground truth sentences
+    output :
+        - scores
+        - explanations
+    """
+
+    def __init__(self, output_dir: str, data_dir: str ) -> None:
+        # self.__eval_chain = self.__initialize_QAEvalChain()
+        self.__data_dir = data_dir
+        self.__result_dir = output_dir
+        # self.__chatbot = evaluater
+        self.__result_df = pd.DataFrame()
+        self.__initialize_df()
+        self.running_cost = 0
+        self.last_cost = 0
+        pass
+
+    def __init_llm_evaluater(self):
+        # llm = ChatOpenAI(temperature=0, model="gpt-4-0613")
+        # search = SerpAPIWrapper()
+        # tools = [
+        #     Tool(
+        #         name="Search",
+        #         func=search.run,
+        #         coroutine=search.arun,
+        #         description="Useful when you need to answer questions about current events. You should ask targeted questions.",
+        #     ),
+        # ]
+        # agents = [
+        #     initialize_agent(tools, llm, agent=AgentType.OPENAI_MULTI_FUNCTIONS, verbose=False),
+        #     initialize_agent(tools, llm, agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION, verbose=False)
+        # ]
+
+        llm = ChatAnthropic(temperature=0)
+
+        fstring = """You are an expert English to Chinese translator specialized in Startcraft2.
+        You are grading the following question:
+        {query}
+        Here is the real answer:
+        {answer}
+        You are grading the following predicted answer:
+        {result}
+        Give two grades, one for completness and another for accuracy and rate them from a scale of 0 to 100, where 0 is the lowest (very low completeness/accuracy) and 100 is the highest (very high completness/accuracy)? 
+        Do not base the two scores off each other give them the scores independently. Give explanations for every single one and if the answer if partially correct that is acceptable. However punish the scores for answers that are 
+        numerically incorrect this also includes values that have the $ in front
+        Please give the completeness score first followed by the accuracy score. 
+        For example: Completeness: 70. Accuracy: 40. Explanation here
+        Do not differ from the format ever
+        """
+        prompt = PromptTemplate.from_template(fstring)
+
+        self.__llm_evaluator = load_evaluator("criteria", llm=llm, criteria="conciseness",prompt=prompt)
+