LLM_eval

Former-commit-id: e006c931f38f3298f295e04cb9b16b3509442aee

evaluation/scores/LLM_eval.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # This script is used to evaluate the performance of Pigeon AI Video Translation system by using Large Language Model.
 
 # Written by Jiaen LIU, 2023/09/18
@@ -5,72 +6,44 @@
 # Import the necessary packages
 from langchain.evaluation import load_evaluator, EvaluatorType
 from langchain.prompts import PromptTemplate
-from langchain.chat_models import ChatAnthropic
-
-import re
-from tqdm import tqdm
-import pandas as pd
-import numpy as np
-import time
-import os
-import argparse
-from pathlib import Path 
-from src.srt_util.srt import SrtScript
-
-class PiegonLLMEvaluator():
-    """
-    input : 
-        - predicted sentences
-        - ground truth sentences
-    output :
-        - scores
-        - explanations
-    """
-
-    def __init__(self, output_dir: str, data_dir: str ) -> None:
-        # self.__eval_chain = self.__initialize_QAEvalChain()
-        self.__data_dir = data_dir
-        self.__result_dir = output_dir
-        # self.__chatbot = evaluater
-        self.__result_df = pd.DataFrame()
-        self.__initialize_df()
-        self.running_cost = 0
-        self.last_cost = 0
-        pass
-
-    def __init_llm_evaluater(self):
-        # llm = ChatOpenAI(temperature=0, model="gpt-4-0613")
-        # search = SerpAPIWrapper()
-        # tools = [
-        #     Tool(
-        #         name="Search",
-        #         func=search.run,
-        #         coroutine=search.arun,
-        #         description="Useful when you need to answer questions about current events. You should ask targeted questions.",
-        #     ),
-        # ]
-        # agents = [
-        #     initialize_agent(tools, llm, agent=AgentType.OPENAI_MULTI_FUNCTIONS, verbose=False),
-        #     initialize_agent(tools, llm, agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION, verbose=False)
-        # ]
-
-        llm = ChatAnthropic(temperature=0)
-
-        fstring = """You are an expert English to Chinese translator specialized in Startcraft2.
-        You are grading the following question:
-        {query}
-        Here is the real answer:
-        {answer}
-        You are grading the following predicted answer:
-        {result}
-        Give two grades, one for completness and another for accuracy and rate them from a scale of 0 to 100, where 0 is the lowest (very low completeness/accuracy) and 100 is the highest (very high completness/accuracy)? 
-        Do not base the two scores off each other give them the scores independently. Give explanations for every single one and if the answer if partially correct that is acceptable. However punish the scores for answers that are 
-        numerically incorrect this also includes values that have the $ in front
-        Please give the completeness score first followed by the accuracy score. 
-        For example: Completeness: 70. Accuracy: 40. Explanation here
-        Do not differ from the format ever
-        """
-        prompt = PromptTemplate.from_template(fstring)
-
-        self.__llm_evaluator = load_evaluator("criteria", llm=llm, criteria="conciseness",prompt=prompt)
-
+from langchain.chat_models import ChatOpenAI
+# from src.srt_util.srt import SrtScript
+
+# Load the evaluator
+
+def init_evaluator():
+
+    llm = ChatOpenAI(temperature=0, model="gpt-4-0613")
+
+    fstring = """You are an expert English to Chinese translator specialized in Startcraft2.
+            You are grading the following question:
+            {input}
+            Here is the real answer:
+            {reference}
+            You are grading the following predicted answer:
+            {output}
+            based on the following criteria: 
+            {criteria}
+            Give two grades, one for completness and another for accuracy and rate them from a scale of 0 to 100, where 0 is the lowest (very low completeness/accuracy) and 100 is the highest (very high completness/accuracy)? 
+            Do not base the two scores off each other give them the scores independently. Give explanations for every single one and if the answer if partially correct that is acceptable. However punish the scores for answers that are 
+            numerically incorrect this also includes values that have the $ in front
+            Please give the completeness score first followed by the accuracy score. 
+            For example: Completeness: 70. Accuracy: 40. Explanation here
+            Do not differ from the format ever
+            """
+    prompt = PromptTemplate.from_template(fstring)
+
+    return load_evaluator("labeled_criteria", llm=llm, prompt=prompt, criteria="correctness")
+
+def evaluate_prediction(input, reference, prediction, evaluator):
+    eval_result = evaluator.evaluate_strings(
+        prediction=prediction,
+        input=input,
+        reference=reference,
+    )
+    return eval_result
+
+if __name__ == "__main__":
+    evaluator = init_evaluator()
+    eval_result = evaluate_prediction("this is an test sentences", "这不是一个测试语句。", "这是一个测试句子。", evaluator)
+    print(eval_result)