File size: 7,020 Bytes
0a026c0 dca2e30 0a026c0 1bad9c0 bb4bda1 0a026c0 fd72b9a 0a026c0 fd72b9a da04c4f 2316f10 0a026c0 da04c4f 0a026c0 ea070cc 491fd45 ea070cc 0eb1a66 ea070cc 0eb1a66 0a026c0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
import json
from util.assistants import GPTAgent
import json_repair
class evaluator:
def __init__(self, model_name='GPT4-turbo'):
self.model = GPTAgent(model_name)
def validate_scores(self, scores):
required_keys = ["Factually Correct", "Useful", "Context Specific", "User Specific", "Provides Pluralism"]
for key in required_keys:
if key not in scores or not isinstance(scores[key], (int, float)) or not (-1 <= scores[key] <= 1):
return {"Factually Correct": -1,"Useful": -1,"Context Specific": -1,"User Specific":-1,"Provides Pluralism":-1}
return scores
def __call__(self, question,explanation):
evaluation_prompt = f"""You are provided with a user's question and the corresponding explanation generated by
an AI model. Your task is to evaluate the explanation based on the following five principles. Each principle
should be scored on a scale from 0 to 1, where 0 indicates that the principle is not met at all,
and 1 indicates that the principle is fully satisfied.
Question:
{question}
Provided Explanation:
{explanation}
Evaluation Criteria:
Factually Correct:
Definition: The explanation must be accurate and relevant to the question and the subject matter.
Score: (0-1) How factually correct is the explanation? Consider the accuracy of the details provided and their relevance to the question.
Useful:
Definition: The explanation should enable the user to understand the answer better and should facilitate further reasoning or decision-making.
Score: (0-1) How useful is the explanation in helping the user understand the answer and make informed decisions?
Context Specific:
Definition: The explanation should be relevant to the specific context or scenario implied by the question.
Score: (0-1) How well does the explanation address the specific context or scenario of the question?
User Specific:
Definition: The explanation should cater to the knowledge level and interests of the user, assuming typical or specified user characteristics.
Score: (0-1) How well does the explanation cater to the needs and knowledge level of the intended user?
Provides Pluralism:
Definition: The explanation should offer or accommodate multiple viewpoints or interpretations, allowing the user to explore various perspectives.
Score: (0-1) How well does the explanation provide or support multiple perspectives?
After evaluating the provided question and explanation based on the five principles, please format your scores in a JSON dictionary.
Example JSON format:
{{"Factually Correct": 0.9,"Useful": 0.85,"Context Specific": 0.8,"User Specific": 0.75,"Provides Pluralism": 0.7}}
Directly provide me with the json without any additional text.
Answer:
"""
response = self.model.invoke(evaluation_prompt,temperature=0.8, max_tokens=150).strip()
#response = """{{"Factually Correct": 0.9,"Useful": 0.85,"Context Specific": 0.8,"User Specific": 0.75,"Provides Pluralism": 0.7}}"""
print(response)
try:
scores = json.loads(response)
except json.JSONDecodeError:
# Attempt to repair the JSON if decoding fails
repaired_json = json_repair.repair_json(response, skip_json_loads=True, return_objects=False)
try:
scores = json.loads(repaired_json)
except json.JSONDecodeError:
print("Failed to decode JSON response even after repair attempt. Skipping this batch.")
return {"Factually Correct": -1,"Useful": -1,"Context Specific": -1,"User Specific":-1,"Provides Pluralism":-1}
return self.validate_scores(scores)
def write_evaluation_commentary(scores):
evaluation_details = []
for principle, score in scores.items():
if score == -1:
evaluation_details.append({'Principle': principle, 'Score': score, 'Commentary': 'Failed to evaluate the explanation.'})
continue
if principle == "Factually Correct":
if score >= 0.8:
comment = "Excellent accuracy! The information is precise and directly relevant to the question."
elif score >= 0.5:
comment = "Moderately accurate, but some details may not be completely correct or are somewhat irrelevant."
else:
comment = "The explanation contains significant inaccuracies or irrelevant information."
elif principle == "Useful":
if score >= 0.8:
comment = "Highly useful! The explanation clearly enhances understanding and aids in further reasoning or decision-making."
elif score >= 0.5:
comment = "Somewhat useful, though it could be more insightful or practical in aiding understanding."
else:
comment = "The explanation does little to help understand or apply the information provided."
elif principle == "Context Specific":
if score >= 0.8:
comment = "Perfectly tailored to the context of the question, addressing the specific scenario effectively."
elif score >= 0.5:
comment = "Generally addresses the context, but may miss specific details or nuances relevant to the question."
else:
comment = "Fails to address the context of the question, lacking relevance or specificity."
elif principle == "User Specific":
if score >= 0.8:
comment = "The explanation is well-adapted to the user's knowledge level and interests, demonstrating thoughtfulness."
elif score >= 0.5:
comment = "Moderately considerate of the user's knowledge level, but could be more tailored."
else:
comment = "Does not consider the user's background or interests, potentially leading to confusion or disinterest."
elif principle == "Provides Pluralism":
if score >= 0.8:
comment = "Provides an excellent range of perspectives or interpretations, fostering a comprehensive understanding."
elif score >= 0.5:
comment = "Offers some alternative perspectives, but more could be provided to enrich understanding."
else:
comment = "Lacks diversity in viewpoints, limiting the depth of exploration into the topic."
evaluation_details.append({'Principle': principle, 'Score': score, 'Commentary': comment})
return evaluation_details
if __name__ == '__main__':
eval = evaluator()
question = "What is the capital of France?"
explanation = "The capital of France is Paris."
print(eval(question, explanation)) |