Spaces:
Sleeping
Sleeping
File size: 3,080 Bytes
4fe4082 195be44 4fe4082 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
import json
import os
import random
from typing import Tuple, Dict
import jsonlines
from Config import *
from models import select_model
def read_all(path: str) -> str:
with open(path, 'r') as f:
return f.read()
class Card:
json_obj: Dict
def __init__(self, path: str):
self.json_obj = json.load(open(path, 'r'))
def get_markdown_str(self) -> str:
m = ""
for k, v in self.json_obj.items():
if isinstance(v, str):
m += f'- {k}: {v}\n'
elif isinstance(v, dict):
m += f"- {k}: {v['overview']}\n"
if v['thinking_pattern'] + v['strength'] + v['weakness'] == '':
continue
m += f" - Thinking Patterns: {v['thinking_pattern']}\n"
m += f" - Strength: {v['strength']}\n"
m += f" - Weakness: {v['weakness']}\n"
else:
raise ValueError(f'Unknown type: {type(v)}')
return m
def __str__(self):
return self.get_markdown_str()
def sample_random_card(dataset: str, topic: str, model: str) -> Tuple[Card, str]:
"""
Returns a random card and the file name of the card.
"""
cards_dir = f"{CARD_DIR}/{dataset}/{topic}"
prefix = f"{model}"
# list all .json files start with prefix in cards_dir
files = [f for f in os.listdir(cards_dir)
if f.startswith(prefix) and f.endswith(".json")]
assert len(files) > 0, f"No card found for {dataset} - {topic} - {model}"
# randomly select a file
card_file = random.choice(files)
card_path = os.path.join(cards_dir, card_file)
return Card(card_path), card_file
def format_qa_entry(qa: Dict) -> str:
question = qa['question']
choices = qa['choices']
ground_truth = qa['ground truth']
choice_str = ''
# choices are in 0 - n, convert to A - Z
for i, c in enumerate(choices):
choice_str += f"{chr(65 + i)}. {c}\n"
choice_str = choice_str[:-1]
return question + '\n\n' + choice_str + '\n\n' + f'Ground Truth: {chr(65 + ground_truth)}'
def sample_random_qa(dataset: str, topic: str, model: str) -> Tuple[str, str, bool]:
"""
Returns qa str, model's answer, and whether the model's answer is correct.
"""
# get qa str, model's answer
qa_path = f"{DATASET_DIR}/{dataset}/{topic}/{model}_test.jsonl"
with jsonlines.open(qa_path) as reader:
lines = list(reader)
item = random.choice(lines)
qa_str = format_qa_entry(item)
model_reason = item["completion"]
model_choice = chr(65 + item["verdict"])
completion = model_reason + "\n\n" + f"Choice: {model_choice}"
return qa_str, completion, item["ground truth"] == item["verdict"]
def summarize_card(summarizer: str, card: Card, qa: str) -> str:
system_prompt = read_all("prompts/summarize/system.txt")
user_prompt = read_all("prompts/summarize/user.txt").format(
card=str(card), qa=qa
)
model = select_model(summarizer, system_prompt)
r = model(user_prompt, cache=True)
return r
|