Spaces:
Sleeping
Sleeping
File size: 4,685 Bytes
abedf13 b11f272 abedf13 336c585 abedf13 91143ec abedf13 336c585 91143ec abedf13 b11f272 336c585 b11f272 336c585 b11f272 336c585 b11f272 336c585 b11f272 336c585 b11f272 abedf13 07a63f0 abedf13 91143ec abedf13 91143ec abedf13 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
import random
import numpy as np
import os
import json
from Config import *
import pandas as pd
from models import HFAPIModel
def format_card_str(card):
entries = []
for k, v in card.items():
r = ''
if isinstance(v, str):
r += f'- {k}: {v}\n'
elif isinstance(v, dict):
r += f"- {k}: {v['overview']}\n"
# r += f"- {k}:\n"
if v['thinking_pattern'] + v['strength'] + v['weakness'] == '':
continue
r += f" - Thinking Patterns: {v['thinking_pattern']}\n"
r += f" - Strength: {v['strength']}\n"
r += f" - Weakness: {v['weakness']}\n"
else:
raise ValueError(f'Unknown type: {type(v)}')
entries.append(r)
return entries
def format_qa_entry(qa):
# concat question + choice
question = qa['question']
choices = qa['choices']
ground_truth = qa['ground truth']
choice_str = ''
# choices are in 0 - n, convert to A - Z
for i, c in enumerate(choices):
choice_str += f"{chr(65+i)}. {c}\n"
choice_str = choice_str[:-1]
return question + '\n\n' + choice_str +'\n\n' + f'Ground Truth: {chr(65+ground_truth)}'
def sample_random_entry(dataset='', topic='', model='', n=1):
"""
Sample n (cache_size) random entries from the dataset, topic, model
"""
if dataset == '':
dataset = random.choice(DATASETS)
if topic == '':
topic = random.choice(TOPICS[dataset])
if model == '':
model = random.choice(MODELS)
# print(f"Sampling {n} random entries from {dataset} - {topic} - {model}")
card_lst = sample_card(dataset, topic, model)
qa, index = sample_QA_entry(dataset, topic, model)
display_dict, info_dict = process_for_display(card_lst, qa)
info_dict['index'] = index
return display_dict, info_dict
def process_for_display(card_lst, qa):
qa_entry = format_qa_entry(qa)
display_dict = {}
display_dict['card'] = select_entry(qa_entry, card_lst)
display_dict['qa'] = qa_entry
info_dict = {**qa}
info_dict.pop('question')
info_dict.pop('choices')
return display_dict, info_dict
def select_entry(qa_entry, card_lst):
system_prompt = '''
Your task is to effectively condense the essential details from the student's evaluation card that are most relevant to predicting the correctness of their answer to a question.
Limit your paraphrase to 50-100 words, focusing on distilling the key observations and outcomes that are directly pertinent to the inquiry.
It's crucial to present an informative, unbiased summary that retains the integrity of the original card's information.
Your goal is to craft a paraphrase that enhances the user's ability to accurately gauge the student's response, by emphasizing relevant insights and conclusions without altering the core facts.
'''
card_str = '\n'.join(card_lst)
prompt = f'''
## Question:
{qa_entry}
## Evaluation Card:
{card_str}
Again, your task is not to answer the question, but summarize the student's ability in answering the question! Only 100 words max! Use bullet points.
Only relevant information to the question is needed.
'''
model_avaliable = {'mixtral': 'mistralai/Mixtral-8x7B-Instruct-v0.1',
'mistral': 'mistralai/Mistral-7B-Instruct-v0.2'}
model = HFAPIModel(system_prompt=system_prompt,
model_name=model_avaliable['mistral'])
response = model(prompt).replace('\n', '\n\n')
del model
return response
def sample_card(dataset='', topic='', model='', card_cnt=2):
card_index = random.randint(0, card_cnt-1)
path = f'dataset/{dataset}/cards/{topic}/{topic}_{model}_{card_index}.jsonl'
# load jsonl
with open(path, 'r') as f:
data = json.load(f)
card = format_card_str(data)
return card
def sample_QA_entry(dataset='', topic='', model='', n=1):
path = f'dataset/{dataset}/{topic}/{topic}_test.jsonl'
# load jsonl
# with jsonlines.open(path) as reader:
# data = list(reader)
# use json
# load line by line
with open(path, 'r') as f:
data = [json.loads(line) for line in f.readlines()]
# transfer into pandas
df = pd.DataFrame(data)
# select whose model equals model
df = df[df['model'] == model]
sample = df.sample(1)
# Convert to dictionary
sample_idx = sample.index[0]
sample = sample.to_dict(orient='records')[0]
return sample, sample_idx
if __name__ == '__main__':
sample_random_entry(n=5) |