Spaces:
Sleeping
Sleeping
| import random | |
| import numpy as np | |
| import os | |
| import json | |
| from Config import * | |
| import pandas as pd | |
| from models import HFAPIModel | |
| def format_card_str(card): | |
| entries = [] | |
| for k, v in card.items(): | |
| r = '' | |
| if isinstance(v, str): | |
| r += f'- {k}: {v}\n' | |
| elif isinstance(v, dict): | |
| r += f"- {k}: {v['overview']}\n" | |
| # r += f"- {k}:\n" | |
| if v['thinking_pattern'] + v['strength'] + v['weakness'] == '': | |
| continue | |
| r += f" - Thinking Patterns: {v['thinking_pattern']}\n" | |
| r += f" - Strength: {v['strength']}\n" | |
| r += f" - Weakness: {v['weakness']}\n" | |
| else: | |
| raise ValueError(f'Unknown type: {type(v)}') | |
| entries.append(r) | |
| return entries | |
| def format_qa_entry(qa): | |
| # concat question + choice | |
| question = qa['question'] | |
| choices = qa['choices'] | |
| ground_truth = qa['ground truth'] | |
| choice_str = '' | |
| # choices are in 0 - n, convert to A - Z | |
| for i, c in enumerate(choices): | |
| choice_str += f"{chr(65+i)}. {c}\n" | |
| choice_str = choice_str[:-1] | |
| return question + '\n\n' + choice_str +'\n\n' + f'Ground Truth: {chr(65+ground_truth)}' | |
| def sample_random_entry(dataset='', topic='', model='', n=1): | |
| """ | |
| Sample n (cache_size) random entries from the dataset, topic, model | |
| """ | |
| if dataset == '': | |
| dataset = random.choice(DATASETS) | |
| if topic == '': | |
| topic = random.choice(TOPICS[dataset]) | |
| if model == '': | |
| model = random.choice(MODELS) | |
| # print(f"Sampling {n} random entries from {dataset} - {topic} - {model}") | |
| card_lst = sample_card(dataset, topic, model) | |
| qa, index = sample_QA_entry(dataset, topic, model) | |
| display_dict, info_dict = process_for_display(card_lst, qa) | |
| info_dict['index'] = index | |
| return display_dict, info_dict | |
| def process_for_display(card_lst, qa): | |
| qa_entry = format_qa_entry(qa) | |
| display_dict = {} | |
| display_dict['card'] = select_entry(qa_entry, card_lst) | |
| display_dict['qa'] = qa_entry | |
| info_dict = {**qa} | |
| info_dict.pop('question') | |
| info_dict.pop('choices') | |
| return display_dict, info_dict | |
| def select_entry(qa_entry, card_lst): | |
| system_prompt = ''' | |
| Your task is to effectively condense the essential details from the student's evaluation card that are most relevant to predicting the correctness of their answer to a question. | |
| Limit your paraphrase to 50-100 words, focusing on distilling the key observations and outcomes that are directly pertinent to the inquiry. | |
| It's crucial to present an informative, unbiased summary that retains the integrity of the original card's information. | |
| Your goal is to craft a paraphrase that enhances the user's ability to accurately gauge the student's response, by emphasizing relevant insights and conclusions without altering the core facts. | |
| ''' | |
| card_str = '\n'.join(card_lst) | |
| prompt = f''' | |
| ## Question: | |
| {qa_entry} | |
| ## Evaluation Card: | |
| {card_str} | |
| Again, your task is not to answer the question, but summarize the student's ability in answering the question! Only 100 words max! Use bullet points. | |
| Only relevant information to the question is needed. | |
| ''' | |
| model_avaliable = {'mixtral': 'mistralai/Mixtral-8x7B-Instruct-v0.1', | |
| 'mistral': 'mistralai/Mistral-7B-Instruct-v0.2'} | |
| model = HFAPIModel(system_prompt=system_prompt, | |
| model_name=model_avaliable['mistral']) | |
| response = model(prompt).replace('\n', '\n\n') | |
| del model | |
| return response | |
| def sample_card(dataset='', topic='', model='', card_cnt=2): | |
| card_index = random.randint(0, card_cnt-1) | |
| path = f'dataset/{dataset}/cards/{topic}/{topic}_{model}_{card_index}.jsonl' | |
| # load jsonl | |
| with open(path, 'r') as f: | |
| data = json.load(f) | |
| card = format_card_str(data) | |
| return card | |
| def sample_QA_entry(dataset='', topic='', model='', n=1): | |
| path = f'dataset/{dataset}/{topic}/{topic}_test.jsonl' | |
| # load jsonl | |
| # with jsonlines.open(path) as reader: | |
| # data = list(reader) | |
| # use json | |
| # load line by line | |
| with open(path, 'r') as f: | |
| data = [json.loads(line) for line in f.readlines()] | |
| # transfer into pandas | |
| df = pd.DataFrame(data) | |
| # select whose model equals model | |
| df = df[df['model'] == model] | |
| sample = df.sample(1) | |
| # Convert to dictionary | |
| sample_idx = sample.index[0] | |
| sample = sample.to_dict(orient='records')[0] | |
| return sample, sample_idx | |
| if __name__ == '__main__': | |
| sample_random_entry(n=5) |