Spaces:
Runtime error
Runtime error
# -*-coding:utf-8 -*- | |
from glob import glob | |
import pandas as pd | |
from ape.instance import Instance, LoadFactory, upload_file | |
from ape.llm import LLMGPT | |
from functools import partial | |
from itertools import chain | |
def load_task(task, file): | |
global instance | |
if task: | |
loader = LoadFactory[task] | |
else: | |
loader = partial(upload_file, file=file.name) | |
instance = Instance.from_file(loader) | |
print(instance.samples[0]) | |
return instance, f'{instance.n_sample} Data Loaded' | |
def sample_data(instance, n_train, n_few_shot, n_eval): | |
instance.sample(n_train, n_few_shot, n_eval) | |
train_str = instance.display(instance.train_samples) | |
eval_str = instance.display(instance.eval_samples) | |
return train_str, eval_str, instance, 'Sample Done' | |
def esttimate_cost(instance): | |
train_text = ''.join(chain(*instance.train_samples)) | |
eval_text = ''.join(chain(*instance.eval_samples)) | |
train_cost = LLMGPT.confirm_cost(train_text, 'train') | |
eval_cost = LLMGPT.confirm_cost(eval_text, 'eval') | |
return f'Train={train_cost} Eval={eval_cost}' | |
def generate(gen_prompt, instance, openai_key): | |
LLM = LLMGPT(openai_key) | |
instructions = [] | |
train_iter = instance.get_train_iter() | |
for few_shot in train_iter: | |
instruction = LLM.generate_instruction(gen_prompt, few_shot) | |
print(instruction) | |
instructions.append(instruction['text']) | |
return '\n'.join(instructions) | |
def single_test(test_prompt, instruction, input, openai_key): | |
LLM = LLMGPT(openai_key) | |
output = [] | |
for i in input.split('\n'): | |
output.append(LLM.generate_output(test_prompt, instruction, i)['text']) | |
return '\n'.join(output) | |
def score_single(eval_prompt, instance, instruction, openai_key): | |
LLM = LLMGPT(openai_key) | |
score = LLM.generate_logprobs(eval_prompt, instruction, instance.eval_samples) | |
return score | |
def evaluate(eval_prompt, all_instructions, instance, openai_key): | |
all_instructions = all_instructions.split('\n') | |
scores = [] | |
for instruction in all_instructions: | |
score = score_single(eval_prompt, instance, instruction, openai_key) | |
scores.append(score) | |
df = pd.DataFrame({'instruction': all_instructions, 'likelihood': scores}) | |
df.sort_values(by='likelihood', ascending=False, inplace=True) | |
return df | |