ape/ape.py

# -*-coding:utf-8 -*-

import pandas as pd
from ape.instance import Instance, LoadFactory, upload_file
from ape.llm import LLMGPT
from functools import partial
from itertools import chain

LLM = None


def load_task(task, file):
    global instance
    if task:
        loader = LoadFactory[task]
    else:
        file = [i.name for i in file]
        print(file)
        loader = partial(upload_file, file=file[0])
    instance = Instance.from_file(loader)
    print(instance.samples[0])
    return instance, f'{instance.n_sample} Data Loaded'


def sample_data(instance, n_train, n_few_shot, n_eval):
    instance.sample(n_train, n_few_shot, n_eval)
    train_str = instance.display(instance.train_samples)
    eval_str = instance.display(instance.eval_samples)
    return train_str, eval_str, instance, 'Sample Done'


def esttimate_cost(instance):
    train_text = ''.join(chain(*instance.train_samples))
    eval_text = ''.join(chain(*instance.eval_samples))
    train_cost = LLMGPT.confirm_cost(train_text, 'train')
    eval_cost = LLMGPT.confirm_cost(eval_text, 'eval')
    return f'Train={train_cost} Eval={eval_cost}'


def generate(gen_prompt, instance, openai_key):
    global LLM
    if LLM is None:
        LLM = LLMGPT(openai_key)

    instructions = []
    train_iter = instance.get_train_iter()
    for few_shot in train_iter:
        instruction = LLM.generate_instruction(gen_prompt, few_shot)
        print(instruction)
        instructions.append(instruction['text'])
    return '\n'.join(instructions)


def single_test(test_prompt, instruction, input, openai_key):
    global LLM
    if LLM is None:
        LLM = LLMGPT(openai_key)

    output = LLM.generate_output(test_prompt, instruction, input)
    return output['text']


def score_single(eval_prompt, instance, instruction, openai_key):
    global LLM
    if LLM is None:
        LLM = LLMGPT(openai_key)

    score = LLM.generate_logprobs(eval_prompt, instruction, instance.eval_samples)
    return score


def evaluate(eval_prompt, all_instructions, instance, openai_key):
    scores = []
    for instruction in all_instructions:
        score = score_single(eval_prompt, instance, instruction, openai_key)
        scores.append(score)
    return pd.DataFrame({'instruction': all_instructions, 'likelihood': scores})