test / src /eval.py
iblfe's picture
Upload folder using huggingface_hub
b585c7f verified
raw history blame
No virus
17.4 kB
import os
import traceback
import numpy as np
import pandas as pd
import torch
from matplotlib import pyplot as plt
from evaluate_params import eval_func_param_names, eval_extra_columns, input_args_list
from gen import get_score_model, get_model, evaluate, check_locals, get_model_retry
from prompter import Prompter
from utils import clear_torch_cache, NullContext, get_kwargs, makedirs
def run_eval( # for local function:
base_model=None, lora_weights=None, inference_server=None, regenerate_clients=None,
prompt_type=None, prompt_dict=None, system_prompt=None,
debug=None, chat=False,
stream_output=None, async_output=None, num_async=None,
eval_filename=None, eval_prompts_only_num=None, eval_prompts_only_seed=None, eval_as_output=None,
examples=None, memory_restriction_level=None,
# for get_model:
score_model=None, load_8bit=None, load_4bit=None, low_bit_mode=None, load_half=None, use_flash_attention_2=None,
load_gptq=None, use_autogptq=None, load_awq=None, load_exllama=None, use_safetensors=None, revision=None,
use_gpu_id=None, tokenizer_base_model=None,
gpu_id=None, n_jobs=None, n_gpus=None, local_files_only=None, resume_download=None, use_auth_token=None,
trust_remote_code=None, offload_folder=None, rope_scaling=None, max_seq_len=None, compile_model=None,
llamacpp_dict=None, llamacpp_path=None,
exllama_dict=None, gptq_dict=None, attention_sinks=None, sink_dict=None, hf_model_dict=None,
truncation_generation=None,
use_pymupdf=None,
use_unstructured_pdf=None,
use_pypdf=None,
enable_pdf_ocr=None,
enable_pdf_doctr=None,
enable_imagegen_high_sd=None,
try_pdf_as_html=None,
# for evaluate args beyond what's already above, or things that are always dynamic and locally created
temperature=None,
top_p=None,
top_k=None,
penalty_alpha=None,
num_beams=None,
max_new_tokens=None,
min_new_tokens=None,
early_stopping=None,
max_time=None,
repetition_penalty=None,
num_return_sequences=None,
do_sample=None,
langchain_mode=None,
langchain_action=None,
langchain_agents=[],
top_k_docs=None,
chunk=None,
chunk_size=None,
document_subset=None,
document_choice=None,
document_source_substrings=None,
document_source_substrings_op=None,
document_content_substrings=None,
document_content_substrings_op=None,
pre_prompt_query=None, prompt_query=None,
pre_prompt_summary=None, prompt_summary=None, hyde_llm_prompt=None,
image_audio_loaders=None,
pdf_loaders=None,
url_loaders=None,
jq_schema=None,
extract_frames=None,
llava_prompt=None,
visible_models=None,
h2ogpt_key=None,
add_search_to_context=None,
chat_conversation=None,
text_context_list=None,
docs_ordering_type=None,
min_max_new_tokens=None,
max_input_tokens=None,
max_total_input_tokens=None,
docs_token_handling=None,
docs_joiner=None,
hyde_level=None,
hyde_template=None,
hyde_show_only_final=None,
hyde_show_intermediate_in_accordion=None,
doc_json_mode=None,
chatbot_role=None,
speaker=None,
tts_language=None,
tts_speed=None,
# for evaluate kwargs:
captions_model=None,
caption_loader=None,
doctr_loader=None,
pix2struct_loader=None,
llava_model=None,
image_gen_loader=None,
image_gen_loader_high=None,
image_change_loader=None,
asr_model=None,
asr_loader=None,
image_audio_loaders_options0=None,
pdf_loaders_options0=None,
url_loaders_options0=None,
jq_schema0=None,
keep_sources_in_context=None,
gradio_errors_to_chatbot=None,
allow_chat_system_prompt=None,
src_lang=None, tgt_lang=None, concurrency_count=None, save_dir=None, sanitize_bot_response=None,
model_state0=None,
max_max_new_tokens=None,
is_public=None,
max_max_time=None,
raise_generate_gpu_exceptions=None, load_db_if_exists=None, use_llm_if_no_docs=None,
my_db_state0=None, selection_docs_state0=None, dbs=None, langchain_modes=None, langchain_mode_paths=None,
detect_user_path_changes_every_query=None,
use_openai_embedding=None, use_openai_model=None,
hf_embedding_model=None, migrate_embedding_model=None, auto_migrate_db=None,
cut_distance=None,
answer_with_sources=None,
append_sources_to_answer=None,
append_sources_to_chat=None,
show_accordions=None,
top_k_docs_max_show=None,
show_link_in_sources=None,
langchain_instruct_mode=None,
add_chat_history_to_context=None,
context=None, iinput=None,
db_type=None, first_para=None, text_limit=None, verbose=None,
gradio=None, cli=None,
use_cache=None,
auto_reduce_chunks=None, max_chunks=None, headsize=None,
model_lock=None, force_langchain_evaluate=None,
model_state_none=None,
):
from_ui = False
# makes no sense to evaluate document content for langchain case
answer_with_sources = False
show_link_in_sources = False
append_sources_to_answer = False
append_sources_to_chat = False
check_locals(**locals())
if not context:
context = ''
if eval_prompts_only_num > 0:
np.random.seed(eval_prompts_only_seed)
example1 = examples[-1] # pick reference example
examples = []
responses = []
if eval_filename is None:
# override default examples with shareGPT ones for human-level eval purposes only
eval_filename = 'ShareGPT_V3_unfiltered_cleaned_split_no_imsorry.json'
if not os.path.isfile(eval_filename):
os.system(
'wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/%s' % eval_filename)
import json
with open(eval_filename, 'r', encoding='utf-8') as f:
data = json.load(f)
# focus on data that starts with human, else likely chopped from other data
turn_start = 0 # odd in general
data = [x for x in data if len(x['conversations']) > turn_start + 1 and
x['conversations'][turn_start]['from'] == 'human' and
x['conversations'][turn_start + 1]['from'] == 'gpt']
for i in sorted(np.random.randint(0, len(data), size=eval_prompts_only_num)):
assert data[i]['conversations'][turn_start]['from'] == 'human'
instruction = data[i]['conversations'][turn_start]['value']
assert data[i]['conversations'][turn_start + 1]['from'] == 'gpt'
output = data[i]['conversations'][turn_start + 1]['value']
examplenew = example1.copy()
assert not chat, "No gradio must use chat=False, uses nochat instruct"
examplenew[eval_func_param_names.index('instruction_nochat')] = instruction
examplenew[eval_func_param_names.index('iinput_nochat')] = iinput
examplenew[eval_func_param_names.index('context')] = context
examples.append(examplenew)
responses.append(output)
else:
# get data, assume in correct format: json of rows of dict of instruction and output
# only instruction is required
import json
with open(eval_filename, 'r', encoding='utf-8') as f:
data = json.load(f)
for i in sorted(np.random.randint(0, len(data), size=eval_prompts_only_num)):
examplenew = example1.copy()
instruction = data[i]['instruction']
output = data[i].get('output', '') # not required
assert not chat, "No gradio must use chat=False, uses nochat instruct"
examplenew[eval_func_param_names.index('instruction_nochat')] = instruction
examplenew[eval_func_param_names.index('iinput_nochat')] = iinput
examplenew[eval_func_param_names.index('context')] = context
examples.append(examplenew)
responses.append(output)
num_examples = len(examples)
scoring_path = 'scoring'
# if no permissions, assume may not want files, put into temp
scoring_path = makedirs(scoring_path, tmp_ok=True, use_base=True)
if eval_as_output:
used_base_model = 'gpt35'
used_lora_weights = ''
used_inference_server = ''
else:
used_base_model = str(base_model.split('/')[-1])
used_lora_weights = str(lora_weights.split('/')[-1])
used_inference_server = str(inference_server.split('/')[-1])
eval_out_filename = "df_scores_%s_%s_%s_%s_%s_%s_%s.parquet" % (num_examples, eval_prompts_only_num,
eval_prompts_only_seed,
eval_as_output,
used_base_model,
used_lora_weights,
used_inference_server,
)
eval_out_filename = os.path.join(scoring_path, eval_out_filename)
# torch.device("cuda") leads to cuda:x cuda:y mismatches for multi-GPU consistently
n_gpus = torch.cuda.device_count() if torch.cuda.is_available() else 0
device = 'cpu' if n_gpus == 0 else 'cuda'
context_class = NullContext if n_gpus > 1 or n_gpus == 0 else torch.device
with context_class(device):
# ensure was set right above before examples generated
assert not stream_output, "stream_output=True does not make sense with example loop"
import time
from functools import partial
# get score model
smodel, stokenizer, sdevice = get_score_model(reward_type=True,
**get_kwargs(get_score_model, exclude_names=['reward_type'],
**locals()))
if not eval_as_output:
model, tokenizer, device = get_model_retry(reward_type=False,
**get_kwargs(get_model, exclude_names=['reward_type'],
**locals()))
model_dict = dict(base_model=base_model, tokenizer_base_model=tokenizer_base_model,
lora_weights=lora_weights,
inference_server=inference_server, prompt_type=prompt_type, prompt_dict=prompt_dict,
visible_models=None, h2ogpt_key=None)
model_state = dict(model=model, tokenizer=tokenizer, device=device)
model_state.update(model_dict)
requests_state0 = {}
roles_state0 = None
args = (model_state, my_db_state0, selection_docs_state0, requests_state0, roles_state0)
assert len(args) == len(input_args_list)
fun = partial(evaluate,
*args,
**get_kwargs(evaluate, exclude_names=input_args_list + eval_func_param_names,
**locals()))
else:
assert eval_prompts_only_num > 0
def get_response(*args, exi=0):
# assumes same ordering of examples and responses
yield responses[exi]
fun = get_response
t0 = time.time()
score_dump = []
score_avg = 0
score_median = 0
for exi, ex in enumerate(examples):
clear_torch_cache(allow_skip=True)
instruction = ex[eval_func_param_names.index('instruction_nochat')]
iinput = ex[eval_func_param_names.index('iinput_nochat')]
context = ex[eval_func_param_names.index('context')]
clear_torch_cache(allow_skip=True)
print("")
print("START" + "=" * 100)
print("Question: %s %s" % (instruction, ('input=%s' % iinput if iinput else '')))
print("-" * 105)
# fun yields as generator, so have to iterate over it
# Also means likely do NOT want --stream_output=True, else would show all generations
t1 = time.time()
# grab other parameters, like langchain_mode
eval_vars = ex.copy()
for k in eval_func_param_names:
if k in locals():
eval_vars[eval_func_param_names.index(k)] = locals()[k]
gener = fun(*tuple(eval_vars), exi=exi) if eval_as_output else fun(*tuple(eval_vars))
for res_fun in gener:
res = res_fun['response']
sources = res_fun.get('sources', 'Failure of Generation')
print(res)
if smodel:
score_with_prompt = False
if score_with_prompt:
data_point = dict(instruction=instruction, input=iinput, context=context)
prompter = Prompter(prompt_type, prompt_dict,
debug=debug, stream_output=stream_output)
prompt = prompter.generate_prompt(data_point, context_from_history=False)
else:
# just raw input and output
if eval_prompts_only_num > 0:
# only our own examples have this filled at moment
assert iinput in [None, ''], iinput # should be no iinput
prompt = instruction
if memory_restriction_level > 0:
cutoff_len = 768 if memory_restriction_level <= 2 else 512
else:
cutoff_len = tokenizer.model_max_length
inputs = stokenizer(prompt, res,
return_tensors="pt",
truncation=True,
max_length=cutoff_len)
try:
score = \
torch.sigmoid(smodel(**inputs.to(smodel.device)).logits[0].float()).cpu().detach().numpy()[
0]
except torch.cuda.OutOfMemoryError as e:
print("GPU OOM 1: question: %s answer: %s exception: %s" % (prompt, res, str(e)),
flush=True)
traceback.print_exc()
score = 0.0
clear_torch_cache()
except (Exception, RuntimeError) as e:
if 'Expected all tensors to be on the same device' in str(e) or \
'expected scalar type Half but found Float' in str(e) or \
'probability tensor contains either' in str(e) or \
'cublasLt ran into an error!' in str(e):
print("GPU error: question: %s answer: %s exception: %s" % (prompt, res, str(e)),
flush=True)
traceback.print_exc()
score = 0.0
clear_torch_cache()
else:
raise
score_dump.append(ex + [prompt, res, score])
# dump every score in case abort
df_scores = pd.DataFrame(score_dump,
columns=eval_func_param_names +
eval_extra_columns)
df_scores.to_parquet(eval_out_filename, index=False)
# plot histogram so far
plt.figure(figsize=(10, 10))
plt.hist(df_scores['score'], bins=20)
score_avg = np.mean(df_scores['score'])
score_median = np.median(df_scores['score'])
print("SCORE %s: %s So far: AVG: %s MEDIAN: %s" % (exi, score, score_avg, score_median),
flush=True)
plt.title("Score avg: %s median: %s" % (score_avg, score_median))
plt.savefig(eval_out_filename.replace('.parquet', '.png'))
plt.close()
print("END" + "=" * 102)
print("")
t2 = time.time()
print("Time taken for example: %s Time taken so far: %.4f about %.4g per example" % (
t2 - t1, t2 - t0, (t2 - t0) / (1 + exi)))
t1 = time.time()
print("Total time taken: %.4f about %.4g per example" % (t1 - t0, (t1 - t0) / num_examples))
print("Score avg: %s median: %s" % (score_avg, score_median), flush=True)
return eval_out_filename