kaz-llm-lb / src /gen /gen_judgment.py
hi-melnikov's picture
First setup of leaderboard
8e67ebe
raw
history blame
8.15 kB
import argparse
import concurrent.futures
import glob
import json
import os
import re
import huggingface_hub
from tqdm import tqdm
from utils import (
chat_completion_anthropic,
chat_completion_openai,
chat_completion_openai_azure,
get_endpoint,
load_model_answers,
load_questions,
make_config,
)
def get_score(judgment, pattern, pairwise=True):
matches = pattern.findall(judgment)
matches = [m for m in matches if m != ""]
if len(set(matches)) == 0:
return None, True
elif len(set(matches)) == 1:
if pairwise:
return matches[0].strip("\n"), False
return int(matches[0])
else:
return None, False
# get answer from model
def get_answer(model, conv, temperature, max_tokens, endpoint_dict=None):
api_dict = get_endpoint(endpoint_dict["endpoints"])
if endpoint_dict["api_type"] == "anthropic":
output = chat_completion_anthropic(model, conv, temperature, max_tokens)
elif endpoint_dict["api_type"] == "azure":
output = chat_completion_openai_azure(model, conv, temperature, max_tokens, api_dict)
else:
output = chat_completion_openai(model, conv, temperature, max_tokens, api_dict)
return output
def judgment(**args):
question = args["question"]
answer = args["answer"]
reference = args["reference"]
baseline = args["baseline_answer"]
configs = args["configs"]
output_file = args["output_file"]
model = configs["judge_model"]
num_games = 2 if configs["pairwise"] else 1
output = {
"question_id":question["question_id"],
"model":answer["model_id"],
"judge": model,
"games":[]
}
for game in range(num_games):
conv = [{"role": "system", "content": configs["system_prompt"]}]
for template in configs["prompt_template"]:
prompt_args = {}
for i, turn in enumerate(question["turns"]):
prompt_args[f"question_{i+1}"] = turn["content"]
base = 1
if baseline:
if game % 2 == 1: # swap position
temp = baseline
baseline = answer
answer = temp
for i, turn in enumerate(baseline["choices"][0]["turns"]):
prompt_args[f"answer_{i+1}"] = turn["content"]
base += 1
if answer:
for i, turn in enumerate(answer["choices"][0]["turns"]):
prompt_args[f"answer_{i+base}"] = turn["content"]
if reference:
for j, ref_answer in enumerate(reference):
for i, turn in enumerate(ref_answer["choices"][0]["turns"]):
prompt_args[f"ref_answer_{i+j+1}"] = turn["content"]
user_prompt = template.format(**prompt_args)
conv.append({"role": "user", "content": user_prompt})
judgment = ""
for _ in range(2):
new_judgment = get_answer(
model,
conv,
configs["temperature"],
configs["max_tokens"],
args["endpoint_dict"],
)
judgment += ("\n" + new_judgment)
score, try_again = get_score(judgment, args["regex_pattern"])
conv.append({"role": "assistant", "content": new_judgment})
if not try_again:
break
conv.append({"role": "user", "content": "continue your judgment and finish by outputting a final verdict label"})
result = {
"user_prompt": conv[1]["content"],
"judgment": judgment,
"score":score
}
output["games"].append(result)
with open(output_file, "a") as f:
f.write(json.dumps(output, ensure_ascii=False) + "\n")
huggingface_hub.HfApi().upload_file(output_file, path_in_repo=f'model_judgment/{configs['judge_model']}/{output_file.split('/')[-1]}', repo_id='Vikhrmodels/openbench-eval', repo_type='dataset')
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--setting-file", type=str, default="./config/judge_config.yaml")
parser.add_argument("--endpoint-file", type=str, default="./config/api_config.yaml")
args = parser.parse_args()
print(args)
configs = make_config(args.setting_file)
endpoint_list = make_config(args.endpoint_file)
print(f'judge model: {configs["judge_model"]}, baseline: {configs["baseline"]}, baseline model: {configs["baseline_model"]}, reference: {configs["reference"]}, '
+ f'reference models: {configs["ref_model"]}, temperature: {configs["temperature"]}, max tokens: {configs["max_tokens"]}, pairwise: {configs["pairwise"]}')
if configs["regex_pattern"]:
pattern = re.compile(configs["regex_pattern"])
question_file = os.path.join("./data", configs["bench_name"], "question.jsonl")
external_dir = os.path.join("./data", configs["bench_name"], "model_answer/external")
internal_dir = os.path.join("./data", configs["bench_name"], "model_answer/internal")
ref_answer_dir = os.path.join("data", configs["bench_name"], "reference_answer")
questions = load_questions(question_file)
model_answers_external = load_model_answers(external_dir)
model_answers_internal = load_model_answers(internal_dir)
# internal has priority
model_answers = {**model_answers_external, **model_answers_internal}
# if user choose a set of models, only judge those models
models = [model.split('/')[-1].split('.')[0] for model in glob.glob('./data/arena-hard-v0.1/model_answer/external/*.jsonl')]
ref_answers = None
if configs["reference"]:
ref_answers = load_model_answers(ref_answer_dir)
ref_answers = [ref_answers[model] for model in configs["ref_model"]]
output_files = {}
output_dir = f"data/{configs['bench_name']}/model_judgment/{configs['judge_model']}"
for model in models:
output_files[model] = os.path.join(
output_dir,
f"{model}.jsonl",
)
for output_file in output_files.values():
os.makedirs(os.path.dirname(output_file), exist_ok=True)
existing_judgments = load_model_answers(output_dir)
endpoint_info = endpoint_list[configs["judge_model"]]
with concurrent.futures.ThreadPoolExecutor(max_workers=endpoint_info["parallel"]) as executor:
futures = []
for model in models:
count = 0
for question in questions[:2]:
question_id = question["question_id"]
kwargs = {}
kwargs["question"] = question
if model in model_answers and question_id not in model_answers[model]:
print(f"Warning: {model} answer to {question['question_id']} cannot be found.")
continue
if model in existing_judgments and question_id in existing_judgments[model]:
count += 1
continue
kwargs["answer"] = model_answers[model][question_id]
if ref_answers:
kwargs["reference"] = [ref_answer[question_id] for ref_answer in ref_answers]
assert len(kwargs["reference"]) == len(configs["ref_model"])
else:
kwargs["reference"] = None
if configs["baseline"]:
kwargs["baseline_answer"] = model_answers[configs["baseline_model"]][question_id]
else:
kwargs["baseline_answer"] = None
kwargs["configs"] = configs
kwargs["endpoint_dict"] = endpoint_info
kwargs["output_file"] = output_files[model]
kwargs["regex_pattern"] = pattern
future = executor.submit(judgment, **kwargs)
futures.append(future)
if count > 0:
print(f"{count} number of existing judgments")
for future in tqdm(
concurrent.futures.as_completed(futures), total=len(futures)
):
future.result()