Spaces:
Runtime error
Runtime error
File size: 9,492 Bytes
0adb6ea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 |
from datasets import load_dataset, Dataset
import os
import json
from datasets import load_dataset
from datasets.utils.logging import disable_progress_bar # type: ignore
from ui_constants import column_names, all_task_types
import random
disable_progress_bar()
import math
from sotopia_space.constants import MODEL_INFO
id_to_data = None
model_len_info = None
def make_clickable_model(model_name):
global MODEL_INFO
if model_name in MODEL_INFO:
if MODEL_INFO[model_name]["hf_model_id"].startswith("http"):
link = MODEL_INFO[model_name]["hf_model_id"]
return f'🔒 <a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{MODEL_INFO[model_name]["pretty_name"]}</a>'
else:
link = f"https://huggingface.co/{MODEL_INFO[model_name]['hf_model_id']}"
return f'🔥 <a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{MODEL_INFO[model_name]["pretty_name"]}</a>'
else:
return model_name
def styled_error(error):
return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
def styled_warning(warn):
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
def styled_message(message):
return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
def estimated_win_rate(elo_a, elo_b, LP=0):
"""
Calculate the estimated win rate for player A against player B using their Elo ratings.
:param elo_a: Elo rating of player A
:param elo_b: Elo rating of player B
:return: Estimated win rate for player A
"""
exponent = (elo_b - elo_a)*(10**LP) / 400
probability_a_wins = 1 / (1 + 10 ** exponent)
return (1-probability_a_wins)*100
# Formats the columns
def formatter(x):
if type(x) is str:
x = x
else:
x = round(x, 1)
return x
def add_winrates(current_df, LP=0):
df = current_df.copy()
elo_column = "Task-Avg Elo"
# Correct way to filter the DataFrame and get the Elo rating for "gpt-4-0125-preview"
model_a_elo = df[df["Model"].str.contains("gpt-4")][elo_column].iloc[0]
# Correct way to filter the DataFrame and get the Elo rating for "gpt-3.5-turbo-0125"
model_b_elo = df[df["Model"].str.contains("gpt-3.5")][elo_column].iloc[0]
# Calculate the win rate of "gpt-4-0125-preview" against all models
df['Win% vs GPT-4'] = df[elo_column].apply(lambda x: estimated_win_rate(model_a_elo, x, LP=LP)).apply(formatter)
df['Win% vs GPT-3.5T'] = df[elo_column].apply(lambda x: estimated_win_rate(model_b_elo, x, LP=LP)).apply(formatter)
# apply the formatter for the two new columns
cols = list(df.columns)
cols.remove("# battles"); cols.append("# battles")
cols.remove("Length"); cols.append("Length")
df = df[cols]
return df
def add_winrates_tasks(current_df, ref="gpt-4", LP=0):
new_df = current_df.copy()
for t in all_task_types:
column = column_names[t]
model_a_elo = current_df[current_df["Model"].str.contains(ref)][column].iloc[0]
new_df[column] = current_df[column].apply(lambda x: estimated_win_rate(model_a_elo, x, LP=LP)).apply(formatter)
return new_df
def post_processing(df, model_len_info):
if model_len_info:
df["Length"] = df["model name "].apply(lambda x: model_len_info[x]["avg_len"])
for col in df.columns:
if col == "model name ":
df[col] = df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
else:
df[col] = df[col].apply(formatter) # For numerical values
df.rename(columns=column_names, inplace=True)
df.sort_values(by="Task-Avg Elo", inplace=True, ascending=False)
# put the "Overall Elo" and "Task-Avg Elo" column to the front
# add the length info
df = df[["Model", "Task-Avg Elo"] + [col for col in df.columns if col not in ["Model", "Task-Avg Elo"]]]
return df
def apply_length_penalty(original_df, ablation_df, length_penalty=0.2, mode='v1', LP_original_dfs=None):
"""
Temporarily disable the length penalty feature
if mode == 'v2' and LP_original_dfs is not None:
L = f"{length_penalty:.1f}"
return LP_original_dfs[L]
original_df = original_df.copy()
ablation_df = ablation_df.copy()
# replace all values in original_df with the values as z = x - y * length_penalty where y is from ablation_df at the same row and column
# except for the "Model" column and the "# battles" column
# do not assume the order of the rows are the same in both dataframes
for i, row in original_df.iterrows():
for col in original_df.columns:
if col == "Model" or col == "# battles" or col == "Length":
continue
# assert that the model names are the same in both dataframes
assert original_df.at[i, "Model"] == ablation_df[ablation_df["Model"] == row["Model"]]["Model"].values[0]
original_df[col] = original_df[col].astype(float)
if mode == "v1":
original_df.at[i, col] = original_df.at[i, col] - ablation_df[ablation_df["Model"] == row["Model"]][col].values[0] * length_penalty
elif mode == "v1.1":
diff = original_df.at[i, col] - ablation_df[ablation_df["Model"] == row["Model"]][col].values[0]
original_df.at[i, col] = original_df.at[i, col] * (1-length_penalty) + diff*length_penalty
# post_processing
original_df = post_processing(original_df, model_len_info=None)
"""
return original_df
def load_benchdata():
print("Loading sotopia data...")
bench_data = load_dataset("cmu-lti/sotopia", split="test")
return bench_data
def load_benchdata_dict():
print("Loading sotopia data....")
bench_data = load_dataset("cmu-lti/sotopia", data_files="sotopia_episodes_v1_hf.jsonl")['train']
id_to_data = {}
for item in bench_data:
id_to_data[item["session_id"]] = item
return id_to_data
def load_eval_results():
print("Loading sotopia Evaluation data...")
eval_results = load_dataset("WildEval/sotopia-Evaluation", "all", split="train")
return eval_results
def load_infer_results(model_name):
print(f"Loading sotopia Results for {model_name}...")
infer_results = load_dataset("WildEval/sotopia-Results", model_name, split="train")
return infer_results
def sample_an_eval_result(eval_results, model_list=[], tag_list=[]):
global id_to_data
eval_results = list(eval_results)
random.shuffle(eval_results)
for eval_item in eval_results:
# print(json.dumps(eval_item, indent=2))
# print(f"## Session ID: {eval_item['session_id']}")
# eval_item["eval_id"]
assignment = eval_item['assignment']
model_1, model_2 = eval_item['model_1'], eval_item['model_2']
model_A = model_1 if assignment['A'] == model_1 else model_2
model_B = model_2 if assignment['B'] == model_2 else model_1
if len(model_list) >= 2:
if model_A not in model_list or model_B not in model_list:
continue
elif len(model_list) == 1:
if model_A != model_list[0] and model_B != model_list[0]:
continue
else:
pass
if tag_list:
if set(tag_list).isdisjoint(set(eval_item['tags'])):
continue
winner = eval_item['winner']
# print(f"## Model A: {model_A} | Model B: {model_B} | Winner: {winner}")
task_type = eval_item['tags'][0] # primary task type
chat_history = eval_item['history']
last_query = eval_item['last_query']
# print(f"## Task Type: {task_type}")
# print(f"## Chat History: {chat_history}")
# print(f"## Last Query --> USER: {last_query}")
model_A_output = eval_item['model_1_output'] if model_1 == model_A else eval_item['model_2_output']
model_B_output = eval_item['model_2_output'] if model_2 == model_B else eval_item['model_1_output']
if len(model_A_output.strip()) == 0 or len(model_B_output.strip()) == 0:
continue
conversation_input = id_to_data[eval_item['session_id']]["conversation_input"]
# print(f"\n\n\n## Model A ({model_A}) Output ##\n{model_A_output}")
# print(f"\n\n\n## Model B ({model_B}) Output ##\n{model_B_output}")
# print(f"\n\n\n## Winner ##\n{winner}")
# print(f"\n\n\n## GPT-4 Judgement ##\n{eval_item['parsed_result']}")
result_dict = {
"session_id": eval_item['session_id'],
"model_A": model_A,
"model_B": model_B,
"winner": winner,
"intent": id_to_data[eval_item['session_id']]["intent"],
"task_type": task_type,
"all_tags": eval_item['tags'],
"chat_history": chat_history,
"last_query": last_query,
"conversation_input": conversation_input,
"model_A_output": model_A_output,
"model_B_output": model_B_output,
"reason": eval_item['parsed_result']["reason"],
"choice": eval_item['parsed_result']["choice"],
"checklist": id_to_data[eval_item['session_id']]["checklist"],
}
break
return result_dict
#id_to_data = load_benchdata_dict() |