Spaces:
Running
on
Zero
Running
on
Zero
File size: 2,645 Bytes
e368cec 2b29775 e368cec |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import fire
import json
import pandas as pd
import pickle
def main(
model_info_file: str,
elo_rating_pkl: str,
output_csv: str
):
model_info = json.load(open(model_info_file))
with open(elo_rating_pkl, "rb") as fin:
elo_rating_results = pickle.load(fin)
anony_elo_rating_results = elo_rating_results["anony"]
full_elo_rating_results = elo_rating_results["full"]
anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
# Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
# set Organization and license to empty for now
all_models = anony_leaderboard_data.index.tolist()
for model in all_models:
if not model in model_info:
model_info[model] = {}
model_info[model]["License"] = "N/A"
model_info[model]["Organization"] = "N/A"
model_info[model]["Link"] = "N/A"
print(f"Model {model} not found in model_info.json")
model_info[model]["Model"] = model
model_info[model]["key"] = model
if model in anony_leaderboard_data.index:
model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
else:
model_info[model]["Arena Elo rating (anony)"] = 0
if model in full_elo_rating_results["leaderboard_table_df"].index:
model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
else:
model_info[model]["Arena Elo rating (full)"] = 0
# if model in anony_leaderboard_data.index:
# model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
# else:
# model_info[model]["Arena Elo rating"] = 0
final_model_info = {}
for model in model_info:
if "Model" in model_info[model]:
final_model_info[model] = model_info[model]
model_info = final_model_info
exclude_keys = ['starting_from']
for key in exclude_keys:
for model in model_info:
if key in model_info[model]:
del model_info[model][key]
df = pd.DataFrame(model_info).T
df = df[fields]
# sort by anony rating
df = df.sort_values(by=["Arena Elo rating (anony)"], ascending=False)
df.to_csv(output_csv, index=False)
print("Leaderboard data saved to", output_csv)
print(df)
if __name__ == "__main__":
fire.Fire(main) |