leaderboard / cli /analysis-cli.py
pminervini's picture
update
e75cd03
#!/usr/bin/env python3
import os
import sys
import json
import pickle
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.cluster.hierarchy import linkage
from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
from src.envs import QUEUE_REPO, RESULTS_REPO, API
from src.utils import my_snapshot_download
def is_float(string):
try:
float(string)
return True
except ValueError:
return False
def find_json_files(json_path):
res = []
for root, dirs, files in os.walk(json_path):
for file in files:
if file.endswith(".json"):
res.append(os.path.join(root, file))
return res
def sanitise_metric(name: str) -> str:
res = name
res = res.replace("prompt_level_strict_acc", "Prompt-Level Accuracy")
res = res.replace("acc", "Accuracy")
res = res.replace("exact_match", "EM")
res = res.replace("avg-selfcheckgpt", "AVG")
res = res.replace("max-selfcheckgpt", "MAX")
res = res.replace("rouge", "ROUGE-")
res = res.replace("bertscore_precision", "BERT-P")
res = res.replace("exact", "EM")
res = res.replace("HasAns_EM", "HasAns")
res = res.replace("NoAns_EM", "NoAns")
res = res.replace("em", "EM")
return res
def sanitise_dataset(name: str) -> str:
res = name
res = res.replace("tqa8", "TriviaQA (8-shot)")
res = res.replace("nq8", "NQ (8-shot)")
res = res.replace("nq_open", "NQ (64-shot)")
res = res.replace("triviaqa", "TriviaQA (64-shot)")
res = res.replace("truthfulqa", "TruthfulQA")
res = res.replace("ifeval", "IFEval")
res = res.replace("selfcheckgpt", "SelfCheckGPT")
res = res.replace("truefalse_cieacf", "True-False")
res = res.replace("mc", "MC")
res = res.replace("race", "RACE")
res = res.replace("squad", "SQuAD")
res = res.replace("memo-trap", "MemoTrap")
res = res.replace("cnndm", "CNN/DM")
res = res.replace("xsum", "XSum")
res = res.replace("qa", "QA")
res = res.replace("summarization", "Summarization")
res = res.replace("dialogue", "Dialog")
res = res.replace("halueval", "HaluEval")
res = res.replace("_v2", "")
res = res.replace("_", " ")
return res
cache_file = 'data_map_cache.pkl'
def load_data_map_from_cache(cache_file):
if os.path.exists(cache_file):
with open(cache_file, 'rb') as f:
return pickle.load(f)
else:
return None
def save_data_map_to_cache(data_map, cache_file):
with open(cache_file, 'wb') as f:
pickle.dump(data_map, f)
# Try to load the data_map from the cache file
data_map = load_data_map_from_cache(cache_file)
if data_map is None:
my_snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
result_path_lst = find_json_files(EVAL_RESULTS_PATH_BACKEND)
request_path_lst = find_json_files(EVAL_REQUESTS_PATH_BACKEND)
model_name_to_model_map = {}
for path in request_path_lst:
with open(path, 'r') as f:
data = json.load(f)
model_name_to_model_map[data["model"]] = data
model_dataset_metric_to_result_map = {}
# data_map[model_name][(dataset_name, sanitised_metric_name)] = value
data_map = {}
for path in result_path_lst:
with open(path, 'r') as f:
data = json.load(f)
model_name = data["config"]["model_name"]
for dataset_name, results_dict in data["results"].items():
for metric_name, value in results_dict.items():
if model_name_to_model_map[model_name]["likes"] > 128:
to_add = True
if 'f1' in metric_name:
to_add = False
if 'stderr' in metric_name:
to_add = False
if 'memo-trap_v2' in dataset_name:
to_add = False
if 'faithdial' in dataset_name:
to_add = False
if 'truthfulqa_gen' in dataset_name:
to_add = False
if 'bertscore' in metric_name:
if 'precision' not in metric_name:
to_add = False
if 'halueval' in dataset_name:
if 'acc' not in metric_name:
to_add = False
if 'ifeval' in dataset_name:
if 'prompt_level_strict_acc' not in metric_name:
to_add = False
if 'squad' in dataset_name:
# to_add = False
if 'best_exact' in metric_name:
to_add = False
if 'fever' in dataset_name:
to_add = False
if ('xsum' in dataset_name or 'cnn' in dataset_name) and 'v2' not in dataset_name:
to_add = False
if isinstance(value, str):
if is_float(value):
value = float(value)
else:
to_add = False
if to_add:
if 'rouge' in metric_name:
value /= 100.0
if 'squad' in dataset_name:
value /= 100.0
sanitised_metric_name = metric_name
if "," in sanitised_metric_name:
sanitised_metric_name = sanitised_metric_name.split(',')[0]
sanitised_metric_name = sanitise_metric(sanitised_metric_name)
sanitised_dataset_name = sanitise_dataset(dataset_name)
model_dataset_metric_to_result_map[(model_name, sanitised_dataset_name, sanitised_metric_name)] = value
if model_name not in data_map:
data_map[model_name] = {}
data_map[model_name][(sanitised_dataset_name, sanitised_metric_name)] = value
print('model_name', model_name, 'dataset_name', sanitised_dataset_name, 'metric_name', sanitised_metric_name, 'value', value)
save_data_map_to_cache(data_map, cache_file)
model_name_lst = [m for m in data_map.keys()]
nb_max_metrics = max(len(data_map[model_name]) for model_name in model_name_lst)
for model_name in model_name_lst:
if len(data_map[model_name]) < nb_max_metrics - 5:
del data_map[model_name]
plot_type_lst = ['all', 'summ', 'qa', 'instr', 'detect', 'rc']
for plot_type in plot_type_lst:
data_map_v2 = {}
for model_name in data_map.keys():
for dataset_metric in data_map[model_name].keys():
if dataset_metric not in data_map_v2:
data_map_v2[dataset_metric] = {}
if plot_type in {'all'}:
to_add = True
if 'ROUGE' in dataset_metric[1] and 'ROUGE-L' not in dataset_metric[1]:
to_add = False
if 'SQuAD' in dataset_metric[0] and 'EM' not in dataset_metric[1]:
to_add = False
if 'SelfCheckGPT' in dataset_metric[0] and 'MAX' not in dataset_metric[1]:
to_add = False
if '64-shot' in dataset_metric[0]:
to_add = False
if to_add is True:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
elif plot_type in {'summ'}:
if 'CNN' in dataset_metric[0] or 'XSum' in dataset_metric[0]:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
elif plot_type in {'qa'}:
if 'TriviaQA' in dataset_metric[0] or 'NQ' in dataset_metric[0] or 'TruthfulQA' in dataset_metric[0]:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
elif plot_type in {'instr'}:
if 'MemoTrap' in dataset_metric[0] or 'IFEval' in dataset_metric[0]:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
elif plot_type in {'detect'}:
if 'HaluEval' in dataset_metric[0] or 'SelfCheck' in dataset_metric[0]:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
elif plot_type in {'rc'}:
if 'RACE' in dataset_metric[0] or 'SQuAD' in dataset_metric[0]:
data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
else:
assert False, f"Unknown plot type: {plot_type}"
# df = pd.DataFrame.from_dict(data_map, orient='index') # Invert the y-axis (rows)
df = pd.DataFrame.from_dict(data_map_v2, orient='index') # Invert the y-axis (rows)
df.index = [', '.join(map(str, idx)) for idx in df.index]
o_df = df.copy(deep=True)
# breakpoint()
print(df)
# Check for NaN or infinite values and replace them
df.replace([np.inf, -np.inf], np.nan, inplace=True) # Replace infinities with NaN
df.fillna(0, inplace=True) # Replace NaN with 0 (or use another imputation strategy)
from sklearn.preprocessing import MinMaxScaler
# scaler = MinMaxScaler()
# df = pd.DataFrame(scaler.fit_transform(df), index=df.index, columns=df.columns)
# Calculate dimensions based on the DataFrame size
cell_height = 1.0 # Height of each cell in inches
cell_width = 1.0 # Width of each cell in inches
n_rows = len(df.index) # Datasets and Metrics
n_cols = len(df.columns) # Models
# Calculate figure size dynamically
fig_width = cell_width * n_cols + 0
fig_height = cell_height * n_rows + 0
col_cluster = True
row_cluster = True
sns.set_context("notebook", font_scale=1.3)
dendrogram_ratio = (.1, .1)
if plot_type in {'detect'}:
fig_width = cell_width * n_cols - 2
fig_height = cell_height * n_rows + 5.2
dendrogram_ratio = (.1, .2)
if plot_type in {'instr'}:
fig_width = cell_width * n_cols - 2
fig_height = cell_height * n_rows + 5.2
dendrogram_ratio = (.1, .4)
if plot_type in {'qa'}:
fig_width = cell_width * n_cols - 2
fig_height = cell_height * n_rows + 4
dendrogram_ratio = (.1, .2)
if plot_type in {'summ'}:
fig_width = cell_width * n_cols - 2
fig_height = cell_height * n_rows + 2.0
dendrogram_ratio = (.1, .1)
row_cluster = False
if plot_type in {'rc'}:
fig_width = cell_width * n_cols - 2
fig_height = cell_height * n_rows + 5.2
dendrogram_ratio = (.1, .4)
print('figsize', (fig_width, fig_height))
o_df.to_json(f'plots/clustermap_{plot_type}.json', orient='split')
print(f'Generating the clustermaps for {plot_type}')
for cmap in [None, 'coolwarm', 'viridis']:
fig = sns.clustermap(df,
method='ward',
metric='euclidean',
cmap=cmap,
figsize=(fig_width, fig_height), # figsize=(24, 16),
annot=True,
mask=o_df.isnull(),
dendrogram_ratio=dendrogram_ratio,
fmt='.2f',
col_cluster=col_cluster,
row_cluster=row_cluster)
# Adjust the size of the cells (less wide)
plt.setp(fig.ax_heatmap.get_yticklabels(), rotation=0)
plt.setp(fig.ax_heatmap.get_xticklabels(), rotation=90)
cmap_suffix = '' if cmap is None else f'_{cmap}'
# Save the clustermap to file
fig.savefig(f'blog/figures/clustermap_{plot_type}{cmap_suffix}.pdf')
fig.savefig(f'blog/figures/clustermap_{plot_type}{cmap_suffix}.png')
fig.savefig(f'blog/figures/clustermap_{plot_type}{cmap_suffix}_t.png', transparent=True, facecolor="none")