# gradio display leaderboard import pandas as pd import numpy as np import matplotlib # matplotlib.use('macosx') import gradio as gr import matplotlib.pyplot as plt import plotly.graph_objects as go from apscheduler.schedulers.background import BackgroundScheduler from texts import * from leaderboards import eng_leaderboards, chi_leaderboards import toml import os from opseval_datasets import * from latex_utils import gen_latex_table config = toml.load("config.toml") def create_lang_tabs(lang, lang_cates): df_dict = {} for dataset, cates in lang_cates: dataset_dt = {} for cat in cates: leaderboard_df = pd.read_csv(f'./data_v2/{dataset}_{lang}_{cat}_gen.csv') dataset_dt[cat] = leaderboard_df df_dict[dataset] = dataset_dt return df_dict dict_lang = { 'English': create_lang_tabs('en', eng_leaderboards), 'Chinese': create_lang_tabs('zh', chi_leaderboards) } def process_mc_df(df, shot=None): # 将name列重命名为Model df = df.rename(columns={"name": "Model"}) # 将zero_naive, zero_self_con, zero_cot, zero_cot_self_con, few_naive, few_self_con, few_cot, few_cot_self_con列重新组织成MultiIndex，一层为Zeroshot, Fewshot,一层为Naive, Self-Consistency, CoT, CoT+Self-Consistency df = df.set_index("Model") # df = df.stack().unstack() df.columns = pd.MultiIndex.from_tuples([("Zeroshot", "Naive"), ("Zeroshot", "SC"), ("Zeroshot", "CoT"), ("Zeroshot", "CoT+SC"), ("Fewshot", "Naive"), ("Fewshot", "SC"), ("Fewshot", "CoT"), ("Fewshot", "CoT+SC")]) # 保留shot的列，比如如果shot=Zeroshot那么只有Zeroshot的列会被保留 if shot: df = df[shot] # 将除了Model列之外的列的value转换为数值型，失败的为NaN df = df.apply(pd.to_numeric, errors="coerce") # 保留小数点后两位 df = df.round(2) # 给每一行添加一列BestScore df["BestScore"] = df.max(axis=1) # 根据BestScore给df排序 df = df.sort_values(by="BestScore", ascending=False) # reset_index df = df.reset_index() # 对于所有空的值，填充为'/' df = df.fillna('/') return df def process_qa_df(df): # 保留小数点后四位 df = df.round(4) return df def dataframe_to_gradio(df, is_mc=True, shot=None): if is_mc: df = process_mc_df(df, shot) else: df = process_qa_df(df) headers = df.columns # types = ["str"] + ["number"] * (len(headers) - 1) return gr.components.Dataframe( value=df.values.tolist(), headers=[label for label in df.columns], # datatype=types, # max_rows=10, ) def plot_radar_chart(df, attributes): fig = go.Figure() for index, row in df.iterrows(): model = row['Model'] values = row[attributes].tolist() fig.add_trace(go.Scatterpolar( r=values, theta=attributes, fill='toself', name=model )) fig.update_layout( title="OpsEval", polar=dict( radialaxis=dict( visible=True, range=[0, 0.9] )), showlegend=True ) return fig def pop_latex_table(caption, label, dataframe): table = gen_latex_table(caption, label, dataframe) return gr.Textbox(table, label="LaTeX Table", visible=True) def generate_csv(df, filename): df.to_csv(filename, index=False) download_link = gr.File(label="Download Link", type="filepath", value=filename, visible=True) return download_link def create_lang_leader_board(lang_dict, lang, dis_lang='en'): best_scores = {} best_plot_datasets = [] for dataset, value in lang_dict.items(): for cat, df in value.items(): if cat == 'mc': processed = process_mc_df(df) bestscores = processed['BestScore'] best_scores[dataset] = bestscores best_plot_datasets.append(dataset) best_df = pd.DataFrame(best_scores) # print(best_scores) # print(best_df) # plot = plot_radar_chart(pd.DataFrame(best_scores), best_plot_datasets) # gr.Plot(plot) tab_list = [] for dataset, value in lang_dict.items(): chosen_dict = dataset_abbr_en_dict if dis_lang == "en" else dataset_abbr_zh_dict with gr.Tab(chosen_dict[dataset]) as tab: for cat, df in value.items(): if cat == 'mc': for shot in ['Zeroshot', 'Fewshot']: with gr.Tab(f'Multiple Choice Question ({shot})'): df_component = dataframe_to_gradio(df, is_mc=True, shot=shot) # 加一个latex表格导出按钮， 按一下弹出一个浮动文本窗口 # with gr.Row(): # latex_button = gr.Button("Export LaTeX Table", variant="primary") # csv_button = gr.Button("Export CSV", variant="primary") # latex_textbox = gr.Textbox(label="LaTeX Table", visible=False) # download_link = gr.File(label="Download Link", type="filepath", # visible=False) # latex_button.click(lambda: pop_latex_table( # caption=f"{chosen_dict[dataset]} Multiple Choice Question ({shot}, {lang}) Leaderboard", # label=f"tab:{dataset}_{shot}_{lang}", # dataframe=df, # ), inputs=[], outputs=[latex_textbox]) # csv_button.click(lambda: generate_csv(df, f"/tmp/opseval-{chosen_dict[dataset]}-mc-{shot}.csv"), inputs=[], outputs=[download_link]) else: with gr.Tab('Question Answering'): df_component = dataframe_to_gradio(df, is_mc=False) # df_list.append(df_component) # button = gr.Button("Export LaTeX Table", variant="primary") # latex_textbox = gr.Textbox(label="LaTeX Table", visible=False) # button.click(lambda: pop_latex_table( # caption=f"{chosen_dict[dataset]} {shot} {lang} Leaderboard", # label=f"tab:{dataset}_{shot}_{lang}", # dataframe=df, # ), inputs=[], outputs=[latex_textbox]) tab_list.append(tab) return tab_list def get_latest_modification_date(): latest = 0 for file in os.listdir(config['dataset']['dataset_dir']): if file.endswith('.csv'): mtime = os.path.getmtime(os.path.join(config['dataset']['dataset_dir'], file)) latest = max(latest, mtime) latest = pd.to_datetime(latest, unit='s') return latest.strftime("%Y-%m-%d %H:%M:%S") translation_dict = { 'zh': { 'intro': ZH_INTRODUCTION_TEXT, 'title': ZH_TITLE, 'lb_sec': f"""# 🏅 排行榜 \n 更新时间: {get_latest_modification_date()}\n""", }, 'en': { 'intro': INTRODUCTION_TEXT, 'title': TITLE, 'lb_sec': f"""# 🏅 Leaderboard \n Latest update: {get_latest_modification_date()}\n""" } } def get_language_lb(language): tab_dict = {'English': None, 'Chinese': None} for key, dict in dict_lang.items(): tab_list = create_lang_leader_board(dict, key, language) tab_dict[key] = tab_list return [*tab_dict['English'], *tab_dict['Chinese']] def switch_language(language): # gr.update(visible=True) return translation_dict[language]['title'], translation_dict[language]['intro'], translation_dict[language]['lb_sec'], *get_language_lb(language), language def get_lb_body(language='en'): tab_dict = {'English': None, 'Chinese': None} with gr.Blocks() as body: for key, dict in dict_lang.items(): with gr.Tab(key): tab_list = create_lang_leader_board(dict, key, language) tab_dict[key] = tab_list return body, tab_dict def launch_gradio(): demo = gr.Blocks() with demo: lang_state = gr.State("en") with gr.Row(): en_button = gr.Button("English", variant="primary") zh_button = gr.Button("中文", variant="primary") title = gr.HTML(TITLE) intro = gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") leaderboard_section = gr.Markdown(f"""# 🏅 Leaderboard \n Latest update: {get_latest_modification_date()}\n""", elem_classes="markdown-text") lb_body, tab_dict = get_lb_body(language=lang_state.value) tab_list = [*tab_dict['English'], *tab_dict['Chinese']] # print(tab_list) en_button.click(switch_language, inputs=[gr.State("en")], outputs=[title, intro, leaderboard_section, *tab_list, lang_state], postprocess=False) zh_button.click(switch_language, inputs=[gr.State("zh")], outputs=[title, intro, leaderboard_section, *tab_list, lang_state], postprocess=False) demo.launch() pd.set_option('display.float_format', '{:.02f}'.format) scheduler = BackgroundScheduler() scheduler.add_job(launch_gradio, 'interval', hours=1) scheduler.start() launch_gradio()