File size: 3,582 Bytes
8bedda3 c3df5b3 8bedda3 c3df5b3 8bedda3 c3df5b3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space."""
import argparse
import pickle
import gradio as gr
notebook_url = "https://colab.research.google.com/drive/1iI_IszGAwSMkdfUrIDI6NfTG7tGDDRxZ?usp=sharing"
def make_leaderboard_md(elo_results):
leaderboard_md = f"""
# Leaderboard
[[Blog](https://lmsys.org/blog/2023-05-03-arena/)] [[Vote](https://arena.lmsys.org/)] [[Github]](https://github.com/lm-sys/FastChat) [[Twitter]](https://twitter.com/lmsysorg) [[Discord]](https://discord.gg/h6kCZb72G7)
We use the Elo rating system to calculate the relative performance of the models. You can view the voting data, basic analyses, and calculation procedure in this [notebook]({notebook_url}). We will periodically release new leaderboards. If you want to see more models, please help us [add them](https://github.com/lm-sys/FastChat/blob/main/docs/arena.md#how-to-add-a-new-model).
Last updated: {elo_results["last_updated_datetime"]}
{elo_results["leaderboard_table"]}
"""
return leaderboard_md
def build_leaderboard_tab(elo_results_file):
if elo_results_file is not None:
with open(elo_results_file, "rb") as fin:
elo_results = pickle.load(fin)
md = make_leaderboard_md(elo_results)
p1 = elo_results["win_fraction_heatmap"]
p2 = elo_results["battle_count_heatmap"]
p3 = elo_results["average_win_rate_bar"]
p4 = elo_results["bootstrap_elo_rating"]
else:
md = "Loading ..."
p1 = p2 = p3 = p4 = None
md_1 = gr.Markdown(md)
gr.Markdown(
f"""## More Statistics\n
Here, we have added some additional figures to show more statistics. The code for generating them is also included in this [notebook]({notebook_url}).
Please note that you may see different orders from different ranking methods. This is expected for models that perform similarly, as demonstrated by the confidence interval in the bootstrap figure. Going forward, we prefer the classical Elo calculation because of its scalability and interpretability. You can find more discussions in this blog [post](https://lmsys.org/blog/2023-05-03-arena/).
"""
)
with gr.Row():
with gr.Column():
gr.Markdown(
"#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles"
)
plot_1 = gr.Plot(p1, show_label=False)
with gr.Column():
gr.Markdown(
"#### Figure 2: Battle Count for Each Combination of Models (without Ties)"
)
plot_2 = gr.Plot(p2, show_label=False)
with gr.Row():
with gr.Column():
gr.Markdown(
"#### Figure 3: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)"
)
plot_3 = gr.Plot(p3, show_label=False)
with gr.Column():
gr.Markdown(
"#### Figure 4: Bootstrap of Elo Estimates (1000 Rounds of Random Sampling)"
)
plot_4 = gr.Plot(p4, show_label=False)
return [md_1, plot_1, plot_2, plot_3, plot_4]
def build_demo(elo_results_file):
with gr.Blocks(
title="Chatbot Arena Leaderboard",
theme=gr.themes.Base(),
) as demo:
leader_components = build_leaderboard_tab(elo_results_file)
return demo
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--share", action="store_true")
args = parser.parse_args()
demo = build_demo("elo_results_20230508.pkl")
demo.launch(share=args.share)
|