|
|
|
import gradio as gr |
|
import pandas as pd |
|
from src.utils import AutoEvalColumn, fields, make_clickable_names, plot_throughput |
|
|
|
|
|
df = pd.read_csv("data/code_eval_board.csv") |
|
|
|
submission_text= """ |
|
<h1 align="center"> |
|
How to submit new results to the leaderboard? |
|
</h1> |
|
We welcome the community to submit evaluation results of new models. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check. |
|
|
|
### 1 - Running Evaluation |
|
|
|
We wrote a detailed guide for running the evaluation on your model. You can find the it in [bigcode-evaluation-harness/leaderboard](https://github.com/bigcode/bigcode-evaluation-harness/tree/main/leaderboard). This will generate a json file summarizing the results, in addition to the raw generations and metric files. |
|
|
|
### 2- Submitting Results 🚀 |
|
|
|
To submit your results create a **Pull Request** in the community tab to add them under the [folder](https://huggingface.co/spaces/bigcode/multilingual-code-evals/tree/main/community_results) `community_results` in this repository: |
|
- Create a folder called `ORG_MODELNAME_USERNAME` for example `bigcode_starcoder_loubnabnl` |
|
- Put your json file with grouped scores from the guide, in addition generations folder and metrics folder in it. |
|
|
|
The title of the PR should be `[Community Submission] Model: org/model, Username: your_username`, replace org and model with those corresponding to the model you evaluated. |
|
""" |
|
|
|
|
|
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] |
|
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden] |
|
COLS_LITE = [ |
|
c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden |
|
] |
|
TYPES_LITE = [ |
|
c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden |
|
] |
|
|
|
|
|
def select_columns(df, columns): |
|
always_here_cols = [ |
|
AutoEvalColumn.model_type_symbol.name, |
|
AutoEvalColumn.model.name, |
|
] |
|
|
|
filtered_df = df[ |
|
always_here_cols |
|
+ [c for c in COLS if c in df.columns and c in columns] |
|
] |
|
return filtered_df |
|
|
|
|
|
def filter_items(df, leaderboard_table, query): |
|
if query == "all": |
|
return df[leaderboard_table.columns] |
|
else: |
|
query = query[0] |
|
filtered_df = df[(df["T"] == query)] |
|
return filtered_df[leaderboard_table.columns] |
|
|
|
|
|
def search_table(df, leaderboard_table, query): |
|
filtered_df = df[(df["Models"].str.contains(query, case=False))] |
|
return filtered_df[leaderboard_table.columns] |
|
|
|
|
|
df = make_clickable_names(df) |
|
|
|
|
|
demo = gr.Blocks() |
|
with demo: |
|
with gr.Row(): |
|
gr.Markdown( |
|
"""<div style="text-align: center;"><h1> ⭐ Multilingual <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Evaluation</span></h1></div>\ |
|
<br>\ |
|
<p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\ |
|
information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>""" |
|
) |
|
|
|
with gr.Tabs(elem_classes="tab-buttons") as tabs: |
|
with gr.Column(): |
|
with gr.Tabs(elem_classes="A100-tabs") as A100_tabs: |
|
with gr.TabItem("🔍 Evaluation table", id=0): |
|
with gr.Column(): |
|
shown_columns = gr.CheckboxGroup( |
|
choices=[ |
|
c |
|
for c in COLS |
|
if c |
|
not in [ |
|
AutoEvalColumn.dummy.name, |
|
AutoEvalColumn.model.name, |
|
AutoEvalColumn.model_type_symbol.name, |
|
] |
|
], |
|
value=[ |
|
c |
|
for c in COLS_LITE |
|
if c |
|
not in [ |
|
AutoEvalColumn.dummy.name, |
|
AutoEvalColumn.model.name, |
|
AutoEvalColumn.model_type_symbol.name, |
|
] |
|
], |
|
label="Select columns to show", |
|
elem_id="column-select", |
|
interactive=True, |
|
) |
|
|
|
with gr.Row(): |
|
search_bar = gr.Textbox( |
|
placeholder="🔍 Search for your model and press ENTER...", |
|
show_label=False, |
|
elem_id="search-bar", |
|
) |
|
filter_columns = gr.Radio( |
|
label="⏚ Filter model types", |
|
choices=["all", "🟢 base", "🔶 instruction-tuned"], |
|
value="all", |
|
elem_id="filter-columns", |
|
) |
|
|
|
leaderboard_df = gr.components.Dataframe( |
|
value=df[ |
|
[ |
|
AutoEvalColumn.model_type_symbol.name, |
|
AutoEvalColumn.model.name, |
|
] |
|
+ shown_columns.value |
|
], |
|
headers=[ |
|
AutoEvalColumn.model_type_symbol.name, |
|
AutoEvalColumn.model.name, |
|
] |
|
+ shown_columns.value, |
|
datatype=TYPES, |
|
elem_id="leaderboard-table", |
|
) |
|
|
|
hidden_leaderboard_df = gr.components.Dataframe( |
|
value=df, |
|
headers=COLS, |
|
datatype=["str" for _ in range(len(COLS))], |
|
visible=False, |
|
) |
|
search_bar.submit( |
|
search_table, |
|
[hidden_leaderboard_df, leaderboard_df, search_bar], |
|
leaderboard_df, |
|
) |
|
filter_columns.change( |
|
filter_items, |
|
[hidden_leaderboard_df, leaderboard_df, filter_columns], |
|
leaderboard_df, |
|
) |
|
shown_columns.change( |
|
select_columns, |
|
[hidden_leaderboard_df, shown_columns], |
|
leaderboard_df, |
|
) |
|
with gr.Row(): |
|
gr.Markdown( |
|
"""Notes: |
|
<ul> |
|
<li> We use the original code completion prompts for HumanEval for all models including those instruction tuned. Some evaluations might be using different prompts for instruction models like <a href="https://github.com/nlpxucan/WizardLM/blob/46d1ce7dbbb1f987ae5e5915c75f33b89a6a17ab/WizardCoder/src/humaneval_gen.py#L38">WizardCoder's instruction</a> or NewHope's instruction with a 1-shot example in the <a href="https://github.com/SLAM-group/newhope/blob/471f3bab7856c2ba6c6181deff9c746ec00da77b/complete.py#L59">prompt</a>. |
|
<li> Throughputs and peak memory usage are measured using <a href="https://github.com/huggingface/optimum-benchmark/tree/main">Optimum-Benchmark</a> which powers <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">Open LLM-Perf Leaderboard</a>. (0 throughput corresponds to OOM).</li> |
|
<li> All models were evaluated with the <a href="https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main">bigcode-evaluation-harness</a> with top-p=0.95, temperature=0.2, max_length_generation 512 and n_samples=50.</li> |
|
<li> HumanEval-Python, reports the pass@1 on HumanEval, the rest is from MultiPL-E benchmark.</li> |
|
<li> Average score is the average pass@1 over all languages. For Win Rate, we compute model rank for each language as <code style="white-space: nowrap; display: inline;">num_models - (rank -1)</code> and average their rankings.</li> |
|
<li> #Languages column represents the number of programming languages included during the pretraining. |
|
</ul>""" |
|
) |
|
with gr.TabItem("📊 Performance Plot", id=1): |
|
with gr.Row(): |
|
bs_1_plot = gr.components.Plot( |
|
value=plot_throughput(df, bs=1), |
|
elem_id="bs1-plot", |
|
show_label=False, |
|
) |
|
bs_50_plt = gr.components.Plot( |
|
value=plot_throughput(df, bs=50), |
|
elem_id="bs50-plot", |
|
show_label=False, |
|
) |
|
with gr.TabItem("Submit results 🚀", id=2): |
|
gr.Markdown(submission_text) |
|
|
|
|
|
demo.launch() |
|
|
