Spaces:
Sleeping
Sleeping
File size: 11,965 Bytes
1cc2077 25f445b b90013e dc9c8a6 4826928 df66c51 1cc2077 3e6bf0e a2e6203 1cc2077 d10decd 1cc2077 d10decd e4eaeef 1cc2077 b20cd7e 1cc2077 b90013e 1cc2077 524ef7e 10afd07 e2f9781 10afd07 b20cd7e 10afd07 b20cd7e e2f9781 b20cd7e 1fee397 0b73795 1fee397 0b73795 1fee397 524ef7e 1fee397 524ef7e e4eaeef 524ef7e 1fee397 0b73795 524ef7e 0b73795 1fee397 0b73795 1fee397 0b73795 1fee397 0b73795 1fee397 0b73795 1fee397 524ef7e 1fee397 0b73795 524ef7e 1fee397 524ef7e 1fee397 0b73795 524ef7e 0b73795 524ef7e 1cc2077 b90013e 1cc2077 b90013e 1cc2077 c806fef 1cc2077 1fee397 1cc2077 9063698 53feeb3 1cc2077 b90013e 1cc2077 c806fef 1cc2077 b20cd7e 1cc2077 b90013e 1cc2077 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 |
__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
import gradio as gr
import pandas as pd
import re
import os
import json
import yaml
import matplotlib.pyplot as plt
import seaborn as sns
import plotnine as p9
import sys
sys.path.append('./src')
sys.path.append('.')
from src.about import *
from src.saving_utils import *
from src.vis_utils import *
from src.bin.PROBE import run_probe
def add_new_eval(
human_file,
skempi_file,
model_name_textbox: str,
revision_name_textbox: str,
benchmark_type,
similarity_tasks,
function_prediction_aspect,
function_prediction_dataset,
family_prediction_dataset,
):
representation_name = model_name_textbox if revision_name_textbox == '' else revision_name_textbox
results = run_probe(benchmark_type, representation_name, human_file, skempi_file, similarity_tasks, function_prediction_aspect, function_prediction_dataset, family_prediction_dataset)
for benchmark_type in results:
if benchmark_type == 'similarity':
save_similarity_output(results['similarity'], representation_name)
elif benchmark_type == 'function':
save_function_output(results['function'], representation_name)
elif benchmark_type == 'family':
save_family_output(results['family'], representation_name)
elif benchmark_type == "affinity":
save_affinity_output(results['affinity', representation_name])
# Function to update leaderboard dynamically based on user selection
def update_leaderboard(selected_methods, selected_metrics):
return get_baseline_df(selected_methods, selected_metrics)
block = gr.Blocks()
with block:
gr.Markdown(LEADERBOARD_INTRODUCTION)
with gr.Tabs(elem_classes="tab-buttons") as tabs:
# table jmmmu bench
with gr.TabItem("🏅 PROBE Leaderboard", elem_id="probe-benchmark-tab-table", id=1):
method_names = pd.read_csv(CSV_RESULT_PATH)['method_name'].unique().tolist()
metric_names = pd.read_csv(CSV_RESULT_PATH).columns.tolist()
metrics_with_method = metric_names.copy()
metric_names.remove('method_name') # Remove method_name from the metric options
# Leaderboard section with method and metric selectors
with gr.Row():
# Add method and metric selectors for leaderboard
leaderboard_method_selector = gr.CheckboxGroup(
choices=method_names, label="Select method_names for Leaderboard", value=method_names, interactive=True
)
leaderboard_metric_selector = gr.CheckboxGroup(
choices=metric_names, label="Select Metrics for Leaderboard", value=metric_names, interactive=True
)
# Display the filtered leaderboard
baseline_value = get_baseline_df(method_names, metric_names)
baseline_header = ["method_name"] + metric_names
baseline_datatype = ['markdown'] + ['number'] * len(metric_names)
data_component = gr.components.Dataframe(
value=baseline_value,
headers=baseline_header,
type="pandas",
datatype=baseline_datatype,
interactive=False,
visible=True,
)
# Update leaderboard when method/metric selection changes
leaderboard_method_selector.change(
update_leaderboard,
inputs=[leaderboard_method_selector, leaderboard_metric_selector],
outputs=data_component
)
leaderboard_metric_selector.change(
update_leaderboard,
inputs=[leaderboard_method_selector, leaderboard_metric_selector],
outputs=data_component
)
# Dynamic selectors
x_metric_selector = gr.Dropdown(choices=[], label="Select X-axis Metric", visible=False)
y_metric_selector = gr.Dropdown(choices=[], label="Select Y-axis Metric", visible=False)
aspect_type_selector = gr.Dropdown(choices=[], label="Select Aspect Type", visible=False)
dataset_type_selector = gr.Dropdown(choices=[], label="Select Dataset Type", visible=False)
dataset_selector = gr.Dropdown(choices=[], label="Select Dataset", visible=False)
single_metric_selector = gr.Dropdown(choices=[], label="Select Metric", visible=False)
# CheckboxGroup for methods
method_selector = gr.CheckboxGroup(choices=method_names, label="Select methods to visualize", interactive=True, value=method_names)
# Button to draw the plot for the selected benchmark
plot_button = gr.Button("Plot")
plot_output = gr.Image(label="Plot")
# Update metric selectors based on benchmark type
def update_metric_choices(benchmark_type):
if benchmark_type == 'similarity':
# Show x and y metric selectors for similarity
metric_names = benchmark_specific_metrics.get(benchmark_type, [])
return (
gr.update(choices=metric_names, value=metric_names[0], visible=True),
gr.update(choices=metric_names, value=metric_names[1], visible=True),
gr.update(visible=False), gr.update(visible=False),
gr.update(visible=False), gr.update(visible=False)
)
elif benchmark_type == 'function':
# Show aspect and dataset type selectors for function
aspect_types = benchmark_specific_metrics[benchmark_type]['aspect_types']
dataset_types = benchmark_specific_metrics[benchmark_type]['dataset_types']
return (
gr.update(visible=False), gr.update(visible=False),
gr.update(choices=aspect_types, value=aspect_types[0], visible=True),
gr.update(choices=dataset_types, value=dataset_types[0], visible=True),
gr.update(visible=False), gr.update(visible=False)
)
elif benchmark_type == 'family':
# Show dataset and metric selectors for family
datasets = benchmark_specific_metrics[benchmark_type]['datasets']
metrics = benchmark_specific_metrics[benchmark_type]['metrics']
return (
gr.update(visible=False), gr.update(visible=False),
gr.update(visible=False), gr.update(visible=False),
gr.update(choices=datasets, value=datasets[0], visible=True),
gr.update(choices=metrics, value=metrics[0], visible=True)
)
elif benchmark_type == 'affinity':
# Show single metric selector for affinity
metrics = benchmark_specific_metrics[benchmark_type]
return (
gr.update(visible=False), gr.update(visible=False),
gr.update(visible=False), gr.update(visible=False),
gr.update(visible=False), gr.update(choices=metrics, value=metrics[0], visible=True)
)
return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
# Dropdown for benchmark type
benchmark_type_selector = gr.Dropdown(choices=list(benchmark_specific_metrics.keys()), label="Select Benchmark Type")
# Update selectors when benchmark type changes
benchmark_type_selector.change(
update_metric_choices,
inputs=[benchmark_type_selector],
outputs=[x_metric_selector, y_metric_selector, aspect_type_selector, dataset_type_selector, dataset_selector, single_metric_selector]
)
plot_button.click(
benchmark_plot,
inputs=[benchmark_type_selector, method_selector, x_metric_selector, y_metric_selector, aspect_type_selector, dataset_type_selector, dataset_selector, single_metric_selector],
outputs=plot_output
)
with gr.TabItem("📝 About", elem_id="probe-benchmark-tab-table", id=2):
with gr.Row():
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
with gr.TabItem("🚀 Submit here! ", elem_id="probe-benchmark-tab-table", id=3):
with gr.Row():
gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
with gr.Row():
gr.Markdown("# ✉️✨ Submit your model's representation files here!", elem_classes="markdown-text")
with gr.Row():
with gr.Column():
model_name_textbox = gr.Textbox(
label="Model name",
)
revision_name_textbox = gr.Textbox(
label="Revision Model Name",
)
benchmark_type = gr.CheckboxGroup(
choices=TASK_INFO,
label="Benchmark Type",
interactive=True,
)
similarity_tasks = gr.CheckboxGroup(
choices=similarity_tasks_options,
label="Select Similarity Tasks",
interactive=True,
)
function_prediction_aspect = gr.Radio(
choices=function_prediction_aspect_options,
label="Select Function Prediction Aspect",
interactive=True,
)
family_prediction_dataset = gr.CheckboxGroup(
choices=family_prediction_dataset_options,
label="Select Family Prediction Dataset",
interactive=True,
)
function_prediction_dataset = "All_Data_Sets"
with gr.Column():
human_file = gr.components.File(label="Click to Upload the representation file (csv) for Human dataset", file_count="single", type='filepath')
skempi_file = gr.components.File(label="Click to Upload the representation file (csv) for SKEMPI dataset", file_count="single", type='filepath')
submit_button = gr.Button("Submit Eval")
submission_result = gr.Markdown()
submit_button.click(
add_new_eval,
inputs=[
human_file,
skempi_file,
model_name_textbox,
revision_name_textbox,
benchmark_type,
similarity_tasks,
function_prediction_aspect,
function_prediction_dataset,
family_prediction_dataset,
],
)
def refresh_data():
value = get_baseline_df(method_names, metric_names)
return value
with gr.Row():
data_run = gr.Button("Refresh")
data_run.click(refresh_data, outputs=[data_component])
with gr.Accordion("Citation", open=False):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
elem_id="citation-button",
show_copy_button=True,
)
block.launch()
|