File size: 13,118 Bytes
1cc2077
 
 
 
 
 
 
25f445b
b90013e
dc9c8a6
4826928
1cc2077
 
a2e6203
1cc2077
 
 
4826928
 
524ef7e
4826928
 
 
 
a176a73
 
8bddae0
a176a73
4826928
 
a176a73
4826928
 
 
 
 
 
 
 
 
a176a73
 
 
b8fb55c
1224c55
 
8964eb3
a176a73
 
96b47ff
a176a73
 
 
4826928
 
a176a73
 
 
 
 
9a3ecc1
a176a73
 
524ef7e
 
4826928
524ef7e
 
4826928
8964eb3
0c329c2
524ef7e
 
 
 
 
 
 
 
b20cd7e
1cc2077
10afd07
 
1cc2077
 
524ef7e
b90013e
ec6bef2
dc9c8a6
 
 
 
 
 
 
ec6bef2
dc9c8a6
 
b90013e
dc9c8a6
b90013e
 
 
 
 
 
 
 
 
 
 
53feeb3
1cc2077
 
 
 
 
d10decd
 
 
 
 
1cc2077
 
d10decd
1cc2077
 
b20cd7e
 
 
 
1cc2077
 
 
b90013e
 
1cc2077
 
524ef7e
10afd07
 
 
e2f9781
10afd07
 
b20cd7e
 
 
 
10afd07
b20cd7e
 
 
 
 
 
 
e2f9781
b20cd7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
524ef7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
013f253
524ef7e
 
 
 
 
 
 
363f92a
524ef7e
 
 
 
 
 
 
1cc2077
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b90013e
1cc2077
 
 
b90013e
1cc2077
 
 
 
 
c806fef
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1cc2077
 
9063698
53feeb3
1cc2077
 
 
 
 
b90013e
1cc2077
 
 
 
c806fef
 
 
 
 
1cc2077
 
 
 
b20cd7e
1cc2077
 
 
 
b90013e
1cc2077
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']

import gradio as gr
import pandas as pd
import re
import os
import json
import yaml
import matplotlib.pyplot as plt
import seaborn as sns
import plotnine as p9

from src.about import *
from src.bin.PROBE import run_probe

global data_component, filter_component

def get_method_color(method):
    return color_dict.get(method, 'black')  # If method is not in color_dict, use black

def draw_scatter_plot_similarity(methods_selected, x_metric, y_metric, title):
    df = pd.read_csv(CSV_RESULT_PATH)
    # Filter the dataframe based on selected methods
    filtered_df = df[df['method_name'].isin(methods_selected)]

    def get_method_color(method):
        return color_dict.get(method.upper(), 'black')

    # Add a new column to the dataframe for the color
    filtered_df['color'] = filtered_df['method_name'].apply(get_method_color)

    adjust_text_dict = {
        'expand_text': (1.15, 1.4), 'expand_points': (1.15, 1.25), 'expand_objects': (1.05, 1.5),
        'expand_align': (1.05, 1.2), 'autoalign': 'xy', 'va': 'center', 'ha': 'center',
        'force_text': (.0, 1.), 'force_objects': (.0, 1.),
        'lim': 500000, 'precision': 1., 'avoid_points': True, 'avoid_text': True
    }

    # Create the scatter plot using plotnine (ggplot)
    g = (p9.ggplot(data=filtered_df,
                   mapping=p9.aes(x=x_metric,  # Use the selected x_metric
                                  y=y_metric,  # Use the selected y_metric
                                  color='color',  # Use the dynamically generated color
                                  label='method_names'))  # Label each point by the method name
         + p9.geom_point(size=3)  # Add points with no jitter, set point size
         + p9.geom_text(nudge_y=0.02, size=8)  # Add method names as labels, nudge slightly above the points
         + p9.labs(title=title, x=f"{x_metric}", y=f"{y_metric}")  # Dynamic labels for X and Y axes
         + p9.scale_color_identity()  # Use colors directly from the dataframe
         + p9.theme(legend_position='none', 
                    figure_size=(8, 8),  # Set figure size
                    axis_text=p9.element_text(size=10),   
                    axis_title_x=p9.element_text(size=12),
                    axis_title_y=p9.element_text(size=12))
    )

    # Save the plot as an image
    save_path = "./plot_images"  # Ensure this folder exists or adjust the path
    os.makedirs(save_path, exist_ok=True)  # Create directory if it doesn't exist
    filename = os.path.join(save_path, title.replace(" ", "_") + "_Similarity_Scatter.png")
    
    g.save(filename=filename, dpi=400)
    
    return filename

def benchmark_plot(benchmark_type, methods_selected, x_metric, y_metric):
    if benchmark_type == 'flexible':
        # Use general visualizer logic
        return general_visualizer_plot(methods_selected, x_metric=x_metric, y_metric=y_metric)
    elif benchmark_type == 'similarity':
        title = f"{x_metric} vs {y_metric}"
        return draw_scatter_plot_similarity(methods_selected, x_metric, y_metric, title)
    elif benchmark_type == 'Benchmark 3':
        return benchmark_3_plot(x_metric, y_metric)
    elif benchmark_type == 'Benchmark 4':
        return benchmark_4_plot(x_metric, y_metric)
    else:
        return "Invalid benchmark type selected."


def get_baseline_df(selected_methods, selected_metrics):
    df = pd.read_csv(CSV_RESULT_PATH)
    present_columns = ["method_name"] + selected_metrics
    df = df[df['method_name'].isin(selected_methods)][present_columns]
    return df

def general_visualizer(methods_selected, x_metric, y_metric):
    df = pd.read_csv(CSV_RESULT_PATH)
    filtered_df = df[df['method_name'].isin(methods_selected)]

    # Create a Seaborn lineplot with method as hue
    plt.figure(figsize=(10, 8))  # Increase figure size
    sns.lineplot(
        data=filtered_df, 
        x=x_metric, 
        y=y_metric, 
        hue="method_name",  # Different colors for different methods
        marker="o",  # Add markers to the line plot
    )
    
    # Add labels and title
    plt.xlabel(x_metric)
    plt.ylabel(y_metric)
    plt.title(f'{y_metric} vs {x_metric} for selected methods')
    plt.grid(True)
    
    # Save the plot to display it in Gradio
    plot_path = "plot.png"
    plt.savefig(plot_path)
    plt.close()
    
    return plot_path

def add_new_eval(
    human_file,
    skempi_file,
    model_name_textbox: str,
    revision_name_textbox: str,
    benchmark_type,
    similarity_tasks,
    function_prediction_aspect,
    function_prediction_dataset,
    family_prediction_dataset,
):
    representation_name = model_name_textbox if revision_name_textbox == '' else revision_name_textbox
    results = run_probe(benchmark_type, representation_name, human_file, skempi_file, similarity_tasks, function_prediction_aspect, function_prediction_dataset, family_prediction_dataset)
    return None

# Function to update leaderboard dynamically based on user selection
def update_leaderboard(selected_methods, selected_metrics):
    return get_baseline_df(selected_methods, selected_metrics)

block = gr.Blocks()

with block:
    gr.Markdown(LEADERBOARD_INTRODUCTION)
    
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        # table jmmmu bench
        with gr.TabItem("🏅 PROBE Leaderboard", elem_id="probe-benchmark-tab-table", id=1):

            method_names = pd.read_csv(CSV_RESULT_PATH)['method_name'].unique().tolist()
            metric_names = pd.read_csv(CSV_RESULT_PATH).columns.tolist()
            metrics_with_method = metric_names.copy()
            metric_names.remove('method_name')  # Remove method_name from the metric options
        
            # Leaderboard section with method and metric selectors
            with gr.Row():
                # Add method and metric selectors for leaderboard
                leaderboard_method_selector = gr.CheckboxGroup(
                    choices=method_names, label="Select method_names for Leaderboard", value=method_names, interactive=True
                )
                leaderboard_metric_selector = gr.CheckboxGroup(
                    choices=metric_names, label="Select Metrics for Leaderboard", value=metric_names, interactive=True
                )

            # Display the filtered leaderboard
            baseline_value = get_baseline_df(method_names, metric_names)
            baseline_header = ["method_name"] + metric_names
            baseline_datatype = ['markdown'] + ['number'] * len(metric_names)

            data_component = gr.components.Dataframe(
                value=baseline_value,
                headers=baseline_header,
                type="pandas",
                datatype=baseline_datatype,
                interactive=False,
                visible=True,
            )

            # Update leaderboard when method/metric selection changes
            leaderboard_method_selector.change(
                update_leaderboard, 
                inputs=[leaderboard_method_selector, leaderboard_metric_selector], 
                outputs=data_component
            )
            leaderboard_metric_selector.change(
                update_leaderboard, 
                inputs=[leaderboard_method_selector, leaderboard_metric_selector], 
                outputs=data_component
            )
            
            # Dropdown for benchmark type
            benchmark_types = TASK_INFO + ['flexible']
            benchmark_type_selector = gr.Dropdown(choices=benchmark_types, label="Select Benchmark Type for Visualization", value="flexible")
            
            # Dynamic metric selectors (will be updated based on benchmark type)
            x_metric_selector = gr.Dropdown(choices=[], label="Select X-axis Metric")
            y_metric_selector = gr.Dropdown(choices=[], label="Select Y-axis Metric")
            method_selector = gr.CheckboxGroup(choices=method_names, label="Select methods to visualize", interactive=True, value=method_names)
            
            # Button to draw the plot for the selected benchmark
            plot_button = gr.Button("Plot Visualization")
            plot_output = gr.Image(label="Plot")

            # Update metric selectors when benchmark type is chosen
            def update_metric_choices(benchmark_type):
                if benchmark_type == 'flexible':
                    # Show all metrics for the flexible visualizer
                    metric_names = df.columns.tolist()
                    return gr.update(choices=metric_names, value=metric_names[0]), gr.update(choices=metric_names, value=metric_names[1])
                elif benchmark_type in benchmark_specific_metrics:
                    metrics = benchmark_specific_metrics[benchmark_type]
                    return gr.update(choices=metrics, value=metrics[0]), gr.update(choices=metrics)
                return gr.update(choices=[]), gr.update(choices=[])

            benchmark_type_selector.change(
                update_metric_choices, 
                inputs=[benchmark_type_selector], 
                outputs=[x_metric_selector, y_metric_selector]
            )

            # Generate the plot based on user input
            plot_button.click(
                benchmark_plot, 
                inputs=[benchmark_type_selector, method_selector, x_metric_selector, y_metric_selector], 
                outputs=plot_output
            )
            
        with gr.TabItem("📝 About", elem_id="probe-benchmark-tab-table", id=2):
            with gr.Row():
                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

        with gr.TabItem("🚀 Submit here! ", elem_id="probe-benchmark-tab-table", id=3):
            with gr.Row():
                gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

            with gr.Row():
                gr.Markdown("# ✉️✨ Submit your model's representation files here!", elem_classes="markdown-text")

            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(
                        label="Model name",
                    )
                    revision_name_textbox = gr.Textbox(
                        label="Revision Model Name",
                    )
                    
                    benchmark_type = gr.CheckboxGroup(
                        choices=TASK_INFO,
                        label="Benchmark Type",
                        interactive=True,
                    )
                    similarity_tasks = gr.CheckboxGroup(
                        choices=similarity_tasks_options,
                        label="Select Similarity Tasks",
                        interactive=True,
                    )
                
                    function_prediction_aspect = gr.Radio(
                        choices=function_prediction_aspect_options,
                        label="Select Function Prediction Aspect",
                        interactive=True,
                    )
                
                    function_prediction_dataset = gr.Radio(
                        choices=function_prediction_dataset_options,
                        label="Select Function Prediction Dataset",
                        interactive=True,
                    )
                
                    family_prediction_dataset = gr.CheckboxGroup(
                        choices=family_prediction_dataset_options,
                        label="Select Family Prediction Dataset",
                        interactive=True,
                    )

            with gr.Column():
                human_file = gr.components.File(label="Click to Upload the representation file (csv) for Human dataset", file_count="single", type='filepath')
                skempi_file = gr.components.File(label="Click to Upload the representation file (csv) for SKEMPI dataset", file_count="single", type='filepath')
    
                submit_button = gr.Button("Submit Eval")
                submission_result = gr.Markdown()
                submit_button.click(
                    add_new_eval,
                    inputs=[
                        human_file,
                        skempi_file,
                        model_name_textbox,
                        revision_name_textbox,
                        benchmark_type,
                        similarity_tasks,
                        function_prediction_aspect,
                        function_prediction_dataset,
                        family_prediction_dataset,
                    ],
                )

    def refresh_data():
        value = get_baseline_df(method_names, metric_names)
        return value

    with gr.Row():
        data_run = gr.Button("Refresh")
        data_run.click(refresh_data, outputs=[data_component])

    with gr.Accordion("Citation", open=False):
        citation_button = gr.Textbox(
            value=CITATION_BUTTON_TEXT,
            label=CITATION_BUTTON_LABEL,
            elem_id="citation-button",
            show_copy_button=True,
        )

block.launch()