Hugging Face's logo

Spaces:
stefan-it
/
hmLeaderboard
Runtime error

App Files Community
hmLeaderboard
File size: 4,487 Bytes 
b73474b
 
 
 
 
 
011d38f
b73474b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
import gradio as gr
import pandas as pd

title = """
# hmLeaderboard

![hmLeaderboard](https://huggingface.co/spaces/hmbench/hmLeaderboard/resolve/main/logo.png)
"""

description = """
## Space for tracking and ranking models on Historic NER Datasets.

At the moment the following models are supported:

* hmBERT: [Historical Multilingual Language Models for Named Entity Recognition](https://huggingface.co/hmbert).
* hmTEAMS: [Historic Multilingual TEAMS Models](https://huggingface.co/hmteams).
"""
footer = "Made from Bavarian Oberland with ❤️ and 🥨."

model_selection_file_names = {
    "Best Configuration": "best_model_configurations.csv",
    "Best Model": "best_models.csv"
}

df_init = pd.read_csv(model_selection_file_names["Best Configuration"])
dataset_names = df_init.columns.values[1:].tolist()
languages = list(set([dataset_name.split(" ")[0] for dataset_name in dataset_names]))


def perform_evaluation_for_datasets(model_selection, selected_datasets):
    df = pd.read_csv(model_selection_file_names.get(model_selection))

    selected_indices = []

    for selected_dataset in selected_datasets:
        selected_indices.append(dataset_names.index(selected_dataset) + 1)

    mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2)

    # Include column with column name
    result_df = df.iloc[:, [0] + selected_indices]
    result_df["Average"] = mean_column

    return result_df

def perform_evaluation_for_languages(model_selection, selected_languages):
    df = pd.read_csv(model_selection_file_names.get(model_selection))

    selected_indices = []

    for selected_language in selected_languages:
        selected_language = selected_language.lower()
        found_indices = [i for i, column_name in enumerate(df.columns) if selected_language in column_name.lower()]

        for found_index in found_indices:
            selected_indices.append(found_index)

    mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2)

    # Include column with column name
    result_df = df.iloc[:, [0] + selected_indices]
    result_df["Average"] = mean_column

    return result_df

with gr.Blocks() as demo:
    gr.Markdown(title)
    gr.Markdown(description)

    with gr.Tab("Overview"):
        gr.Markdown("### Best Configuration\nThe best hyper-parameter configuration for each model is used and average F1-score over runs with different seeds is reported here:")

        df_result = perform_evaluation_for_datasets("Best Configuration", dataset_names)

        gr.Dataframe(value=df_result)

        gr.Markdown("### Best Model\nThe best hyper-parameter configuration for each model is used and the model with highest F1-score is used and its performance is reported here:")

        df_result = perform_evaluation_for_datasets("Best Model", dataset_names)

        gr.Dataframe(value=df_result)

    with gr.Tab("Filtering"):

        gr.Markdown("### Filtering\nSwiss-knife filtering for single datasets and languages is possible.")

        model_selection = gr.Radio(choices=["Best Configuration", "Best Model"],
                                   label="Model Selection",
                                   info="Defines if best configuration or best model should be used for evaluation. When 'Best Configuration' is used, the best hyper-parameter configuration is used and then averaged F1-score over all runs is calculated. When 'Best Model' is chosen, the best hyper-parameter configuration and model with highest F1-score on development dataset is used (best model).",
                                   value="Best Configuration")

        with gr.Tab("Dataset Selection"):
            datasets_selection = gr.CheckboxGroup(
                dataset_names, label="Datasets", info="Select datasets for evaluation"
            )
            output_df = gr.Dataframe()

            evaluation_button = gr.Button("Evaluate")
            evaluation_button.click(fn=perform_evaluation_for_datasets, inputs=[model_selection, datasets_selection], outputs=output_df)


        with gr.Tab("Language Selection"):
            language_selection = gr.CheckboxGroup(
                languages, label="Languages", info="Select languages for evaluation"
            )
            output_df = gr.Dataframe()

            evaluation_button = gr.Button("Evaluate")
            evaluation_button.click(fn=perform_evaluation_for_languages, inputs=[model_selection, language_selection], outputs=output_df)



    gr.Markdown(footer)

demo.launch()