File size: 4,951 Bytes
4b2522c
 
 
43c04ea
4b2522c
 
84b5dfa
 
4b2522c
 
 
 
 
84b5dfa
 
e5438c2
84b5dfa
 
 
 
4b43014
84b5dfa
 
4b43014
84b5dfa
 
4b43014
84b5dfa
 
4b43014
84b5dfa
67e855c
4b43014
84b5dfa
 
4b43014
e5438c2
 
4b43014
84b5dfa
4b43014
 
 
 
 
 
 
 
 
 
 
 
 
84b5dfa
4b2522c
 
 
 
84b5dfa
4b2522c
 
43c04ea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f161dff
43c04ea
 
b521c84
 
84b5dfa
4b2522c
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import gradio as gr
from apscheduler.schedulers.background import BackgroundScheduler
from src.static.env import API, REPO_ID, HF_TOKEN
from src.static.about import TITLE, INTRO, ABOUT, DOCUMENTATION, SUBMIT

from src.leaderboards.get_from_hub import get_leaderboard_info
from src.static.tag_info import *
from src.static.display import make_clickable


def restart_space():
    API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)

LEADERBOARDS_TO_INFO, INFO_TO_LEADERBOARDS = get_leaderboard_info()

def update_leaderboards(show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags):
    spaces_of_interest = []
    if show_all:
        spaces_of_interest = INFO_TO_LEADERBOARDS["all"]
    else:
        number_of_tag_types = 0
        for tag in modality_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["modality"][tag.lower()])
            number_of_tag_types += 1
        for tag in submission_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["submission"][tag.lower()])
            number_of_tag_types += 1
        for tag in test_set_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["test"][tag.lower()])
            number_of_tag_types += 1
        for tag in evaluation_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["eval"][tag.lower()])
            number_of_tag_types += 1
        for tag in language_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["language"][tag.lower()])
            number_of_tag_types += 1
        for tag in judge_tags:
            spaces_of_interest.extend(INFO_TO_LEADERBOARDS["judge"][tag.lower()])
            number_of_tag_types += 1

    if number_of_tag_types > 1:
        # Intersection of tag types
        spaces_of_interest = [space for space in spaces_of_interest if spaces_of_interest.count(space) > 1]
    
    spaces_of_interest = sorted(list(set(spaces_of_interest)))

    return "\n".join(
        [
            make_clickable(space) + 
            f"*Tags: {', '.join(LEADERBOARDS_TO_INFO[space]) if len(LEADERBOARDS_TO_INFO[space]) > 0 else 'None. Please fill the tags!'}*" 
            for space in sorted(spaces_of_interest)
        ]
    )



demo = gr.Blocks()
with demo:
    gr.Markdown(TITLE)
    gr.Markdown(INTRO, elem_classes="markdown-text")

    with gr.Row():
        with gr.Column():
            modality_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in Modality],
                value=[],
                label="Modality of choice"
            )
            submission_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in SubmissionType],
                value=[],
                label="Submission type"
            )
            test_set_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in TestSetStatus],
                value=[],
                label="Test set status"
            )
            judge_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in Judge],
                value=[],
                label="Judge used for the evaluation"
            )
        with gr.Column():
            show_all = gr.Checkbox(
                value=False, 
                label="Show all leaderboards"
            )
            evaluation_tags = gr.CheckboxGroup(
                choices=[tag.name for tag in EvaluationCategory],
                value=[],
                label="Specific evaluation categories"
            )
            language_tags = gr.CheckboxGroup(
                choices=[tag.capitalize() for tag in sorted(list(INFO_TO_LEADERBOARDS["language"].keys()))],
                value=[],
                label="Specific languages"
            ) 
    with gr.Row():
        leaderboards = gr.Markdown(
            value="",
        )

    for selector in [modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
        selector.change(
            lambda _: False,
            outputs=show_all
        )
    for selector in [show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags]:
        selector.change(
            update_leaderboards,
            [show_all, modality_tags, submission_tags, test_set_tags, evaluation_tags, language_tags, judge_tags],
            leaderboards,
            queue=True,
        )

    with gr.Accordion("How to submit your leaderboard?", open=False):
        gr.Markdown(SUBMIT, elem_classes="markdown-text")

    with gr.Accordion("What do the tags mean?", open=False):
        gr.Markdown(ABOUT, elem_classes="markdown-text")

    #with gr.Accordion("How to build your own leaderboard?", open=False):
    #    gr.Markdown(DOCUMENTATION, elem_classes="markdown-text")

scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
scheduler.start()

demo.queue(default_concurrency_limit=40).launch()