File size: 7,485 Bytes
9203553
e2473e2
 
cdf268e
e2473e2
6c92442
 
 
 
 
 
 
 
 
 
 
 
9203553
a9273cf
e2473e2
6c92442
 
 
 
 
 
e2473e2
9203553
 
 
 
 
 
04b315a
 
 
e2473e2
 
b7e5123
 
 
 
 
 
 
307c2a1
b7e5123
0ab9033
fd03829
 
 
307c2a1
5613f02
fd03829
 
 
 
 
 
 
e2473e2
6c92442
e2473e2
 
 
6c92442
e2473e2
 
 
6c92442
 
e2473e2
6c92442
e2473e2
fd03829
 
f098e7d
fd03829
 
7471397
fd03829
 
 
0ab9033
fd03829
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e2473e2
6c92442
0548807
e2473e2
 
96ff3bc
e2473e2
6c92442
e2473e2
 
 
 
 
 
04b315a
e2473e2
 
 
 
 
2d26479
 
 
 
 
 
 
 
04f40cd
2d26479
e2473e2
 
 
 
 
 
9203553
e2473e2
 
 
 
9203553
 
e2473e2
 
9203553
e2473e2
 
 
 
 
 
 
04f40cd
 
 
 
 
 
e2473e2
 
 
 
 
 
 
6c92442
e2473e2
 
 
2d26479
e2473e2
 
 
04b315a
 
e2473e2
 
 
 
0548807
e2473e2
 
b7e5123
e2473e2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import logging
import os

import gradio as gr  # type: ignore[import]

from src.content import (
    INTRODUCTION_TEXT,
    INTRODUCTION_TITLE,
    LEADERBOARD_TEXT,
    LEADERBOARD_TITLE,
    SUBMISSION_TEXT_FILES,
    SUBMISSION_TEXT_INTRO,
    SUBMISSION_TEXT_METADATA,
    SUBMISSION_TEXT_SUBMIT,
    SUBMISSION_TEXT_TASK,
    SUBMISSION_TITLE,
)
from src.get_results_for_task import get_results_for_task
from src.leaderboard_formatting import get_types_per_task
from src.submission_uploader import SubmissionUploader
from src.tasks_content import (
    TASKS_DESCRIPTIONS,
    TASKS_PRETTY,
    TASKS_PRETTY_REVERSE,
    get_submission_text_files_for_task,
)

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[logging.StreamHandler()],
)

submission_uploader = SubmissionUploader(
    dataset_id=os.environ["DATASET_ID"], private_dataset_id=os.environ["PRIVATE_DATASET_ID"]
)


def get_leaderboard_for_task(task_pretty: str) -> gr.components.Dataframe:
    return gr.components.Dataframe(
        value=get_results_for_task(task_pretty),
        interactive=False,
        datatype=get_types_per_task(TASKS_PRETTY_REVERSE[task_pretty]),
    )

code_completion_dataset_names = get_results_for_task(TASKS_PRETTY['project_code_completion'])['Dataset Name'].unique().tolist()

def get_leaderboard_for_completion_task(dataset_name: str | None):
    df = get_results_for_task(TASKS_PRETTY['project_code_completion'])
    if dataset_name is None:
        dataset_name = code_completion_dataset_names[0]
    filtered_df = df[df['Dataset Name']==dataset_name]
    filtered_df.drop(columns=['Dataset'], inplace=True)
    return gr.components.Dataframe(
        value=filtered_df,
        interactive=False,
        datatype=get_types_per_task('project_code_completion'),
    )


with gr.Blocks() as demo:
    # intro
    gr.HTML(INTRODUCTION_TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    # leaderboard
    gr.HTML(LEADERBOARD_TITLE)
    gr.Markdown(LEADERBOARD_TEXT, elem_classes="markdown-text")
    with gr.Tabs():
        for task_pretty in TASKS_PRETTY_REVERSE:
            with gr.TabItem(task_pretty):
                with gr.Row():
                    gr.Markdown(TASKS_DESCRIPTIONS[TASKS_PRETTY_REVERSE[task_pretty]])

                if task_pretty == TASKS_PRETTY['project_code_completion']:
                    leaderboard_table = get_leaderboard_for_completion_task(dataset_name=None)
                    
                else:
                    leaderboard_table = get_leaderboard_for_task(task_pretty)
                
                task_input = gr.Text(value=task_pretty, visible=False)

                if task_pretty == TASKS_PRETTY['project_code_completion']:
                    dataset_dropdown = gr.Dropdown(choices=code_completion_dataset_names, label="Select the Dataset")
                    dataset_dropdown.change(
                        fn=get_leaderboard_for_completion_task,
                        inputs=dataset_dropdown,
                        outputs=leaderboard_table
                    )
                    refresh_button = gr.Button("πŸ”„ Refresh", variant="secondary")
                    refresh_button.click(
                        fn=get_leaderboard_for_completion_task,
                        inputs=dataset_dropdown,
                        outputs=leaderboard_table,
                    )
                else:
                    refresh_button = gr.Button("πŸ”„ Refresh", variant="secondary")
                    refresh_button.click(
                        fn=get_leaderboard_for_task,
                        inputs=task_input,
                        outputs=leaderboard_table,
                    )

    # submission
    '''
    gr.HTML(SUBMISSION_TITLE)
    gr.Markdown(SUBMISSION_TEXT_INTRO, elem_classes="markdown-text")
    with gr.Accordion("πŸš€ Submit new results", open=False):
        gr.Markdown(SUBMISSION_TEXT_TASK, elem_classes="markdown-text")
        task_selection = gr.Radio(TASKS_PRETTY_REVERSE.keys(), label="Task")

        gr.Markdown(SUBMISSION_TEXT_METADATA, elem_classes="markdown-text")
        with gr.Row():
            with gr.Column():
                model_folder_textbox = gr.Textbox(
                    label="Model Folder",
                    placeholder="How to call a folder related to this submission in our results dataset (should be unique).",
                )
                model_name_textbox = gr.Textbox(
                    label="Model Name",
                    placeholder="How to display model's name on the leaderboard.",
                )
                model_url_textbox = gr.Textbox(
                    label="Model URL",
                    placeholder="Link to a model's page - will be clickable on a leaderboard (optional).",
                )

            with gr.Column():
                url_textbox = gr.Textbox(
                    label="Relevant URLs",
                    placeholder='URLs to relevant resources with additional details about your submission (optional). Use the following format: "[text1](link1), [text2](link2)".',
                )
                model_availability_textbox = gr.Textbox(
                    label="Availability",
                    placeholder="Information about the model's availability and licensing.",
                )
                context_size_textbox = gr.Textbox(
                    label="Context Size",
                    placeholder="Context size in tokens used for the submission (should be an integer).",
                )
            with gr.Column():
                submitted_by_textbox = gr.Textbox(
                    label="Submitted By",
                    placeholder="How to display on the leaderboard who submitted the model.",
                )
                contact_textbox = gr.Textbox(
                    label="Contact Information",
                    placeholder="How Long Code Arena team can contact you (won't go to public dataset).",
                )
                comment_textbox = gr.Textbox(
                    label="Comment",
                    placeholder="Any comments you have for Long Code Arena team (optional, won't go to public dataset).",
                )

        gr.Markdown(SUBMISSION_TEXT_FILES, elem_classes="markdown-text")
        with gr.Row():
            with gr.Column(variant="panel"):
                task_specific_instructions = gr.Markdown(get_submission_text_files_for_task(None))
                task_selection.select(get_submission_text_files_for_task, [task_selection], task_specific_instructions)
            with gr.Column():
                file_output = gr.File(file_count="multiple")

        gr.Markdown(SUBMISSION_TEXT_SUBMIT, elem_classes="markdown-text")
        submit_button = gr.Button("Submit")
        submission_result = gr.Markdown()
        submit_button.click(
            submission_uploader.upload_files,
            [
                task_selection,
                model_folder_textbox,
                model_name_textbox,
                model_availability_textbox,
                model_url_textbox,
                url_textbox,
                context_size_textbox,
                submitted_by_textbox,
                contact_textbox,
                comment_textbox,
                file_output,
            ],
            submission_result,
        )
    '''

if __name__ == "__main__":
    demo.queue()
    demo.launch()