File size: 28,850 Bytes
0ba78e9
580b4e4
0ba78e9
0d5e0f7
54f6b18
0ba78e9
0b46237
1e40fe5
0ba78e9
ef65795
0d5e0f7
79d85b6
675f890
0b46237
e502d68
2ed83bb
675f890
 
aacdddf
d0649fc
675f890
 
 
 
 
0d5e0f7
0ba78e9
 
0d5e0f7
0ba78e9
 
 
6c14077
0d5e0f7
b36527b
294f139
b36527b
 
294f139
 
533bc81
294f139
1e7e407
10eada1
294f139
833c734
294f139
 
9bb22fc
 
a16df4c
7b6efd3
 
 
a16df4c
 
533bc81
9bb22fc
1e7e407
72d11c4
 
fcdf4a0
 
 
 
 
 
 
833c734
9bb22fc
 
fcdf4a0
 
 
 
 
 
d68b7d5
 
fcdf4a0
24b0def
5b19fc7
ef65795
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9bb22fc
5b19fc7
0779c9b
 
 
31b9ddb
da19d23
 
ef65795
 
 
b585c09
ef65795
 
 
9fd1031
 
e14de0c
da19d23
 
294f139
6c14077
d7705b9
a081a7c
 
 
d7705b9
a081a7c
 
 
d7705b9
31b9ddb
 
 
 
0b46237
 
31b9ddb
d7705b9
f574f70
9c61a68
 
 
 
 
 
 
 
 
a92816b
9c61a68
0d5e0f7
de92d49
e502d68
6c14077
 
0ba78e9
2859204
0779c9b
5b19fc7
 
24b0def
 
9c61a68
a92816b
5b19fc7
0779c9b
6c14077
9c61a68
 
 
 
 
 
 
 
b76fdef
aacdddf
 
6c14077
0779c9b
 
 
 
 
 
6c14077
 
 
 
 
 
 
aacdddf
 
1e40fe5
 
5b19fc7
 
 
1e40fe5
9c61a68
5b19fc7
6c14077
0779c9b
6c14077
4ec914d
bf0dd52
0779c9b
 
 
 
 
6c14077
54f6b18
294f139
9c61a68
 
 
 
 
294f139
 
da19d23
5b19fc7
fd18ef6
54f6b18
 
fd18ef6
 
 
 
 
 
54f6b18
5b19fc7
9c61a68
5b19fc7
aacdddf
 
 
5b19fc7
54f6b18
9c61a68
5b19fc7
aacdddf
 
 
54f6b18
 
 
 
d68b7d5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
533bc81
 
 
 
 
 
 
 
aacdddf
533bc81
 
 
 
 
aacdddf
533bc81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54f6b18
 
 
 
 
 
 
 
 
 
9c61a68
54f6b18
aacdddf
 
 
54f6b18
 
9c61a68
5b19fc7
aacdddf
 
 
54f6b18
 
 
 
 
 
 
 
 
 
 
 
 
5b19fc7
9c61a68
5b19fc7
aacdddf
 
 
5b19fc7
54f6b18
9c61a68
5b19fc7
aacdddf
 
 
54f6b18
 
 
 
 
 
 
 
 
 
 
 
 
5b19fc7
9c61a68
5b19fc7
aacdddf
 
 
5b19fc7
 
9c61a68
5b19fc7
aacdddf
 
 
5b19fc7
54f6b18
 
 
 
aacdddf
 
31b9ddb
 
54f6b18
fd18ef6
 
 
 
 
 
54f6b18
 
 
 
 
 
 
 
 
 
 
5b19fc7
 
 
aacdddf
5b19fc7
54f6b18
5b19fc7
 
aacdddf
54f6b18
 
5b19fc7
 
aacdddf
54f6b18
 
9c61a68
54f6b18
aacdddf
 
 
54f6b18
 
 
 
 
fcdf4a0
 
 
 
 
 
 
 
 
 
 
 
aacdddf
 
 
fcdf4a0
 
 
 
aacdddf
 
 
fcdf4a0
 
 
0d5e0f7
72d11c4
7b6efd3
9bb22fc
a16df4c
 
 
faca395
 
a16df4c
 
 
 
 
9bb22fc
 
 
ef65795
 
 
 
a16df4c
0d5e0f7
7b6efd3
eef70c0
31b9ddb
 
 
ef65795
73f2f7a
31b9ddb
e502d68
af2acd4
b36527b
 
e502d68
0d5e0f7
fd18ef6
b23bddd
 
4677a77
 
b23bddd
d658c8a
4677a77
d658c8a
18133af
 
 
 
 
 
 
 
 
d658c8a
 
 
74c1d10
d658c8a
 
 
 
 
 
 
b6fc567
d68b7d5
 
 
d658c8a
 
df35625
d658c8a
 
 
 
 
 
c671908
 
d658c8a
 
 
 
fd18ef6
 
 
d658c8a
 
 
 
 
 
 
d68b7d5
 
 
d658c8a
 
d68b7d5
d658c8a
c671908
 
 
d658c8a
 
dde0fd4
 
d658c8a
 
 
580b4e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d658c8a
8a21997
580b4e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d658c8a
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
import os
import time
from pathlib import Path

import pandas as pd
import streamlit as st
import yaml
from datasets import get_dataset_config_names
from dotenv import load_dotenv
from huggingface_hub import list_datasets

from evaluation import filter_evaluated_models
from utils import (
    AUTOTRAIN_TASK_TO_HUB_TASK,
    commit_evaluation_log,
    create_autotrain_project_name,
    format_col_mapping,
    get_compatible_models,
    get_config_metadata,
    get_dataset_card_url,
    get_key,
    get_metadata,
    http_get,
    http_post,
)

if Path(".env").is_file():
    load_dotenv(".env")

HF_TOKEN = os.getenv("HF_TOKEN")
AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
DATASETS_PREVIEW_API = os.getenv("DATASETS_PREVIEW_API")

# Put image tasks on top
TASK_TO_ID = {
    "image_binary_classification": 17,
    "image_multi_class_classification": 18,
    "binary_classification": 1,
    "multi_class_classification": 2,
    "natural_language_inference": 22,
    "entity_extraction": 4,
    "extractive_question_answering": 5,
    "translation": 6,
    "summarization": 8,
    "text_zero_shot_classification": 23,
}

TASK_TO_DEFAULT_METRICS = {
    "binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],
    "multi_class_classification": [
        "f1",
        "precision",
        "recall",
        "accuracy",
    ],
    "natural_language_inference": ["f1", "precision", "recall", "auc", "accuracy"],
    "entity_extraction": ["precision", "recall", "f1", "accuracy"],
    "extractive_question_answering": ["f1", "exact_match"],
    "translation": ["sacrebleu"],
    "summarization": ["rouge1", "rouge2", "rougeL", "rougeLsum"],
    "image_binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],
    "image_multi_class_classification": [
        "f1",
        "precision",
        "recall",
        "accuracy",
    ],
    "text_zero_shot_classification": ["accuracy", "loss"],
}

AUTOTRAIN_TASK_TO_LANG = {
    "translation": "en2de",
    "image_binary_classification": "unk",
    "image_multi_class_classification": "unk",
}

AUTOTRAIN_MACHINE = {"text_zero_shot_classification": "r5.16x"}


SUPPORTED_TASKS = list(TASK_TO_ID.keys())

# Extracted from utils.get_supported_metrics
# Hardcoded for now due to speed / caching constraints
SUPPORTED_METRICS = [
    "accuracy",
    "bertscore",
    "bleu",
    "cer",
    "chrf",
    "code_eval",
    "comet",
    "competition_math",
    "coval",
    "cuad",
    "exact_match",
    "f1",
    "frugalscore",
    "google_bleu",
    "mae",
    "mahalanobis",
    "matthews_correlation",
    "mean_iou",
    "meteor",
    "mse",
    "pearsonr",
    "perplexity",
    "precision",
    "recall",
    "roc_auc",
    "rouge",
    "sacrebleu",
    "sari",
    "seqeval",
    "spearmanr",
    "squad",
    "squad_v2",
    "ter",
    "trec_eval",
    "wer",
    "wiki_split",
    "xnli",
    "angelina-wang/directional_bias_amplification",
    "jordyvl/ece",
    "lvwerra/ai4code",
    "lvwerra/amex",
]


#######
# APP #
#######
st.title("Evaluation on the Hub")
st.markdown(
    """
    Welcome to Hugging Face's automatic model evaluator πŸ‘‹!

    This application allows you to evaluate πŸ€— Transformers
    [models](https://huggingface.co/models?library=transformers&sort=downloads)
    across a wide variety of [datasets](https://huggingface.co/datasets) on the
    Hub. Please select the dataset and configuration below. The results of your
    evaluation will be displayed on the [public
    leaderboards](https://huggingface.co/spaces/autoevaluate/leaderboards). For
    more details, check out out our [blog
    post](https://huggingface.co/blog/eval-on-the-hub).
    """
)

all_datasets = [d.id for d in list_datasets()]
query_params = st.experimental_get_query_params()
if "first_query_params" not in st.session_state:
    st.session_state.first_query_params = query_params
first_query_params = st.session_state.first_query_params
default_dataset = all_datasets[0]
if "dataset" in first_query_params:
    if len(first_query_params["dataset"]) > 0 and first_query_params["dataset"][0] in all_datasets:
        default_dataset = first_query_params["dataset"][0]

selected_dataset = st.selectbox(
    "Select a dataset",
    all_datasets,
    index=all_datasets.index(default_dataset),
    help="""Datasets with metadata can be evaluated with 1-click. Configure an evaluation job to add \
        new metadata to a dataset card.""",
)
st.experimental_set_query_params(**{"dataset": [selected_dataset]})

# Check if selected dataset can be streamed
is_valid_dataset = http_get(
    path="/is-valid",
    domain=DATASETS_PREVIEW_API,
    params={"dataset": selected_dataset},
).json()
if is_valid_dataset["valid"] is False:
    st.error(
        """The dataset you selected is not currently supported. Open a \
            [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) for support."""
    )

metadata = get_metadata(selected_dataset, token=HF_TOKEN)
print(f"INFO -- Dataset metadata: {metadata}")
if metadata is None:
    st.warning("No evaluation metadata found. Please configure the evaluation job below.")

with st.expander("Advanced configuration"):
    # Select task
    selected_task = st.selectbox(
        "Select a task",
        SUPPORTED_TASKS,
        index=SUPPORTED_TASKS.index(metadata[0]["task_id"]) if metadata is not None else 0,
        help="""Don't see your favourite task here? Open a \
            [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) to request it!""",
    )
    # Select config
    configs = get_dataset_config_names(selected_dataset)
    selected_config = st.selectbox(
        "Select a config",
        configs,
        help="""Some datasets contain several sub-datasets, known as _configurations_. \
            Select one to evaluate your models on. \
            See the [docs](https://huggingface.co/docs/datasets/master/en/load_hub#configurations) for more details.
            """,
    )
    # Some datasets have multiple metadata (one per config), so we grab the one associated with the selected config
    config_metadata = get_config_metadata(selected_config, metadata)
    print(f"INFO -- Config metadata: {config_metadata}")

    # Select splits
    splits_resp = http_get(
        path="/splits",
        domain=DATASETS_PREVIEW_API,
        params={"dataset": selected_dataset},
    )
    if splits_resp.status_code == 200:
        split_names = []
        all_splits = splits_resp.json()
        for split in all_splits["splits"]:
            if split["config"] == selected_config:
                split_names.append(split["split"])

        if config_metadata is not None:
            eval_split = config_metadata["splits"].get("eval_split", None)
        else:
            eval_split = None
        selected_split = st.selectbox(
            "Select a split",
            split_names,
            index=split_names.index(eval_split) if eval_split is not None else 0,
            help="Be wary when evaluating models on the `train` split.",
        )

    # Select columns
    rows_resp = http_get(
        path="/first-rows",
        domain=DATASETS_PREVIEW_API,
        params={
            "dataset": selected_dataset,
            "config": selected_config,
            "split": selected_split,
        },
    ).json()
    col_names = list(pd.json_normalize(rows_resp["rows"][0]["row"]).columns)

    st.markdown("**Map your dataset columns**")
    st.markdown(
        """The model evaluator uses a standardised set of column names for the input examples and labels. \
        Please define the mapping between your dataset columns (right) and the standardised column names (left)."""
    )
    col1, col2 = st.columns(2)

    # TODO: find a better way to layout these items
    # TODO: need graceful way of handling dataset <--> task mismatch for datasets with metadata
    col_mapping = {}
    if selected_task in ["binary_classification", "multi_class_classification"]:
        with col1:
            st.markdown("`text` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            text_col = st.selectbox(
                "This column should contain the text to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the labels associated with the text",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[text_col] = "text"
            col_mapping[target_col] = "target"

    elif selected_task == "text_zero_shot_classification":
        with col1:
            st.markdown("`text` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`classes` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            text_col = st.selectbox(
                "This column should contain the text to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
                if config_metadata is not None
                else 0,
            )
            classes_col = st.selectbox(
                "This column should contain the classes associated with the text",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "classes"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the index of the correct class",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[text_col] = "text"
            col_mapping[classes_col] = "classes"
            col_mapping[target_col] = "target"

    if selected_task in ["natural_language_inference"]:
        config_metadata = get_config_metadata(selected_config, metadata)
        with col1:
            st.markdown("`text1` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`text2` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            text1_col = st.selectbox(
                "This column should contain the first text passage to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "text1"))
                if config_metadata is not None
                else 0,
            )
            text2_col = st.selectbox(
                "This column should contain the second text passage to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "text2"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the labels associated with the text",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[text1_col] = "text1"
            col_mapping[text2_col] = "text2"
            col_mapping[target_col] = "target"

    elif selected_task == "entity_extraction":
        with col1:
            st.markdown("`tokens` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`tags` column")
        with col2:
            tokens_col = st.selectbox(
                "This column should contain the array of tokens to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "tokens"))
                if config_metadata is not None
                else 0,
            )
            tags_col = st.selectbox(
                "This column should contain the labels associated with each part of the text",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "tags"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[tokens_col] = "tokens"
            col_mapping[tags_col] = "tags"

    elif selected_task == "translation":
        with col1:
            st.markdown("`source` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            text_col = st.selectbox(
                "This column should contain the text to be translated",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "source"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the target translation",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[text_col] = "source"
            col_mapping[target_col] = "target"

    elif selected_task == "summarization":
        with col1:
            st.markdown("`text` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            text_col = st.selectbox(
                "This column should contain the text to be summarized",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the target summary",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[text_col] = "text"
            col_mapping[target_col] = "target"

    elif selected_task == "extractive_question_answering":
        if config_metadata is not None:
            col_mapping = config_metadata["col_mapping"]
            # Hub YAML parser converts periods to hyphens, so we remap them here
            col_mapping = format_col_mapping(col_mapping)
        with col1:
            st.markdown("`context` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`question` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`answers.text` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`answers.answer_start` column")
        with col2:
            context_col = st.selectbox(
                "This column should contain the question's context",
                col_names,
                index=col_names.index(get_key(col_mapping, "context")) if config_metadata is not None else 0,
            )
            question_col = st.selectbox(
                "This column should contain the question to be answered, given the context",
                col_names,
                index=col_names.index(get_key(col_mapping, "question")) if config_metadata is not None else 0,
            )
            answers_text_col = st.selectbox(
                "This column should contain example answers to the question, extracted from the context",
                col_names,
                index=col_names.index(get_key(col_mapping, "answers.text")) if config_metadata is not None else 0,
            )
            answers_start_col = st.selectbox(
                "This column should contain the indices in the context of the first character of each `answers.text`",
                col_names,
                index=col_names.index(get_key(col_mapping, "answers.answer_start"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[context_col] = "context"
            col_mapping[question_col] = "question"
            col_mapping[answers_text_col] = "answers.text"
            col_mapping[answers_start_col] = "answers.answer_start"
    elif selected_task in ["image_binary_classification", "image_multi_class_classification"]:
        with col1:
            st.markdown("`image` column")
            st.text("")
            st.text("")
            st.text("")
            st.text("")
            st.markdown("`target` column")
        with col2:
            image_col = st.selectbox(
                "This column should contain the images to be classified",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "image"))
                if config_metadata is not None
                else 0,
            )
            target_col = st.selectbox(
                "This column should contain the labels associated with the images",
                col_names,
                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
                if config_metadata is not None
                else 0,
            )
            col_mapping[image_col] = "image"
            col_mapping[target_col] = "target"

    # Select metrics
    st.markdown("**Select metrics**")
    st.markdown("The following metrics will be computed")
    html_string = " ".join(
        [
            '<div style="padding-right:5px;padding-left:5px;padding-top:5px;padding-bottom:5px;float:left">'
            + '<div style="background-color:#D3D3D3;border-radius:5px;display:inline-block;padding-right:5px;'
            + 'padding-left:5px;color:white">'
            + metric
            + "</div></div>"
            for metric in TASK_TO_DEFAULT_METRICS[selected_task]
        ]
    )
    st.markdown(html_string, unsafe_allow_html=True)
    selected_metrics = st.multiselect(
        "(Optional) Select additional metrics",
        sorted(list(set(SUPPORTED_METRICS) - set(TASK_TO_DEFAULT_METRICS[selected_task]))),
        help="""User-selected metrics will be computed with their default arguments. \
            For example, `f1` will report results for binary labels. \
            Check out the [available metrics](https://huggingface.co/metrics) for more details.""",
    )

with st.form(key="form"):
    compatible_models = get_compatible_models(selected_task, [selected_dataset])
    selected_models = st.multiselect(
        "Select the models you wish to evaluate",
        compatible_models,
        help="""Don't see your favourite model in this list? Add the dataset and task it was trained on to the \
            [model card metadata.](https://huggingface.co/docs/hub/models-cards#model-card-metadata)""",
    )
    print("INFO -- Selected models before filter:", selected_models)

    hf_username = st.text_input("Enter your πŸ€— Hub username to be notified when the evaluation is finished")

    submit_button = st.form_submit_button("Evaluate models πŸš€")

    if submit_button:
        if len(hf_username) == 0:
            st.warning("No πŸ€— Hub username provided! Please enter your username and try again.")
        elif len(selected_models) == 0:
            st.warning("⚠️ No models were selected for evaluation! Please select at least one model and try again.")
        elif len(selected_models) > 10:
            st.warning("Only 10 models can be evaluated at once. Please select fewer models and try again.")
        else:
            # Filter out previously evaluated models
            selected_models = filter_evaluated_models(
                selected_models,
                selected_task,
                selected_dataset,
                selected_config,
                selected_split,
                selected_metrics,
            )
            print("INFO -- Selected models after filter:", selected_models)
            if len(selected_models) > 0:
                project_payload = {
                    "username": AUTOTRAIN_USERNAME,
                    "proj_name": create_autotrain_project_name(selected_dataset, selected_config),
                    "task": TASK_TO_ID[selected_task],
                    "config": {
                        "language": AUTOTRAIN_TASK_TO_LANG[selected_task]
                        if selected_task in AUTOTRAIN_TASK_TO_LANG
                        else "en",
                        "max_models": 5,
                        "instance": {
                            "provider": "sagemaker" if selected_task in AUTOTRAIN_MACHINE.keys() else "ovh",
                            "instance_type": AUTOTRAIN_MACHINE[selected_task]
                            if selected_task in AUTOTRAIN_MACHINE.keys()
                            else "p3",
                            "max_runtime_seconds": 172800,
                            "num_instances": 1,
                            "disk_size_gb": 200,
                        },
                        "evaluation": {
                            "metrics": selected_metrics,
                            "models": selected_models,
                            "hf_username": hf_username,
                        },
                    },
                }
                print(f"INFO -- Payload: {project_payload}")
                project_json_resp = http_post(
                    path="/projects/create",
                    payload=project_payload,
                    token=HF_TOKEN,
                    domain=AUTOTRAIN_BACKEND_API,
                ).json()
                print(f"INFO -- Project creation response: {project_json_resp}")

                if project_json_resp["created"]:
                    data_payload = {
                        "split": 4,  # use "auto" split choice in AutoTrain
                        "col_mapping": col_mapping,
                        "load_config": {"max_size_bytes": 0, "shuffle": False},
                        "dataset_id": selected_dataset,
                        "dataset_config": selected_config,
                        "dataset_split": selected_split,
                    }
                    data_json_resp = http_post(
                        path=f"/projects/{project_json_resp['id']}/data/dataset",
                        payload=data_payload,
                        token=HF_TOKEN,
                        domain=AUTOTRAIN_BACKEND_API,
                    ).json()
                    print(f"INFO -- Dataset creation response: {data_json_resp}")
                    if data_json_resp["download_status"] == 1:
                        train_json_resp = http_post(
                            path=f"/projects/{project_json_resp['id']}/data/start_processing",
                            token=HF_TOKEN,
                            domain=AUTOTRAIN_BACKEND_API,
                        ).json()
                        # For local development we process and approve projects on-the-fly
                        if "localhost" in AUTOTRAIN_BACKEND_API:
                            with st.spinner("⏳ Waiting for data processing to complete ..."):
                                is_data_processing_success = False
                                while is_data_processing_success is not True:
                                    project_status = http_get(
                                        path=f"/projects/{project_json_resp['id']}",
                                        token=HF_TOKEN,
                                        domain=AUTOTRAIN_BACKEND_API,
                                    ).json()
                                    if project_status["status"] == 3:
                                        is_data_processing_success = True
                                    time.sleep(10)

                            # Approve training job
                            train_job_resp = http_post(
                                path=f"/projects/{project_json_resp['id']}/start_training",
                                token=HF_TOKEN,
                                domain=AUTOTRAIN_BACKEND_API,
                            ).json()
                            st.success("βœ…  Data processing and project approval complete - go forth and evaluate!")
                        else:
                            # Prod/staging submissions are evaluated in a cron job via run_evaluation_jobs.py
                            print(f"INFO -- AutoTrain job response: {train_json_resp}")
                            if train_json_resp["success"]:
                                train_eval_index = {
                                    "train-eval-index": [
                                        {
                                            "config": selected_config,
                                            "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
                                            "task_id": selected_task,
                                            "splits": {"eval_split": selected_split},
                                            "col_mapping": col_mapping,
                                        }
                                    ]
                                }
                                selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
                                dataset_card_url = get_dataset_card_url(selected_dataset)
                                st.success("βœ… Successfully submitted evaluation job!")
                                st.markdown(
                                    f"""
                                Evaluation can take up to 1 hour to complete, so grab a β˜•οΈ or 🍡 while you wait:

                                * πŸ”” A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
                                * πŸ“Š Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
                                * πŸ₯± Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
                                """  # noqa
                                )
                                st.markdown(
                                    f"""
                                ```yaml
                                {selected_metadata}
                                """
                                )
                                print("INFO -- Pushing evaluation job logs to the Hub")
                                evaluation_log = {}
                                evaluation_log["project_id"] = project_json_resp["id"]
                                evaluation_log["autotrain_env"] = (
                                    "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod"
                                )
                                evaluation_log["payload"] = project_payload
                                evaluation_log["project_creation_response"] = project_json_resp
                                evaluation_log["dataset_creation_response"] = data_json_resp
                                evaluation_log["autotrain_job_response"] = train_json_resp
                                commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
                            else:
                                st.error("πŸ™ˆ Oh no, there was an error submitting your evaluation job!")
            else:
                st.warning("⚠️ No models left to evaluate! Please select other models and try again.")