|
from functools import partial |
|
import json |
|
import numpy as np |
|
|
|
from datasets import load_dataset |
|
import gradio as gr |
|
from huggingface_hub import get_hf_file_metadata, HfApi, hf_hub_download, hf_hub_url |
|
from huggingface_hub.repocard import metadata_load |
|
import pandas as pd |
|
from tqdm.autonotebook import tqdm |
|
|
|
TASKS = [ |
|
"BitextMining", |
|
"Classification", |
|
"Clustering", |
|
"PairClassification", |
|
"Reranking", |
|
"Retrieval", |
|
"STS", |
|
"Summarization", |
|
] |
|
|
|
TASK_LIST_BITEXT_MINING = ['BUCC (de-en)', 'BUCC (fr-en)', 'BUCC (ru-en)', 'BUCC (zh-en)', 'Tatoeba (afr-eng)', 'Tatoeba (amh-eng)', 'Tatoeba (ang-eng)', 'Tatoeba (ara-eng)', 'Tatoeba (arq-eng)', 'Tatoeba (arz-eng)', 'Tatoeba (ast-eng)', 'Tatoeba (awa-eng)', 'Tatoeba (aze-eng)', 'Tatoeba (bel-eng)', 'Tatoeba (ben-eng)', 'Tatoeba (ber-eng)', 'Tatoeba (bos-eng)', 'Tatoeba (bre-eng)', 'Tatoeba (bul-eng)', 'Tatoeba (cat-eng)', 'Tatoeba (cbk-eng)', 'Tatoeba (ceb-eng)', 'Tatoeba (ces-eng)', 'Tatoeba (cha-eng)', 'Tatoeba (cmn-eng)', 'Tatoeba (cor-eng)', 'Tatoeba (csb-eng)', 'Tatoeba (cym-eng)', 'Tatoeba (dan-eng)', 'Tatoeba (deu-eng)', 'Tatoeba (dsb-eng)', 'Tatoeba (dtp-eng)', 'Tatoeba (ell-eng)', 'Tatoeba (epo-eng)', 'Tatoeba (est-eng)', 'Tatoeba (eus-eng)', 'Tatoeba (fao-eng)', 'Tatoeba (fin-eng)', 'Tatoeba (fra-eng)', 'Tatoeba (fry-eng)', 'Tatoeba (gla-eng)', 'Tatoeba (gle-eng)', 'Tatoeba (glg-eng)', 'Tatoeba (gsw-eng)', 'Tatoeba (heb-eng)', 'Tatoeba (hin-eng)', 'Tatoeba (hrv-eng)', 'Tatoeba (hsb-eng)', 'Tatoeba (hun-eng)', 'Tatoeba (hye-eng)', 'Tatoeba (ido-eng)', 'Tatoeba (ile-eng)', 'Tatoeba (ina-eng)', 'Tatoeba (ind-eng)', 'Tatoeba (isl-eng)', 'Tatoeba (ita-eng)', 'Tatoeba (jav-eng)', 'Tatoeba (jpn-eng)', 'Tatoeba (kab-eng)', 'Tatoeba (kat-eng)', 'Tatoeba (kaz-eng)', 'Tatoeba (khm-eng)', 'Tatoeba (kor-eng)', 'Tatoeba (kur-eng)', 'Tatoeba (kzj-eng)', 'Tatoeba (lat-eng)', 'Tatoeba (lfn-eng)', 'Tatoeba (lit-eng)', 'Tatoeba (lvs-eng)', 'Tatoeba (mal-eng)', 'Tatoeba (mar-eng)', 'Tatoeba (max-eng)', 'Tatoeba (mhr-eng)', 'Tatoeba (mkd-eng)', 'Tatoeba (mon-eng)', 'Tatoeba (nds-eng)', 'Tatoeba (nld-eng)', 'Tatoeba (nno-eng)', 'Tatoeba (nob-eng)', 'Tatoeba (nov-eng)', 'Tatoeba (oci-eng)', 'Tatoeba (orv-eng)', 'Tatoeba (pam-eng)', 'Tatoeba (pes-eng)', 'Tatoeba (pms-eng)', 'Tatoeba (pol-eng)', 'Tatoeba (por-eng)', 'Tatoeba (ron-eng)', 'Tatoeba (rus-eng)', 'Tatoeba (slk-eng)', 'Tatoeba (slv-eng)', 'Tatoeba (spa-eng)', 'Tatoeba (sqi-eng)', 'Tatoeba (srp-eng)', 'Tatoeba (swe-eng)', 'Tatoeba (swg-eng)', 'Tatoeba (swh-eng)', 'Tatoeba (tam-eng)', 'Tatoeba (tat-eng)', 'Tatoeba (tel-eng)', 'Tatoeba (tgl-eng)', 'Tatoeba (tha-eng)', 'Tatoeba (tuk-eng)', 'Tatoeba (tur-eng)', 'Tatoeba (tzl-eng)', 'Tatoeba (uig-eng)', 'Tatoeba (ukr-eng)', 'Tatoeba (urd-eng)', 'Tatoeba (uzb-eng)', 'Tatoeba (vie-eng)', 'Tatoeba (war-eng)', 'Tatoeba (wuu-eng)', 'Tatoeba (xho-eng)', 'Tatoeba (yid-eng)', 'Tatoeba (yue-eng)', 'Tatoeba (zsm-eng)'] |
|
TASK_LIST_BITEXT_MINING_OTHER = ["BornholmBitextMining"] |
|
|
|
TASK_LIST_CLASSIFICATION = [ |
|
"AmazonCounterfactualClassification (en)", |
|
"AmazonPolarityClassification", |
|
"AmazonReviewsClassification (en)", |
|
"Banking77Classification", |
|
"EmotionClassification", |
|
"ImdbClassification", |
|
"MassiveIntentClassification (en)", |
|
"MassiveScenarioClassification (en)", |
|
"MTOPDomainClassification (en)", |
|
"MTOPIntentClassification (en)", |
|
"ToxicConversationsClassification", |
|
"TweetSentimentExtractionClassification", |
|
] |
|
|
|
TASK_LIST_CLASSIFICATION_NORM = [x.replace(" (en)", "") for x in TASK_LIST_CLASSIFICATION] |
|
|
|
|
|
TASK_LIST_CLASSIFICATION_SV = [ |
|
"DalajClassification", |
|
"MassiveIntentClassification (sv)", |
|
"MassiveScenarioClassification (sv)", |
|
"NordicLangClassification", |
|
"ScalaSvClassification", |
|
"SweRecClassification", |
|
] |
|
|
|
TASK_LIST_CLASSIFICATION_OTHER = ['AmazonCounterfactualClassification (de)', 'AmazonCounterfactualClassification (ja)', 'AmazonReviewsClassification (de)', 'AmazonReviewsClassification (es)', 'AmazonReviewsClassification (fr)', 'AmazonReviewsClassification (ja)', 'AmazonReviewsClassification (zh)', 'MTOPDomainClassification (de)', 'MTOPDomainClassification (es)', 'MTOPDomainClassification (fr)', 'MTOPDomainClassification (hi)', 'MTOPDomainClassification (th)', 'MTOPIntentClassification (de)', 'MTOPIntentClassification (es)', 'MTOPIntentClassification (fr)', 'MTOPIntentClassification (hi)', 'MTOPIntentClassification (th)', 'MassiveIntentClassification (af)', 'MassiveIntentClassification (am)', 'MassiveIntentClassification (ar)', 'MassiveIntentClassification (az)', 'MassiveIntentClassification (bn)', 'MassiveIntentClassification (cy)', 'MassiveIntentClassification (de)', 'MassiveIntentClassification (el)', 'MassiveIntentClassification (es)', 'MassiveIntentClassification (fa)', 'MassiveIntentClassification (fi)', 'MassiveIntentClassification (fr)', 'MassiveIntentClassification (he)', 'MassiveIntentClassification (hi)', 'MassiveIntentClassification (hu)', 'MassiveIntentClassification (hy)', 'MassiveIntentClassification (id)', 'MassiveIntentClassification (is)', 'MassiveIntentClassification (it)', 'MassiveIntentClassification (ja)', 'MassiveIntentClassification (jv)', 'MassiveIntentClassification (ka)', 'MassiveIntentClassification (km)', 'MassiveIntentClassification (kn)', 'MassiveIntentClassification (ko)', 'MassiveIntentClassification (lv)', 'MassiveIntentClassification (ml)', 'MassiveIntentClassification (mn)', 'MassiveIntentClassification (ms)', 'MassiveIntentClassification (my)', 'MassiveIntentClassification (nl)', 'MassiveIntentClassification (pt)', 'MassiveIntentClassification (ro)', 'MassiveIntentClassification (ru)', 'MassiveIntentClassification (sl)', 'MassiveIntentClassification (sq)', 'MassiveIntentClassification (sw)', 'MassiveIntentClassification (ta)', 'MassiveIntentClassification (te)', 'MassiveIntentClassification (th)', 'MassiveIntentClassification (tl)', 'MassiveIntentClassification (tr)', 'MassiveIntentClassification (ur)', 'MassiveIntentClassification (vi)', 'MassiveIntentClassification (zh-TW)', 'MassiveScenarioClassification (af)', 'MassiveScenarioClassification (am)', 'MassiveScenarioClassification (ar)', 'MassiveScenarioClassification (az)', 'MassiveScenarioClassification (bn)', 'MassiveScenarioClassification (cy)', 'MassiveScenarioClassification (de)', 'MassiveScenarioClassification (el)', 'MassiveScenarioClassification (es)', 'MassiveScenarioClassification (fa)', 'MassiveScenarioClassification (fi)', 'MassiveScenarioClassification (fr)', 'MassiveScenarioClassification (he)', 'MassiveScenarioClassification (hi)', 'MassiveScenarioClassification (hu)', 'MassiveScenarioClassification (hy)', 'MassiveScenarioClassification (id)', 'MassiveScenarioClassification (is)', 'MassiveScenarioClassification (it)', 'MassiveScenarioClassification (ja)', 'MassiveScenarioClassification (jv)', 'MassiveScenarioClassification (ka)', 'MassiveScenarioClassification (km)', 'MassiveScenarioClassification (kn)', 'MassiveScenarioClassification (ko)', 'MassiveScenarioClassification (lv)', 'MassiveScenarioClassification (ml)', 'MassiveScenarioClassification (mn)', 'MassiveScenarioClassification (ms)', 'MassiveScenarioClassification (my)', 'MassiveScenarioClassification (nl)', 'MassiveScenarioClassification (pt)', 'MassiveScenarioClassification (ro)', 'MassiveScenarioClassification (ru)', 'MassiveScenarioClassification (sl)', 'MassiveScenarioClassification (sq)', 'MassiveScenarioClassification (sw)', 'MassiveScenarioClassification (ta)', 'MassiveScenarioClassification (te)', 'MassiveScenarioClassification (th)', 'MassiveScenarioClassification (tl)', 'MassiveScenarioClassification (tr)', 'MassiveScenarioClassification (ur)', 'MassiveScenarioClassification (vi)', 'MassiveScenarioClassification (zh-TW)'] |
|
|
|
TASK_LIST_CLUSTERING = [ |
|
"ArxivClusteringP2P", |
|
"ArxivClusteringS2S", |
|
"BiorxivClusteringP2P", |
|
"BiorxivClusteringS2S", |
|
"MedrxivClusteringP2P", |
|
"MedrxivClusteringS2S", |
|
"RedditClustering", |
|
"RedditClusteringP2P", |
|
"StackExchangeClustering", |
|
"StackExchangeClusteringP2P", |
|
"TwentyNewsgroupsClustering", |
|
] |
|
|
|
TASK_LIST_PAIR_CLASSIFICATION = [ |
|
"SprintDuplicateQuestions", |
|
"TwitterSemEval2015", |
|
"TwitterURLCorpus", |
|
] |
|
|
|
TASK_LIST_RERANKING = [ |
|
"AskUbuntuDupQuestions", |
|
"MindSmallReranking", |
|
"SciDocsRR", |
|
"StackOverflowDupQuestions", |
|
] |
|
|
|
TASK_LIST_RETRIEVAL = [ |
|
"ArguAna", |
|
"ClimateFEVER", |
|
"CQADupstackRetrieval", |
|
"DBPedia", |
|
"FEVER", |
|
"FiQA2018", |
|
"HotpotQA", |
|
"MSMARCO", |
|
"NFCorpus", |
|
"NQ", |
|
"QuoraRetrieval", |
|
"SCIDOCS", |
|
"SciFact", |
|
"Touche2020", |
|
"TRECCOVID", |
|
] |
|
|
|
TASK_LIST_RETRIEVAL_NORM = TASK_LIST_RETRIEVAL + [ |
|
"CQADupstackAndroidRetrieval", |
|
"CQADupstackEnglishRetrieval", |
|
"CQADupstackGamingRetrieval", |
|
"CQADupstackGisRetrieval", |
|
"CQADupstackMathematicaRetrieval", |
|
"CQADupstackPhysicsRetrieval", |
|
"CQADupstackProgrammersRetrieval", |
|
"CQADupstackStatsRetrieval", |
|
"CQADupstackTexRetrieval", |
|
"CQADupstackUnixRetrieval", |
|
"CQADupstackWebmastersRetrieval", |
|
"CQADupstackWordpressRetrieval" |
|
] |
|
|
|
TASK_LIST_STS = [ |
|
"BIOSSES", |
|
"SICK-R", |
|
"STS12", |
|
"STS13", |
|
"STS14", |
|
"STS15", |
|
"STS16", |
|
"STS17 (en-en)", |
|
"STS22 (en)", |
|
"STSBenchmark", |
|
] |
|
|
|
TASK_LIST_STS_OTHER = ["STS17 (ar-ar)", "STS17 (en-ar)", "STS17 (en-de)", "STS17 (en-tr)", "STS17 (es-en)", "STS17 (es-es)", "STS17 (fr-en)", "STS17 (it-en)", "STS17 (ko-ko)", "STS17 (nl-en)", "STS22 (ar)", "STS22 (de)", "STS22 (de-en)", "STS22 (de-fr)", "STS22 (de-pl)", "STS22 (es)", "STS22 (es-en)", "STS22 (es-it)", "STS22 (fr)", "STS22 (fr-pl)", "STS22 (it)", "STS22 (pl)", "STS22 (pl-en)", "STS22 (ru)", "STS22 (tr)", "STS22 (zh-en)", "STSBenchmark",] |
|
TASK_LIST_STS_NORM = [x.replace(" (en)", "").replace(" (en-en)", "") for x in TASK_LIST_STS] |
|
|
|
TASK_LIST_SUMMARIZATION = ["SummEval",] |
|
|
|
TASK_LIST_EN = TASK_LIST_CLASSIFICATION + TASK_LIST_CLUSTERING + TASK_LIST_PAIR_CLASSIFICATION + TASK_LIST_RERANKING + TASK_LIST_RETRIEVAL + TASK_LIST_STS + TASK_LIST_SUMMARIZATION |
|
|
|
TASK_TO_METRIC = { |
|
"BitextMining": "f1", |
|
"Clustering": "v_measure", |
|
"Classification": "accuracy", |
|
"PairClassification": "cos_sim_ap", |
|
"Reranking": "map", |
|
"Retrieval": "ndcg_at_10", |
|
"STS": "cos_sim_spearman", |
|
"Summarization": "cos_sim_spearman", |
|
} |
|
|
|
def make_clickable_model(model_name, link=None): |
|
if link is None: |
|
link = "https://huggingface.co/" + model_name |
|
|
|
return ( |
|
f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name.split("/")[-1]}</a>' |
|
) |
|
|
|
|
|
EXTERNAL_MODELS = [ |
|
"all-MiniLM-L12-v2", |
|
"all-MiniLM-L6-v2", |
|
"all-mpnet-base-v2", |
|
"allenai-specter", |
|
"bert-base-uncased", |
|
"contriever-base-msmarco", |
|
"dfm-encoder-large-v1", |
|
"dfm-sentence-encoder-large-1", |
|
"distiluse-base-multilingual-cased-v2", |
|
"e5-base", |
|
"e5-large", |
|
"e5-small", |
|
"gbert-base", |
|
"gbert-large", |
|
"gelectra-base", |
|
"gelectra-large", |
|
"gottbert-base", |
|
"glove.6B.300d", |
|
"gtr-t5-base", |
|
"gtr-t5-large", |
|
"gtr-t5-xl", |
|
"gtr-t5-xxl", |
|
"herbert-base-retrieval-v2", |
|
"komninos", |
|
"luotuo-bert-medium", |
|
"LASER2", |
|
"LaBSE", |
|
"m3e-base", |
|
"m3e-large", |
|
"msmarco-bert-co-condensor", |
|
"multilingual-e5-base", |
|
"multilingual-e5-large", |
|
"multilingual-e5-small", |
|
"nb-bert-base", |
|
"nb-bert-large", |
|
"norbert3-base", |
|
"norbert3-large", |
|
"paraphrase-multilingual-MiniLM-L12-v2", |
|
"paraphrase-multilingual-mpnet-base-v2", |
|
"sentence-t5-base", |
|
"sentence-t5-large", |
|
"sentence-t5-xl", |
|
"sentence-t5-xxl", |
|
"sup-simcse-bert-base-uncased", |
|
"text-embedding-3-small", |
|
"text-embedding-3-large", |
|
"text-embedding-3-large-256", |
|
"titan-embed-text-v1", |
|
"unsup-simcse-bert-base-uncased", |
|
"use-cmlm-multilingual", |
|
"voyage-lite-01-instruct", |
|
"voyage-lite-02-instruct", |
|
"xlm-roberta-base", |
|
"xlm-roberta-large", |
|
] |
|
|
|
EXTERNAL_MODEL_TO_LINK = { |
|
"allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter", |
|
"allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter", |
|
"all-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2", |
|
"all-MiniLM-L6-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2", |
|
"all-mpnet-base-v2": "https://huggingface.co/sentence-transformers/all-mpnet-base-v2", |
|
"bert-base-uncased": "https://huggingface.co/bert-base-uncased", |
|
"contriever-base-msmarco": "https://huggingface.co/nthakur/contriever-base-msmarco", |
|
"distiluse-base-multilingual-cased-v2": "https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased-v2", |
|
"dfm-encoder-large-v1": "https://huggingface.co/chcaa/dfm-encoder-large-v1", |
|
"dfm-sentence-encoder-large-1": "https://huggingface.co/chcaa/dfm-encoder-large-v1", |
|
"e5-base": "https://huggingface.co/intfloat/e5-base", |
|
"e5-large": "https://huggingface.co/intfloat/e5-large", |
|
"e5-small": "https://huggingface.co/intfloat/e5-small", |
|
"gbert-base": "https://huggingface.co/deepset/gbert-base", |
|
"gbert-large": "https://huggingface.co/deepset/gbert-large", |
|
"gelectra-base": "https://huggingface.co/deepset/gelectra-base", |
|
"gelectra-large": "https://huggingface.co/deepset/gelectra-large", |
|
"glove.6B.300d": "https://huggingface.co/sentence-transformers/average_word_embeddings_glove.6B.300d", |
|
"gottbert-base": "https://huggingface.co/uklfr/gottbert-base", |
|
"gtr-t5-base": "https://huggingface.co/sentence-transformers/gtr-t5-base", |
|
"gtr-t5-large": "https://huggingface.co/sentence-transformers/gtr-t5-large", |
|
"gtr-t5-xl": "https://huggingface.co/sentence-transformers/gtr-t5-xl", |
|
"gtr-t5-xxl": "https://huggingface.co/sentence-transformers/gtr-t5-xxl", |
|
"herbert-base-retrieval-v2": "https://huggingface.co/ipipan/herbert-base-retrieval-v2", |
|
"komninos": "https://huggingface.co/sentence-transformers/average_word_embeddings_komninos", |
|
"luotuo-bert-medium": "https://huggingface.co/silk-road/luotuo-bert-medium", |
|
"LASER2": "https://github.com/facebookresearch/LASER", |
|
"LaBSE": "https://huggingface.co/sentence-transformers/LaBSE", |
|
"m3e-base": "https://huggingface.co/moka-ai/m3e-base", |
|
"m3e-large": "https://huggingface.co/moka-ai/m3e-large", |
|
"msmarco-bert-co-condensor": "https://huggingface.co/sentence-transformers/msmarco-bert-co-condensor", |
|
"multilingual-e5-base": "https://huggingface.co/intfloat/multilingual-e5-base", |
|
"multilingual-e5-large": "https://huggingface.co/intfloat/multilingual-e5-large", |
|
"multilingual-e5-small": "https://huggingface.co/intfloat/multilingual-e5-small", |
|
"nb-bert-base": "https://huggingface.co/NbAiLab/nb-bert-base", |
|
"nb-bert-large": "https://huggingface.co/NbAiLab/nb-bert-large", |
|
"norbert3-base": "https://huggingface.co/ltg/norbert3-base", |
|
"norbert3-large": "https://huggingface.co/ltg/norbert3-large", |
|
"paraphrase-multilingual-mpnet-base-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2", |
|
"paraphrase-multilingual-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", |
|
"sentence-t5-base": "https://huggingface.co/sentence-transformers/sentence-t5-base", |
|
"sentence-t5-large": "https://huggingface.co/sentence-transformers/sentence-t5-large", |
|
"sentence-t5-xl": "https://huggingface.co/sentence-transformers/sentence-t5-xl", |
|
"sentence-t5-xxl": "https://huggingface.co/sentence-transformers/sentence-t5-xxl", |
|
"sup-simcse-bert-base-uncased": "https://huggingface.co/princeton-nlp/sup-simcse-bert-base-uncased", |
|
"text-embedding-3-small": "https://openai.com/blog/new-embedding-models-and-api-updates", |
|
"text-embedding-3-large": "https://openai.com/blog/new-embedding-models-and-api-updates", |
|
"text-embedding-3-large-256": "https://openai.com/blog/new-embedding-models-and-api-updates", |
|
"titan-embed-text-v1": "https://docs.aws.amazon.com/bedrock/latest/userguide/embeddings.html", |
|
"unsup-simcse-bert-base-uncased": "https://huggingface.co/princeton-nlp/unsup-simcse-bert-base-uncased", |
|
"use-cmlm-multilingual": "https://huggingface.co/sentence-transformers/use-cmlm-multilingual", |
|
"voyage-lite-01-instruct": "https://docs.voyageai.com/embeddings/", |
|
"voyage-lite-02-instruct": "https://docs.voyageai.com/embeddings/", |
|
"xlm-roberta-base": "https://huggingface.co/xlm-roberta-base", |
|
"xlm-roberta-large": "https://huggingface.co/xlm-roberta-large", |
|
} |
|
|
|
EXTERNAL_MODEL_TO_DIM = { |
|
"all-MiniLM-L12-v2": 384, |
|
"all-MiniLM-L6-v2": 384, |
|
"all-mpnet-base-v2": 768, |
|
"allenai-specter": 768, |
|
"bert-base-uncased": 768, |
|
"contriever-base-msmarco": 768, |
|
"distiluse-base-multilingual-cased-v2": 512, |
|
"dfm-encoder-large-v1": 1024, |
|
"dfm-sentence-encoder-large-1": 1024, |
|
"e5-base": 768, |
|
"e5-small": 384, |
|
"e5-large": 1024, |
|
"luotuo-bert-medium": 768, |
|
"LASER2": 1024, |
|
"LaBSE": 768, |
|
"gbert-base": 768, |
|
"gbert-large": 1024, |
|
"gelectra-base": 768, |
|
"gelectra-large": 1024, |
|
"glove.6B.300d": 300, |
|
"gottbert-base": 768, |
|
"gtr-t5-base": 768, |
|
"gtr-t5-large": 768, |
|
"gtr-t5-xl": 768, |
|
"gtr-t5-xxl": 768, |
|
"herbert-base-retrieval-v2": 768, |
|
"komninos": 300, |
|
"m3e-base": 768, |
|
"m3e-large": 768, |
|
"msmarco-bert-co-condensor": 768, |
|
"multilingual-e5-base": 768, |
|
"multilingual-e5-small": 384, |
|
"multilingual-e5-large": 1024, |
|
"nb-bert-base": 768, |
|
"nb-bert-large": 1024, |
|
"norbert3-base": 768, |
|
"norbert3-large": 1024, |
|
"paraphrase-multilingual-MiniLM-L12-v2": 384, |
|
"paraphrase-multilingual-mpnet-base-v2": 768, |
|
"sentence-t5-base": 768, |
|
"sentence-t5-large": 768, |
|
"sentence-t5-xl": 768, |
|
"sentence-t5-xxl": 768, |
|
"sup-simcse-bert-base-uncased": 768, |
|
"text-embedding-3-large": 3072, |
|
"text-embedding-3-large-256": 256, |
|
"text-embedding-3-small": 1536, |
|
"titan-embed-text-v1": 1536, |
|
"unsup-simcse-bert-base-uncased": 768, |
|
"use-cmlm-multilingual": 768, |
|
"voyage-lite-01-instruct": 1024, |
|
"voyage-lite-02-instruct": 1024, |
|
"xlm-roberta-base": 768, |
|
"xlm-roberta-large": 1024, |
|
} |
|
|
|
EXTERNAL_MODEL_TO_SEQLEN = { |
|
"all-MiniLM-L12-v2": 512, |
|
"all-MiniLM-L6-v2": 512, |
|
"all-mpnet-base-v2": 514, |
|
"allenai-specter": 512, |
|
"bert-base-uncased": 512, |
|
"contriever-base-msmarco": 512, |
|
"dfm-encoder-large-v1": 512, |
|
"dfm-sentence-encoder-large-1": 512, |
|
"distiluse-base-multilingual-cased-v2": 512, |
|
"e5-base": 512, |
|
"e5-large": 512, |
|
"e5-small": 512, |
|
"gbert-base": 512, |
|
"gbert-large": 512, |
|
"gelectra-base": 512, |
|
"gelectra-large": 512, |
|
"gottbert-base": 512, |
|
"glove.6B.300d": "N/A", |
|
"gtr-t5-base": 512, |
|
"gtr-t5-large": 512, |
|
"gtr-t5-xl": 512, |
|
"gtr-t5-xxl": 512, |
|
"herbert-base-retrieval-v2": 514, |
|
"komninos": "N/A", |
|
"luotuo-bert-medium": 512, |
|
"LASER2": "N/A", |
|
"LaBSE": 512, |
|
"m3e-base": 512, |
|
"m3e-large": 512, |
|
"msmarco-bert-co-condensor": 512, |
|
"multilingual-e5-base": 514, |
|
"multilingual-e5-large": 514, |
|
"multilingual-e5-small": 512, |
|
"nb-bert-base": 512, |
|
"nb-bert-large": 512, |
|
"norbert3-base": 512, |
|
"norbert3-large": 512, |
|
"paraphrase-multilingual-MiniLM-L12-v2": 512, |
|
"paraphrase-multilingual-mpnet-base-v2": 514, |
|
"sentence-t5-base": 512, |
|
"sentence-t5-large": 512, |
|
"sentence-t5-xl": 512, |
|
"sentence-t5-xxl": 512, |
|
"sup-simcse-bert-base-uncased": 512, |
|
"text-embedding-3-large": 8191, |
|
"text-embedding-3-large-256": 8191, |
|
"text-embedding-3-small": 8191, |
|
"titan-embed-text-v1": 8000, |
|
"use-cmlm-multilingual": 512, |
|
"unsup-simcse-bert-base-uncased": 512, |
|
"voyage-lite-01-instruct": 4000, |
|
"voyage-lite-02-instruct": 4000, |
|
"xlm-roberta-base": 514, |
|
"xlm-roberta-large": 514, |
|
} |
|
|
|
EXTERNAL_MODEL_TO_SIZE = { |
|
"allenai-specter": 0.44, |
|
"all-MiniLM-L12-v2": 0.13, |
|
"all-MiniLM-L6-v2": 0.09, |
|
"all-mpnet-base-v2": 0.44, |
|
"bert-base-uncased": 0.44, |
|
"contriever-base-msmarco": 0.44, |
|
"distiluse-base-multilingual-cased-v2": 0.54, |
|
"dfm-encoder-large-v1": 1.42, |
|
"dfm-sentence-encoder-large-1": 1.63, |
|
"e5-base": 0.44, |
|
"e5-small": 0.13, |
|
"e5-large": 1.34, |
|
"gbert-base": 0.44, |
|
"gbert-large": 1.35, |
|
"gelectra-base": 0.44, |
|
"gelectra-large": 1.34, |
|
"glove.6B.300d": 0.48, |
|
"gottbert-base": 0.51, |
|
"gtr-t5-base": 0.22, |
|
"gtr-t5-large": 0.67, |
|
"gtr-t5-xl": 2.48, |
|
"gtr-t5-xxl": 9.73, |
|
"herbert-base-retrieval-v2": 0.50, |
|
"komninos": 0.27, |
|
"luotuo-bert-medium": 1.31, |
|
"LASER2": 0.17, |
|
"LaBSE": 1.88, |
|
"m3e-base": 0.41, |
|
"m3e-large": 0.41, |
|
"msmarco-bert-co-condensor": 0.44, |
|
"multilingual-e5-base": 1.11, |
|
"multilingual-e5-small": 0.47, |
|
"multilingual-e5-large": 2.24, |
|
"nb-bert-base": 0.71, |
|
"nb-bert-large": 1.42, |
|
"norbert3-base": 0.52, |
|
"norbert3-large": 1.47, |
|
"paraphrase-multilingual-mpnet-base-v2": 1.11, |
|
"paraphrase-multilingual-MiniLM-L12-v2": 0.47, |
|
"sentence-t5-base": 0.22, |
|
"sentence-t5-large": 0.67, |
|
"sentence-t5-xl": 2.48, |
|
"sentence-t5-xxl": 9.73, |
|
"sup-simcse-bert-base-uncased": 0.44, |
|
"unsup-simcse-bert-base-uncased": 0.44, |
|
"use-cmlm-multilingual": 1.89, |
|
"xlm-roberta-base": 1.12, |
|
"xlm-roberta-large": 2.24, |
|
} |
|
|
|
MODELS_TO_SKIP = { |
|
"baseplate/instructor-large-1", |
|
"radames/e5-large", |
|
"gentlebowl/instructor-large-safetensors", |
|
"Consensus/instructor-base", |
|
"GovCompete/instructor-xl", |
|
"GovCompete/e5-large-v2", |
|
"t12e/instructor-base", |
|
"michaelfeil/ct2fast-e5-large-v2", |
|
"michaelfeil/ct2fast-e5-large", |
|
"michaelfeil/ct2fast-e5-small-v2", |
|
"newsrx/instructor-xl-newsrx", |
|
"newsrx/instructor-large-newsrx", |
|
"fresha/e5-large-v2-endpoint", |
|
"ggrn/e5-small-v2", |
|
"michaelfeil/ct2fast-e5-small", |
|
"jncraton/e5-small-v2-ct2-int8", |
|
"anttip/ct2fast-e5-small-v2-hfie", |
|
"newsrx/instructor-large", |
|
"newsrx/instructor-xl", |
|
"dmlls/all-mpnet-base-v2", |
|
"cgldo/semanticClone", |
|
"Malmuk1/e5-large-v2_Sharded", |
|
"jncraton/gte-small-ct2-int8", |
|
"Einas/einas_ashkar", |
|
"gruber/e5-small-v2-ggml", |
|
"jncraton/bge-small-en-ct2-int8", |
|
"vectoriseai/bge-small-en", |
|
"recipe/embeddings", |
|
"dhairya0907/thenlper-get-large", |
|
"Narsil/bge-base-en", |
|
"kozistr/fused-large-en", |
|
"sionic-ai/sionic-ai-v2", |
|
"sionic-ai/sionic-ai-v1", |
|
"BAAI/bge-large-en", |
|
"BAAI/bge-base-en", |
|
"BAAI/bge-small-en", |
|
"d0rj/e5-large-en-ru", |
|
"d0rj/e5-base-en-ru", |
|
"d0rj/e5-small-en-ru", |
|
"aident-ai/bge-base-en-onnx", |
|
"barisaydin/bge-base-en", |
|
"barisaydin/gte-large", |
|
"barisaydin/gte-base", |
|
"barisaydin/gte-small", |
|
"barisaydin/bge-small-en", |
|
"odunola/e5-base-v2", |
|
"goldenrooster/multilingual-e5-large", |
|
"davidpeer/gte-small", |
|
"barisaydin/bge-large-en", |
|
"jamesgpt1/english-large-v1", |
|
"vectoriseai/bge-large-en-v1.5", |
|
"vectoriseai/bge-base-en-v1.5", |
|
"vectoriseai/instructor-large", |
|
"vectoriseai/instructor-base", |
|
"vectoriseai/gte-large", |
|
"vectoriseai/gte-base", |
|
"vectoriseai/e5-large-v2", |
|
"vectoriseai/bge-small-en-v1.5", |
|
"vectoriseai/e5-base-v2", |
|
"vectoriseai/e5-large", |
|
"vectoriseai/multilingual-e5-large", |
|
"vectoriseai/gte-small", |
|
"vectoriseai/ember-v1", |
|
"vectoriseai/e5-base", |
|
"vectoriseai/e5-small-v2", |
|
"michaelfeil/ct2fast-bge-large-en-v1.5", |
|
"michaelfeil/ct2fast-bge-large-en-v1.5", |
|
"michaelfeil/ct2fast-bge-base-en-v1.5", |
|
"michaelfeil/ct2fast-gte-large", |
|
"michaelfeil/ct2fast-gte-base", |
|
"michaelfeil/ct2fast-bge-small-en-v1.5", |
|
"rizki/bgr-tf", |
|
"ef-zulla/e5-multi-sml-torch", |
|
"cherubhao/yogamodel", |
|
"morgendigital/multilingual-e5-large-quantized", |
|
"jncraton/gte-tiny-ct2-int8", |
|
"Research2NLP/electrical_stella", |
|
"Intel/bge-base-en-v1.5-sts-int8-static", |
|
"Intel/bge-base-en-v1.5-sts-int8-dynamic", |
|
"Intel/bge-base-en-v1.5-sst2", |
|
"Intel/bge-base-en-v1.5-sst2-int8-static", |
|
"Intel/bge-base-en-v1.5-sst2-int8-dynamic", |
|
"Intel/bge-small-en-v1.5-sst2", |
|
"Intel/bge-small-en-v1.5-sst2-int8-dynamic", |
|
"Intel/bge-small-en-v1.5-sst2-int8-static", |
|
"binqiangliu/EmbeddingModlebgelargeENv1.5", |
|
"DecisionOptimizationSystem/DeepFeatEmbeddingLargeContext", |
|
"woody72/multilingual-e5-base", |
|
"Severian/embed", |
|
"Frazic/udever-bloom-3b-sentence", |
|
"jamesgpt1/zzz", |
|
"karrar-alwaili/UAE-Large-V1", |
|
"odunola/UAE-Large-VI", |
|
"shubham-bgi/UAE-Large", |
|
"retrainai/instructor-xl", |
|
"weakit-v/bge-base-en-v1.5-onnx", |
|
"ieasybooks/multilingual-e5-large-onnx", |
|
"gizmo-ai/Cohere-embed-multilingual-v3.0", |
|
"jingyeom/korean_embedding_model", |
|
"barisaydin/text2vec-base-multilingual", |
|
"mlx-community/multilingual-e5-large-mlx", |
|
"mlx-community/multilingual-e5-base-mlx", |
|
"mlx-community/multilingual-e5-small-mlx", |
|
"maiyad/multilingual-e5-small", |
|
"khoa-klaytn/bge-base-en-v1.5-angle", |
|
"khoa-klaytn/bge-small-en-v1.5-angle", |
|
"mixamrepijey/instructor-small", |
|
"mixamrepijey/instructor-models", |
|
"lsf1000/bge-evaluation", |
|
} |
|
|
|
EXTERNAL_MODEL_RESULTS = {model: {k: {v: []} for k, v in TASK_TO_METRIC.items()} for model in EXTERNAL_MODELS} |
|
|
|
def add_lang(examples): |
|
if not(examples["eval_language"]): |
|
examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"] |
|
else: |
|
examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"] + f' ({examples["eval_language"]})' |
|
return examples |
|
|
|
def add_task(examples): |
|
|
|
if examples["mteb_dataset_name"] in TASK_LIST_CLASSIFICATION_NORM + TASK_LIST_CLASSIFICATION_SV: |
|
examples["mteb_task"] = "Classification" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_CLUSTERING: |
|
examples["mteb_task"] = "Clustering" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_PAIR_CLASSIFICATION: |
|
examples["mteb_task"] = "PairClassification" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_RERANKING: |
|
examples["mteb_task"] = "Reranking" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_RETRIEVAL_NORM: |
|
examples["mteb_task"] = "Retrieval" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_STS_NORM: |
|
examples["mteb_task"] = "STS" |
|
elif examples["mteb_dataset_name"] in TASK_LIST_SUMMARIZATION: |
|
examples["mteb_task"] = "Summarization" |
|
elif examples["mteb_dataset_name"] in [x.split(" ")[0] for x in TASK_LIST_BITEXT_MINING + TASK_LIST_BITEXT_MINING_OTHER]: |
|
examples["mteb_task"] = "BitextMining" |
|
else: |
|
print("WARNING: Task not found for dataset", examples["mteb_dataset_name"]) |
|
examples["mteb_task"] = "Unknown" |
|
return examples |
|
|
|
pbar = tqdm(EXTERNAL_MODELS, desc="Fetching external model results") |
|
for model in pbar: |
|
pbar.set_description(f"Fetching external model results for {model!r}") |
|
ds = load_dataset("mteb/results", model, trust_remote_code=True) |
|
|
|
|
|
ds = ds.map(add_lang) |
|
ds = ds.map(add_task) |
|
base_dict = {"Model": make_clickable_model(model, link=EXTERNAL_MODEL_TO_LINK.get(model, "https://huggingface.co/spaces/mteb/leaderboard"))} |
|
|
|
for task, metric in TASK_TO_METRIC.items(): |
|
ds_dict = ds.filter(lambda x: (x["mteb_task"] == task) and (x["metric"] == metric))["test"].to_dict() |
|
ds_dict = {k: round(v, 2) for k, v in zip(ds_dict["mteb_dataset_name_with_lang"], ds_dict["score"])} |
|
EXTERNAL_MODEL_RESULTS[model][task][metric].append({**base_dict, **ds_dict}) |
|
|
|
def get_dim_seq_size(model): |
|
filenames = [sib.rfilename for sib in model.siblings] |
|
dim, seq, size = "", "", "" |
|
if "1_Pooling/config.json" in filenames: |
|
st_config_path = hf_hub_download(model.modelId, filename="1_Pooling/config.json") |
|
dim = json.load(open(st_config_path)).get("word_embedding_dimension", "") |
|
elif "2_Pooling/config.json" in filenames: |
|
st_config_path = hf_hub_download(model.modelId, filename="2_Pooling/config.json") |
|
dim = json.load(open(st_config_path)).get("word_embedding_dimension", "") |
|
if "config.json" in filenames: |
|
config_path = hf_hub_download(model.modelId, filename="config.json") |
|
config = json.load(open(config_path)) |
|
if not dim: |
|
dim = config.get("hidden_dim", config.get("hidden_size", config.get("d_model", ""))) |
|
seq = config.get("n_positions", config.get("max_position_embeddings", config.get("n_ctx", config.get("seq_length", "")))) |
|
|
|
if "pytorch_model.bin" in filenames: |
|
url = hf_hub_url(model.modelId, filename="pytorch_model.bin") |
|
meta = get_hf_file_metadata(url) |
|
size = round(meta.size / 1e9, 2) |
|
elif "pytorch_model.bin.index.json" in filenames: |
|
index_path = hf_hub_download(model.modelId, filename="pytorch_model.bin.index.json") |
|
""" |
|
{ |
|
"metadata": { |
|
"total_size": 28272820224 |
|
},.... |
|
""" |
|
size = json.load(open(index_path)) |
|
if ("metadata" in size) and ("total_size" in size["metadata"]): |
|
size = round(size["metadata"]["total_size"] / 1e9, 2) |
|
elif "model.safetensors" in filenames: |
|
url = hf_hub_url(model.modelId, filename="model.safetensors") |
|
meta = get_hf_file_metadata(url) |
|
size = round(meta.size / 1e9, 2) |
|
elif "model.safetensors.index.json" in filenames: |
|
index_path = hf_hub_download(model.modelId, filename="model.safetensors.index.json") |
|
""" |
|
{ |
|
"metadata": { |
|
"total_size": 14483464192 |
|
},.... |
|
""" |
|
size = json.load(open(index_path)) |
|
if ("metadata" in size) and ("total_size" in size["metadata"]): |
|
size = round(size["metadata"]["total_size"] / 1e9, 2) |
|
return dim, seq, size |
|
|
|
def make_datasets_clickable(df): |
|
"""Does not work""" |
|
if "BornholmBitextMining" in df.columns: |
|
link = "https://huggingface.co/datasets/strombergnlp/bornholmsk_parallel" |
|
df = df.rename( |
|
columns={f'BornholmBitextMining': '<a target="_blank" style="text-decoration: underline" href="{link}">BornholmBitextMining</a>',}) |
|
return df |
|
|
|
def add_rank(df): |
|
cols_to_rank = [col for col in df.columns if col not in ["Model", "Model Size (GB)", "Embedding Dimensions", "Max Tokens"]] |
|
if len(cols_to_rank) == 1: |
|
df.sort_values(cols_to_rank[0], ascending=False, inplace=True) |
|
else: |
|
df.insert(1, "Average", df[cols_to_rank].mean(axis=1, skipna=False)) |
|
df.sort_values("Average", ascending=False, inplace=True) |
|
df.insert(0, "Rank", list(range(1, len(df) + 1))) |
|
df = df.round(2) |
|
|
|
df.fillna("", inplace=True) |
|
return df |
|
|
|
def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_emb_dim=False, task_to_metric=TASK_TO_METRIC, rank=True): |
|
api = HfApi() |
|
models = api.list_models(filter="mteb") |
|
|
|
df_list = [] |
|
for model in EXTERNAL_MODEL_RESULTS: |
|
results_list = [res for task in tasks for res in EXTERNAL_MODEL_RESULTS[model][task][task_to_metric[task]]] |
|
if len(datasets) > 0: |
|
res = {k: v for d in results_list for k, v in d.items() if (k == "Model") or any([x in k for x in datasets])} |
|
elif langs: |
|
|
|
langs_format = [f"({lang})" for lang in langs] |
|
res = {k: v for d in results_list for k, v in d.items() if any([k.split(" ")[-1] in (k, x) for x in langs_format])} |
|
else: |
|
res = {k: v for d in results_list for k, v in d.items()} |
|
|
|
if len(res) > 1: |
|
if add_emb_dim: |
|
res["Model Size (GB)"] = EXTERNAL_MODEL_TO_SIZE.get(model, "") |
|
res["Embedding Dimensions"] = EXTERNAL_MODEL_TO_DIM.get(model, "") |
|
res["Max Tokens"] = EXTERNAL_MODEL_TO_SEQLEN.get(model, "") |
|
df_list.append(res) |
|
|
|
for model in models: |
|
if model.modelId in MODELS_TO_SKIP: |
|
continue |
|
print("MODEL", model) |
|
readme_path = hf_hub_download(model.modelId, filename="README.md") |
|
meta = metadata_load(readme_path) |
|
if "model-index" not in meta: |
|
continue |
|
|
|
if len(datasets) > 0: |
|
task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and any([x in sub_res.get("dataset", {}).get("name", "") for x in datasets])] |
|
elif langs: |
|
task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and (sub_res.get("dataset", {}).get("config", "default") in ("default", *langs))] |
|
else: |
|
task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks)] |
|
out = [{res["dataset"]["name"].replace("MTEB ", ""): [round(score["value"], 2) for score in res["metrics"] if score["type"] == task_to_metric.get(res["task"]["type"])][0]} for res in task_results] |
|
out = {k: v for d in out for k, v in d.items()} |
|
out["Model"] = make_clickable_model(model.modelId) |
|
|
|
if len(out) > 1: |
|
if add_emb_dim: |
|
try: |
|
|
|
out["Embedding Dimensions"], out["Max Tokens"], out["Model Size (GB)"] = get_dim_seq_size(model) |
|
except: |
|
pass |
|
df_list.append(out) |
|
df = pd.DataFrame(df_list) |
|
|
|
|
|
df = df.groupby("Model", as_index=False).first() |
|
|
|
cols = sorted(list(df.columns)) |
|
cols.insert(0, cols.pop(cols.index("Model"))) |
|
df = df[cols] |
|
if rank: |
|
df = add_rank(df) |
|
if fillna: |
|
df.fillna("", inplace=True) |
|
return df |
|
|
|
def get_mteb_average(): |
|
global DATA_OVERALL |
|
DATA_OVERALL = get_mteb_data( |
|
tasks=[ |
|
"Classification", |
|
"Clustering", |
|
"PairClassification", |
|
"Reranking", |
|
"Retrieval", |
|
"STS", |
|
"Summarization", |
|
], |
|
datasets=TASK_LIST_CLASSIFICATION + TASK_LIST_CLUSTERING + TASK_LIST_PAIR_CLASSIFICATION + TASK_LIST_RERANKING + TASK_LIST_RETRIEVAL + TASK_LIST_STS + TASK_LIST_SUMMARIZATION, |
|
fillna=False, |
|
add_emb_dim=True, |
|
rank=False, |
|
) |
|
|
|
|
|
|
|
DATA_OVERALL.insert(1, "Average", DATA_OVERALL[TASK_LIST_EN].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(2, "Classification Average", DATA_OVERALL[TASK_LIST_CLASSIFICATION].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(3, "Clustering Average", DATA_OVERALL[TASK_LIST_CLUSTERING].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(4, "Pair Classification Average", DATA_OVERALL[TASK_LIST_PAIR_CLASSIFICATION].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(5, "Reranking Average", DATA_OVERALL[TASK_LIST_RERANKING].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(6, "Retrieval Average", DATA_OVERALL[TASK_LIST_RETRIEVAL].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(7, "STS Average", DATA_OVERALL[TASK_LIST_STS].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.insert(8, "Summarization Average", DATA_OVERALL[TASK_LIST_SUMMARIZATION].mean(axis=1, skipna=False)) |
|
DATA_OVERALL.sort_values("Average", ascending=False, inplace=True) |
|
|
|
DATA_OVERALL.insert(0, "Rank", list(range(1, len(DATA_OVERALL) + 1))) |
|
|
|
|
|
numeric_columns = DATA_OVERALL.select_dtypes(include=['float64', 'int64']).columns |
|
DATA_OVERALL[numeric_columns] = DATA_OVERALL[numeric_columns].fillna(np.nan) |
|
|
|
|
|
string_columns = DATA_OVERALL.select_dtypes(include=['object']).columns |
|
DATA_OVERALL[string_columns] = DATA_OVERALL[string_columns].fillna('') |
|
|
|
DATA_OVERALL = DATA_OVERALL.round(2) |
|
|
|
|
|
DATA_OVERALL.fillna("", inplace=True) |
|
|
|
DATA_OVERALL = DATA_OVERALL[["Rank", "Model", "Model Size (GB)", "Embedding Dimensions", "Max Tokens", "Average", |
|
"Classification Average", "Clustering Average", "Pair Classification Average", "Reranking Average", "Retrieval Average", |
|
"STS Average", "Summarization Average"]] |
|
DATA_OVERALL = DATA_OVERALL[DATA_OVERALL.iloc[:, 5:].ne("").any(axis=1)] |
|
|
|
return DATA_OVERALL |
|
|
|
DATA_OVERALL=get_mteb_average() |
|
|
|
print(type(DATA_OVERALL)) |
|
|
|
import unicodedata |
|
|
|
def is_valid_unicode(char): |
|
try: |
|
unicodedata.name(char) |
|
return True |
|
except ValueError: |
|
return False |
|
|
|
def remove_invalid_unicode(input_string): |
|
if isinstance(input_string, str): |
|
valid_chars = [char for char in input_string if is_valid_unicode(char)] |
|
return ''.join(valid_chars) |
|
else: |
|
return input_string |
|
|
|
from dataclasses import dataclass |
|
|
|
@dataclass |
|
class LeaderboardColumn: |
|
name: str |
|
type: str |
|
|
|
DATA_OVERALL_COLUMN_TO_DATATYPE = [ |
|
LeaderboardColumn("Rank", "number"), |
|
LeaderboardColumn("Model Size (GB)", "number"), |
|
LeaderboardColumn("Embedding Dimensions", "number"), |
|
LeaderboardColumn("Max Tokens", "number"), |
|
LeaderboardColumn("Average", "number"), |
|
LeaderboardColumn("Classification Average", "number"), |
|
LeaderboardColumn("Clustering Average", "number"), |
|
LeaderboardColumn("Pair Classification Average", "number"), |
|
LeaderboardColumn("Reranking Average", "number"), |
|
LeaderboardColumn("Retrieval Average", "number"), |
|
LeaderboardColumn("STS Average", "number"), |
|
LeaderboardColumn("Summarization Average", "number") |
|
] |
|
|
|
COLS = [col.name for col in DATA_OVERALL_COLUMN_TO_DATATYPE] |
|
TYPES = [col.type for col in DATA_OVERALL_COLUMN_TO_DATATYPE] |
|
|
|
data_overall = gr.components.Dataframe( |
|
headers=COLS, |
|
datatype=TYPES, |
|
visible=False, |
|
line_breaks=False, |
|
interactive=False |
|
) |
|
print(data_overall) |
|
|
|
def display(x, y): |
|
global DATA_OVERALL |
|
return DATA_OVERALL |
|
|
|
|
|
dummy1 = gr.Textbox(visible=False) |
|
|
|
INTRODUCTION_TEXT = """ |
|
This is a copied space from LLM Trustworthy Leaderboard. Instead of displaying |
|
the results as table this space was modified to simply provides a gradio API interface. |
|
Using the following python script below, users can access the full leaderboard data easily. |
|
Python on how to access the data: |
|
```python |
|
# Import dependencies |
|
from gradio_client import Client |
|
# Initialize the Gradio client with the API URL |
|
client = Client("https://rodrigomasini-data-only-llm-perf-leaderboard.hf.space/") |
|
try: |
|
# Perform the API call |
|
response = client.predict("","", api_name='/predict') |
|
# Check if response it's directly accessible |
|
if len(response) > 0: |
|
print("Response received!") |
|
headers = response.get('headers', []) |
|
data = response.get('data', []) |
|
print(headers) |
|
# Remove commenst if you want to download the dataset and save in csv format |
|
# Specify the path to your CSV file |
|
#csv_file_path = 'llm-perf-benchmark.csv' |
|
# Open the CSV file for writing |
|
#with open(csv_file_path, mode='w', newline='', encoding='utf-8') as file: |
|
# writer = csv.writer(file) |
|
# Write the headers |
|
# writer.writerow(headers) |
|
# Write the data |
|
# for row in data: |
|
# writer.writerow(row) |
|
#print(f"Results saved to {csv_file_path}") |
|
# If the above line prints a string that looks like JSON, you can parse it with json.loads(response) |
|
# Otherwise, you might need to adjust based on the actual structure of `response` |
|
except Exception as e: |
|
print(f"An error occurred: {e}") |
|
``` |
|
""" |
|
|
|
interface = gr.Interface( |
|
fn=display, |
|
inputs=[gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text"), dummy1], |
|
outputs=[data_overall] |
|
) |
|
|
|
interface.launch() |
