Merge pull request #33 from Samoed/fix_leaderboard2

config.yaml

@@ -23,7 +23,7 @@ tasks:
     metric: max_ap
     metric_description: "Average Precision (AP) based on the models similarity metric (usually cosine)"
     task_description: "Pair classification is the task of determining whether two texts are similar."
-  Reranking: 
+  Reranking:
     icon: "🥈"
     metric: map
     metric_description: "Mean Average Precision (MAP)"
@@ -345,35 +345,35 @@ boards:
     credits: "[Roman Solomatin](https://github.com/Samoed) and SaluteDevices: [Alena Fenogenova](https://github.com/Alenush), [Aleksandr Abramov](https://github.com/Ab1992ao), [Artem Snegirev](https://github.com/artemsnegirev), [Anna Maksimova](https://github.com/anpalmak2003), [Maria Tikhonova](https://github.com/MariyaTikhonova)"
     tasks:
       Classification:
-        - GeoreviewClassification (rus-Cyrl)
-        - HeadlineClassification (rus-Cyrl)
-        - InappropriatenessClassification (rus-Cyrl)
-        - KinopoiskClassification (rus-Cyrl)
-        - RuReviewsClassification (rus-Cyrl)
-        - RuSciBenchGRNTIClassification (rus-Cyrl)
-        - RuSciBenchOECDClassification (rus-Cyrl)
-        - MassiveIntentClassification (rus-Cyrl)
-        - MassiveScenarioClassification (rus-Cyrl)
+        - GeoreviewClassification
+        - HeadlineClassification
+        - InappropriatenessClassification
+        - KinopoiskClassification
+        - RuReviewsClassification
+        - RuSciBenchGRNTIClassification
+        - RuSciBenchOECDClassification
+        - MassiveIntentClassification (ru)
+        - MassiveScenarioClassification (ru)
       Clustering:
-        - GeoreviewClusteringP2P (rus-Cyrl)
-        - RuSciBenchGRNTIClusteringP2P (rus-Cyrl)
-        - RuSciBenchOECDClusteringP2P (rus-Cyrl)
+        - GeoreviewClusteringP2P
+        - RuSciBenchGRNTIClusteringP2P
+        - RuSciBenchOECDClusteringP2P
       PairClassification:
-        - TERRa (rus-Cyrl)
+        - TERRa
       Reranking:
-        - RuBQReranking (rus-Cyrl)
-        - MIRACLReranking (rus-Cyrl)
+        - RuBQReranking
+        - MIRACLReranking (ru)
       Retrieval:
-        - RiaNewsRetrieval (rus-Cyrl)
-        - RuBQRetrieval (rus-Cyrl)
-        - MIRACLRetrieval (rus-Cyrl)
+        - RiaNewsRetrieval
+        - RuBQRetrieval
+        - MIRACLRetrieval (ru)
       STS:
-        - RUParaPhraserSTS (rus-Cyrl)
-        - RuSTSBenchmarkSTS (rus-Cyrl)
-        - STS22 (rus-Cyrl)
+        - RUParaPhraserSTS
+        - RuSTSBenchmarkSTS
+        - STS22 (ru)
       MultilabelClassification:
-        - CEDRClassification (rus-Cyrl)
-        - SensitiveTopicsClassification (rus-Cyrl)
+        - CEDRClassification
+        - SensitiveTopicsClassification
   se:
     title: Swedish
     language_long: Swedish
@@ -530,23 +530,23 @@ boards:
     metric: nDCG@10
     tasks:
       Retrieval:
-        - AppsRetrieval (eng-Latn_python-Code)
-        - CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)
-        - CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)
-        - CodeSearchNetCCRetrieval (python-Code)
-        - CodeSearchNetCCRetrieval (javascript-Code)
-        - CodeSearchNetCCRetrieval (go-Code)
-        - CodeSearchNetCCRetrieval (ruby-Code)
-        - CodeSearchNetCCRetrieval (java-Code)
-        - CodeSearchNetCCRetrieval (php-Code)
-        - CodeSearchNetRetrieval (python-Code)
-        - CodeSearchNetRetrieval (javascript-Code)
-        - CodeSearchNetRetrieval (go-Code)
-        - CodeSearchNetRetrieval (ruby-Code)
-        - CodeSearchNetRetrieval (java-Code)
-        - CodeSearchNetRetrieval (php-Code)
-        - CodeTransOceanContest (python-Code_c++-Code)
+        - AppsRetrieval
+        - CodeFeedbackMT
+        - CodeFeedbackST
+        - CodeSearchNetCCRetrieval (python)
+        - CodeSearchNetCCRetrieval (javascript)
+        - CodeSearchNetCCRetrieval (go)
+        - CodeSearchNetCCRetrieval (ruby)
+        - CodeSearchNetCCRetrieval (java)
+        - CodeSearchNetCCRetrieval (php)
+        - CodeSearchNetRetrieval (python)
+        - CodeSearchNetRetrieval (javascript)
+        - CodeSearchNetRetrieval (go)
+        - CodeSearchNetRetrieval (ruby)
+        - CodeSearchNetRetrieval (java)
+        - CodeSearchNetRetrieval (php)
+        - CodeTransOceanContest
         - CodeTransOceanDL
-        - CosQA (eng-Latn_python-Code)
+        - CosQA
         - StackOverflowQA
-        - SyntheticText2SQL (eng-Latn_sql-Code)
+        - SyntheticText2SQL

refresh.py

@@ -131,12 +131,12 @@ def make_clickable_model(model_name: str, link: None | str = None) -> str:
     return f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name.split("/")[-1]}</a>'
 
 
-def add_lang(examples):
-    if not (examples["eval_language"]) or (examples["eval_language"] == "default"):
+def add_subset(examples):
+    if not (examples["hf_subset"]) or (examples["hf_subset"] == "default"):
         examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"]
     else:
         examples["mteb_dataset_name_with_lang"] = (
-            examples["mteb_dataset_name"] + f' ({examples["eval_language"]})'
+            examples["mteb_dataset_name"] + f' ({examples["hf_subset"]})'
         )
     return examples
 
@@ -265,7 +265,7 @@ def get_external_model_results():
             print(f"Can't fined model {model} in results repository. Exception: {e}")
             continue
 
-        ds = ds.map(add_lang)
+        ds = ds.map(add_subset)
         ds = ds.map(add_task)
         base_dict = {
             "Model": make_clickable_model(
@@ -313,7 +313,7 @@ def get_external_model_results():
 
     # Save & cache EXTERNAL_MODEL_RESULTS
     with open("EXTERNAL_MODEL_RESULTS.json", "w") as f:
-        json.dump(EXTERNAL_MODEL_RESULTS, f, indent=4)
+        json.dump(dict(sorted(EXTERNAL_MODEL_RESULTS.items())), f, indent=4)
 
     return EXTERNAL_MODEL_RESULTS
 
@@ -332,6 +332,10 @@ def download_or_use_cache(modelId: str):
     return meta
 
 
+def simplify_dataset_name(name):
+    return name.replace("MTEB ", "").replace(" (default)", "")
+
+
 def get_mteb_data(
     tasks: list = ["Clustering"],
     langs: list = [],
@@ -450,11 +454,11 @@ def get_mteb_data(
         try:
             out = [
                 {
-                    res["dataset"]["name"].replace("MTEB ", ""): [
+                    simplify_dataset_name(res["dataset"]["name"]): [
                         round(score["value"], 2)
                         for score in res["metrics"]
                         if filter_metric_fetched(
-                            res["dataset"]["name"].replace("MTEB ", ""),
+                            simplify_dataset_name(res["dataset"]["name"]),
                             score["type"],
                             task_to_metric.get(res["task"]["type"]),
                             res["dataset"]["split"],