Automatic Evaluation Available. Read the README filegit add add_new_model/README.md add_new_model/add_new_model.py app.py add_new_model/execute_evaluation.py

add_new_model/README.md

@@ -2,11 +2,15 @@
 
 Here we are going to explain how to add a new model to the Leaderboard. The next steps must be followed:
 1. `Git clone` this repository and `cd add_new_model`. 
-2. Perform evaluation of the new model using MTEB library. That will give you a folder as result. Example: `sentence-t5-large` is the folder name
-when evaluating `sentence-transformers/sentence-t5-large`. 
-3. Once evaluated, move that folder to this folder, so it will be inside `add_new_model` folder.
-4. Execute the file `MTEB_metadata_to_yaml.py`. That will create a file named `mteb_medadata.yaml` that contains the metadata regarding your evaluation.
-5. Execute the file `add_new_model.py`. That file will add your model to the Leaderboard.
-6. Add, commit and `git push` the changes without uploading the results and the `mteb_metadata.yaml`. 
-7. It is recommended to launch the app by running `python3 app.py` from parent folder, and confirm that there are no errors in the leaderboard and we
-are uploading it as we wanted. 
+2. If you want to evaluate the model and add it to the Leaderboard, you just need to execute the following command:
+    `python3 add_new_model.py --model_id <HF_model> --execute_eval True`
+This command will execute the evaluation in your runtime. That will create 2 elements: a results folder in which will store all the information regarding the evaluation process and a file called `mteb_metadata.yaml` that contains the processed metadata regarding your evaluation process. 
+That metadata is read and re-ordered and added to the three CSV files that you can find in the `data` folder of this repository. The `app.py` file will read those CSV files, so you must be very careful with them.
+3. If you do not want to evaluate the model because you already have done it, you just need to execute the following command:
+`python3 add_new_model.py --output_folder <results_folder>`. That folder is where you have your JSON files containing the previous evaluation you did using MTEB library. If you select any different subfolder, then the code will raise an error (i.e if you select a parent folder of the actual results folder). This methodology will run very fast. 
+4. After you added your model to the Leaderboard, remember you must execute two more commands:
+`rm -r <results_folder>`
+`rm mteb_metadata.yaml`
+We do not need that, so we won't accept any PR that contains those files. 
+5. Once the previous steps are done, you can execute `python3 app.py` from the parent folder of this repository and you will be able to see the Leaderboard before pushing it to the hub. 
+6. Add, commit and `git push` the changes. Remember not to push your results.  

add_new_model/add_new_model.py

@@ -1,9 +1,15 @@
-
 import pandas as pd
 import yaml
 import numpy as np
+import argparse
+from execute_evaluation import evaluate
+import logging
+import os
+import json
+import sys
+from mteb import MTEB
 
-def add_model(metadata_archive):
+def add_model():
     """
     Esto actualiza el archivo del cual app.py coge la información para crear la leaderboard. Entonces, cuando
     alguien quiera añadir un nuevo modelo, tiene que ejecutar este archivo. 
@@ -88,5 +94,126 @@ def add_model(metadata_archive):
     general_dataframe = pd.concat([general_dataframe, new_row_df], ignore_index=True)
     general_dataframe.to_csv("../data/general.csv",index=False)
 
-add_model('mteb_metadata.yaml')
 
+def results_to_yaml(results_folder):
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
+
+    
+    model_name = results_folder.split("/")[-1]
+
+    all_results = {}
+
+    for file_name in os.listdir(results_folder):
+        if not file_name.endswith(".json"):
+            logger.info(f"Skipping non-json {file_name}")
+            raise ValueError("This is not the proper folder. It does not contain the corresponding Json files.")
+            continue
+        with open(os.path.join(results_folder, file_name), "r", encoding="utf-8") as f:
+            results = json.load(f)
+            all_results = {**all_results, **{file_name.replace(".json", ""): results}}
+
+    # Use "train" split instead
+    TRAIN_SPLIT = ["DanishPoliticalCommentsClassification"]
+    # Use "validation" split instead
+    VALIDATION_SPLIT = ["AFQMC", "Cmnli", "IFlyTek", "TNews", "MSMARCO", "MultilingualSentiment", "Ocnli"]
+    # Use "dev" split instead
+    DEV_SPLIT = ["CmedqaRetrieval", "CovidRetrieval", "DuRetrieval", "EcomRetrieval", "MedicalRetrieval", "MMarcoReranking", "MMarcoRetrieval", "MSMARCO", "T2Reranking", "T2Retrieval", "VideoRetrieval"]
+
+    MARKER = "---"
+    TAGS = "tags:"
+    MTEB_TAG = "- mteb"
+    HEADER = "model-index:"
+    MODEL = f"- name: {model_name}"
+    RES = "  results:"
+
+    META_STRING = "\n".join([MARKER, TAGS, MTEB_TAG, HEADER, MODEL, RES])
+
+
+    ONE_TASK = "  - task:\n      type: {}\n    dataset:\n      type: {}\n      name: {}\n      config: {}\n      split: {}\n      revision: {}\n    metrics:"
+    ONE_METRIC = "    - type: {}\n      value: {}"
+    SKIP_KEYS = ["std", "evaluation_time", "main_score", "threshold"]
+
+    for ds_name, res_dict in sorted(all_results.items()):
+        mteb_desc = (
+            MTEB(tasks=[ds_name.replace("CQADupstackRetrieval", "CQADupstackAndroidRetrieval")]).tasks[0].description
+        )
+        hf_hub_name = mteb_desc.get("hf_hub_name", mteb_desc.get("beir_name"))
+        if "CQADupstack" in ds_name:
+            hf_hub_name = "BeIR/cqadupstack"
+        mteb_type = mteb_desc["type"]
+        revision = res_dict.get("dataset_revision")  # Okay if it's None
+        split = "test"
+        if (ds_name in TRAIN_SPLIT) and ("train" in res_dict):
+            split = "train"
+        elif (ds_name in VALIDATION_SPLIT) and ("validation" in res_dict):
+            split = "validation"
+        elif (ds_name in DEV_SPLIT) and ("dev" in res_dict):
+            split = "dev"
+        elif "test" not in res_dict:
+            logger.info(f"Skipping {ds_name} as split {split} not present.")
+            continue
+        res_dict = res_dict.get(split)
+        for lang in mteb_desc["eval_langs"]:
+            mteb_name = f"MTEB {ds_name}"
+            mteb_name += f" ({lang})" if len(mteb_desc["eval_langs"]) > 1 else ""
+            # For English there is no language key if it's the only language
+            test_result_lang = res_dict.get(lang) if len(mteb_desc["eval_langs"]) > 1 else res_dict
+            # Skip if the language was not found but it has other languages
+            if test_result_lang is None:
+                continue
+            META_STRING += "\n" + ONE_TASK.format(
+                mteb_type, hf_hub_name, mteb_name, lang if len(mteb_desc["eval_langs"]) > 1 else "default", split, revision
+            )
+            for metric, score in test_result_lang.items():
+                if not isinstance(score, dict):
+                    score = {metric: score}
+                for sub_metric, sub_score in score.items():
+                    if any([x in sub_metric for x in SKIP_KEYS]):
+                        continue
+                    META_STRING += "\n" + ONE_METRIC.format(
+                        f"{metric}_{sub_metric}" if metric != sub_metric else metric,
+                        # All MTEB scores are 0-1, multiply them by 100 for 3 reasons:
+                        # 1) It's easier to visually digest (You need two chars less: "0.1" -> "1")
+                        # 2) Others may multiply them by 100, when building on MTEB making it confusing what the range is
+                        # This happend with Text and Code Embeddings paper (OpenAI) vs original BEIR paper
+                        # 3) It's accepted practice (SuperGLUE, GLUE are 0-100)
+                        sub_score * 100,
+                    )
+
+    META_STRING += "\n" + MARKER
+    if os.path.exists(f"./mteb_metadata.yaml"):
+        logger.warning("Overwriting mteb_metadata.md")
+    with open(f"./mteb_metadata.yaml", "w") as f:
+        f.write(META_STRING)
+
+
+def main():
+    
+    if args.execute_eval:
+        output_folder = evaluate(args.model_id)
+        results_to_yaml(output_folder)
+        add_model()
+    else:
+        print('Hola')
+        print(args.output_folder)
+        if args.output_folder == None:
+            raise ValueError("You must indicate where your results are located")
+        else: 
+            results_to_yaml(args.output_folder)
+            add_model()
+
+
+if __name__ == "__main__":
+    
+    parser = argparse.ArgumentParser(description="Select the model that you want to add to the Leaderboard.")
+    parser.add_argument("--model_id", type=str, required=True, help="HuggingFace model path that you want to evaluate.")
+    parser.add_argument("--execute_eval",type=bool, default=False, help="Select if you want to execute evaluation.")
+    parser.add_argument("--output_folder", type=str, help = "Select the folder in which the results are stored.")
+    args = parser.parse_args()
+    main()
+
+    
+    

add_new_model/execute_evaluation.py

@@ -0,0 +1,39 @@
+import huggingface_hub
+import os
+import argparse
+import logging
+import torch
+from mteb import MTEB
+from sentence_transformers import SentenceTransformer
+from huggingface_hub import login
+
+try:
+    huggingface_hub.login(
+        token=os.environ["HUGGINGFACE_TOKEN"], write_permission=True
+    )
+except Exception as e:
+    print(f"Original error: {e}")
+
+def get_device():
+    return "cuda" if torch.cuda.is_available() else "cpu"
+
+def load_model(model_id, device):
+    model = SentenceTransformer(model_id).to(device)
+    logging.info(f"Loaded model {model_id} to the device {device}")
+    return model
+
+def run_evaluation(model, model_id,output_folder):
+    evaluation = MTEB(task_langs=["es"])
+    evaluation.run(model, output_folder=output_folder, eval_splits=["test"])
+
+def evaluate(model_id):
+    logging.basicConfig(level=logging.INFO)
+    output_folder = f'results/{model_id}'
+    device = get_device()
+    model = load_model(model_id, device)
+    run_evaluation(model, model_id,output_folder)
+    
+    return output_folder
+
+
+

app.py

@@ -34,6 +34,17 @@ def clustering_dataframe_update():
 def retrieval_dataframe_update():
     pass
 
+def make_clickable_model(link):
+    """
+    Load json from models. Este update lo tengo que hacer antes de pasarle el df al gradio. 
+    """
+    model_display_name = link.split("/")[-1]
+    # Remove user from model name
+    return (
+        f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_display_name.split("/")[-1]}</a>'
+    )
+    
+
 block = gr.Blocks()
 with block:
     gr.Markdown(f"""**Leaderboard de modelos de Embeddings en español