Added retrieval and revaluated every model

add_new_model/add_new_model.py

@@ -18,7 +18,7 @@ def add_model():
     
     """
     # Initialize an empty DataFrame
-    df = pd.DataFrame(columns=['dataset_name', 'Accuracy', 'Spearman','V_measure', 'Category'])
+    df = pd.DataFrame(columns=['dataset_name', 'Accuracy', 'Spearman','V_measure', 'ndcg_at_10', 'Category'])
 
     metadata_archive = 'mteb_metadata.yaml'
 
@@ -36,7 +36,7 @@ def add_model():
                         dataset_name = results_list[i]['dataset']['name']
                         
                         # Initialize the row with NaN values
-                        row = {'dataset_name': dataset_name, 'Accuracy': None, 'Spearman': None}
+                        row = {'dataset_name': dataset_name, 'Accuracy': None, 'Spearman': None, 'V_measure': None, 'ndcg_at_10': None}
                         
                         if task_name == "Classification":
                             accuracy = next((metric.get('value') for metric in results_list[i].get('metrics', []) if metric.get('type') == 'accuracy'), None)
@@ -50,6 +50,10 @@ def add_model():
                             v_measure =  next((metric.get('value') for metric in results_list[i].get('metrics', []) if metric.get('type') == 'v_measure'), None)
                             row['V_measure'] = v_measure
                             row['Category'] = "Clustering"
+                        elif task_name == "Retrieval":
+                            ndcg_at_10 =  next((metric.get('value') for metric in results_list[i].get('metrics', []) if metric.get('type') == 'ndcg_at_10'), None)
+                            row['ndcg_at_10'] = ndcg_at_10
+                            row['Category'] = "Retrieval"
                         # Append the row to the DataFrame using pd.concat
                         new_df = pd.DataFrame([row])
                         df = pd.concat([df, new_df], ignore_index=True)
@@ -66,6 +70,9 @@ def add_model():
     df['V_measure'] = pd.to_numeric(df['V_measure'], errors='coerce')
     clustering_v_measure_average = round(df.loc[df['Category'] == 'Clustering', 'V_measure'].mean(),2)
 
+    df['ndcg_at_10'] = pd.to_numeric(df['ndcg_at_10'], errors='coerce')
+    retrieval_average = round(df.loc[df['Category'] == 'Retrieval', 'ndcg_at_10'].mean(),2)
+
     
     ## CLASSIFICATION
     classification_dataframe = pd.read_csv('../data/classification.csv')
@@ -107,13 +114,27 @@ def add_model():
     clustering_dataframe = pd.concat([clustering_dataframe,new_row_df],ignore_index=True)
     clustering_dataframe.to_csv('../data/clustering.csv',index=False)
 
+    ## Retrieval
+    retrieval_dataframe = pd.read_csv("../data/retrieval.csv")
+    retrieval_df = df[df['Category']=='Retrieval']
+    new_row_data = {'Model name': model_name, 'Average': retrieval_average}
+    for index, row in retrieval_df.iterrows():
+        column_name = row['dataset_name']
+        ndcg_at_10_value = row['ndcg_at_10']
+        new_row_data[column_name] = round(ndcg_at_10_value,2)
+
+    new_row_df = pd.DataFrame(new_row_data,index = [0])
+    retrieval_dataframe = pd.concat([retrieval_dataframe,new_row_df],ignore_index=True)
+    retrieval_dataframe.to_csv('../data/retrieval.csv',index=False)
+
+
 
     ## GENERAL
     general_dataframe = pd.read_csv("../data/general.csv")
 
-    average = round(np.mean([classification_average,sts_spearman_average]),2)
+    average = round(np.mean([classification_average,sts_spearman_average,clustering_v_measure_average,retrieval_average]),2)
     ## TODO: solucionar la meta-data como Model Size o Embedding Dimensions. 
-    new_instance = {'Model name':model_name, 'Model Size (GB)': None, 'Embedding Dimensions': None, 'Average':average, 'Classification Average': classification_average, 'Clustering Average': clustering_v_measure_average, 'STS Average': sts_spearman_average, 'Retrieval Average': None}
+    new_instance = {'Model name':model_name, 'Model Size (GB)': None, 'Embedding Dimensions': None, 'Average':average, 'Classification Average': classification_average, 'Clustering Average': clustering_v_measure_average, 'STS Average': sts_spearman_average, 'Retrieval Average': retrieval_average}
     new_row_df = pd.DataFrame(new_instance, index=[0])
     general_dataframe = pd.concat([general_dataframe, new_row_df], ignore_index=True)
     general_dataframe.to_csv("../data/general.csv",index=False)
@@ -215,26 +236,26 @@ def results_to_yaml(results_folder):
 
 
 def main():
-    if args.execute_eval:
-        output_folder = evaluate(args.model_id)
-        #results_to_yaml(output_folder)
-        add_model()
-    else:
-        if args.output_folder == None and args.already_yaml == False:
-            raise ValueError("You must indicate where your results are located")
-        else: 
-            #results_to_yaml(args.output_folder)
-            add_model()
-            print('Model added')
+#     if args.execute_eval:
+#         output_folder = evaluate(args.model_id)
+#         #results_to_yaml(output_folder)
+#         add_model()
+#     else:
+    # if args.output_folder == None and args.already_yaml == False:
+    #     raise ValueError("You must indicate where your results are located")
+    # else: 
+        #results_to_yaml(args.output_folder)
+    add_model()
+    print('Model added')
 
 
 if __name__ == "__main__":
     
     parser = argparse.ArgumentParser(description="Select the model that you want to add to the Leaderboard.")
-    parser.add_argument("--model_id", type=str, required=True, help="HuggingFace model path that you want to evaluate.")
-    parser.add_argument("--execute_eval",type=bool, default=False, help="Select if you want to execute evaluation.")
-    parser.add_argument("--output_folder", type=str, help = "Select the folder in which the results are stored.")
-    parser.add_argument("--already_yaml",default=False, help="Select if you already have the yaml file.")
+    #parser.add_argument("--model_id", type=str, required=True, help="HuggingFace model path that you want to evaluate.")
+    #parser.add_argument("--execute_eval",type=bool, default=False, help="Select if you want to execute evaluation.")
+    #parser.add_argument("--output_folder", type=str, help = "Select the folder in which the results are stored.")
+    #parser.add_argument("--already_yaml",default=False, help="Select if you already have the yaml file.")
     args = parser.parse_args()
     main()
 

app.py

@@ -33,9 +33,14 @@ def clustering_dataframe_update():
     Returns clustering dataframe for clustering table. 
     """
     dataframe = pd.read_csv("data/clustering.csv")
+    return dataframe 
 
 def retrieval_dataframe_update():
-    pass
+    """
+    Returns retrieval dataframe for retrieval table.
+    """
+    dataframe = pd.read_csv('data/retrieval.csv')
+    return dataframe
 
 def make_clickable_model(link):
     """
@@ -92,7 +97,7 @@ with block:
         with gr.TabItem("STS"):
             with gr.Row():
                     gr.Markdown("""
-                    **Tabla Classification de Embeddings**
+                    **Tabla STS de Embeddings**
                     
                     - **Metricas:** Spearman correlation basada en cosine similarity. 
                     - **Idioma:** Español
@@ -108,7 +113,7 @@ with block:
         with gr.TabItem("Clustering"):
             with gr.Row():
                     gr.Markdown("""
-                    **Tabla Classification de Embeddings**
+                    **Tabla Clustering de Embeddings**
                     
                     - **Metricas:** V_measure.
                     - **Idioma:** Español
@@ -122,6 +127,13 @@ with block:
                         wrap=True,
                     )
         with gr.TabItem("Retrieval"):
+            with gr.Row():
+                    gr.Markdown("""
+                    **Tabla Retrieval de Embeddings**
+                    
+                    - **Metricas:** ncdg_10.
+                    - **Idioma:** Español
+                    """)
             with gr.Row():
                 # Create and display a sample DataFrame
                 sts = retrieval_dataframe_update()

data/classification.csv

@@ -1,30 +1,15 @@
 Model name,Average,MTEB AmazonReviewsClassification (es),MTEB MTOPDomainClassification (es),MTEB MTOPIntentClassification (es),MTEB MassiveIntentClassification (es),MTEB MassiveScenarioClassification (es)
-multilingual-e5-LARGE,66.59,42.66,89.95,66.84,64.68,68.85
-bge-SMALL-en-v1.5,52.86,32.03,76.93,52.15,48.77,54.42
-multilingual-e5-BASE,63.87,42.47,89.62,60.27,60.51,66.52
-multilingual-e5-SMALL,61.13,41.3,87.33,55.87,58.06,63.1
-paraphrase-multilingual-mpnet-BASE-v2,65.67,39.99,86.96,66.59,64.43,70.42
-sentence-t5-LARGE,57.87,42.89,80.78,52.07,54.1,59.56
-sentence-t5-xl,61.64,45.01,85.32,57.38,57.97,62.52
-paraphrase-spanish-distilroberta,63.98,38.24,86.81,65.94,60.52,68.39
-sentence_similarity_spanish_es,61.77,35.08,85.86,65.21,58.51,64.21
-paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,64.0,37.25,86.93,66.28,62.6,66.96
-mstsb-paraphrase-multilingual-mpnet-BASE-v2,64.47,38.29,86.04,67.06,63.47,67.53
-multilingual-e5-BASE-b16-e10,65.09,43.4,89.02,61.7,63.06,68.25
-multilingual-e5-LARGE-stsb-tuned-b32-e10,66.19,43.31,89.3,64.04,64.62,69.69
-multilingual-e5-LARGE-stsb-tuned-b64-e10,64.58,43.71,88.84,60.2,62.74,67.4
-LaBSE,61.97,39.39,84.07,64.44,58.32,63.61
-multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,64.39,43.04,88.81,59.25,63.04,67.8
-bge-BASE-tuned-b16-e10,50.83,31.34,74.1,45.63,48.72,54.36
-multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,64.93,42.98,87.96,60.91,63.78,69.02
-bge-LARGE-tuned-b16-e10,51.67,31.72,76.04,48.01,48.9,53.66
-multilingual-e5-LARGE-stsb-tuned-b16-e15,67.56,43.35,90.98,67.27,65.35,70.87
-text2vec-base-multilingual,58.64,34.82,79.24,58.69,58.28,62.2
-paraphrase-multilingual-MiniLM-L12-v2,61.1,37.49,83.04,60.28,59.66,65.04
-text2vec-stsb-tuned-b16-e5,57.19,34.64,77.72,56.46,56.94,60.22
-paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,60.41,37.28,82.83,60.45,58.56,62.96
+bge-large-en-v1.5,52.27,34.63,78.8,47.4,45.58,54.95
+e5-large-v2,56.15,40.01,82.74,49.66,49.82,58.5
+multilingual-e5-base,63.88,42.47,89.62,60.27,60.51,66.52
 multilingual-e5-LARGE-tuned-double-dataset-b16-e5,62.65,42.33,85.21,57.01,62.09,66.61
-multilingual-e5-LARGE-stsb-tuned-b16-e5,66.33,43.02,89.71,64.4,64.8,69.72
-multilingual-e5-LARGE-stsb-tuned-b16-e10,66.77,43.86,89.45,64.87,65.13,70.51
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,64.23,44.25,86.83,59.76,63.08,67.21
-multilingual-e5-LARGE-double-finetuning-b16-e5,63.55,44.23,86.21,57.96,62.58,66.75
+multilingual-e5-large,66.6,42.66,89.95,66.84,64.68,68.85
+paraphrase-multilingual-mpnet-base-v2,65.68,39.99,86.96,66.59,64.43,70.42
+paraphrase-spanish-distilroberta,63.98,38.25,86.81,65.95,60.51,68.39
+retromae_es_67000,58.27,33.14,84.09,62.75,51.93,59.45
+RetroMAE-finetuned-stsb_multi_es_aug_gpt3.5-turbo_v2,59.63,29.7,87.64,60.15,55.43,65.22
+sentence_similarity_spanish_es,61.77,35.08,85.86,65.21,58.49,64.21
+sentence-t5-xl,61.64,45.01,85.32,57.38,57.97,62.52
+bert-base-spanish-wwm-uncased,57.35,33.94,79.68,61.96,52.54,58.61
+bertin-roberta-base-spanish,28.57,23.4,39.95,22.77,25.32,31.41
+roberta-base-bne,56.27,34.15,78.91,63.17,50.3,54.82

data/clustering.csv

@@ -1,3 +1,15 @@
 Model name,Average,MTEB BiorxivClusteringS2S_Spanish,MTEB RedditClusteringSpanish
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,33.86,26.64,41.07
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,33.86,26.64,41.07
+bge-large-en-v1.5,34.55,27.41,41.69
+e5-large-v2,30.97,23.39,38.55
+multilingual-e5-base,32.28,25.09,39.47
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,33.34,26.57,40.11
+multilingual-e5-large,34.37,28.1,40.65
+paraphrase-multilingual-mpnet-base-v2,35.9,28.27,43.53
+paraphrase-spanish-distilroberta,34.82,26.14,43.5
+retromae_es_67000,19.57,19.78,19.36
+RetroMAE-finetuned-stsb_multi_es_aug_gpt3.5-turbo_v2,32.17,24.06,40.27
+sentence_similarity_spanish_es,28.58,22.84,34.33
+sentence-t5-xl,32.34,21.56,43.12
+bert-base-spanish-wwm-uncased,21.2,18.19,24.2
+bertin-roberta-base-spanish,4.72,4.13,5.32
+roberta-base-bne,21.44,19.22,23.66

data/general.csv

@@ -1,30 +1,15 @@
 Model name,Model Size (GB),Embedding Dimensions,Average,Classification Average,Clustering Average,STS Average,Retrieval Average
-multilingual-e5-LARGE,,,72.22,66.6,,77.83,
-bge-SMALL-en-v1.5,,,59.73,52.86,,66.6,
-multilingual-e5-BASE,,,70.7,63.88,,77.53,
-multilingual-e5-SMALL,,,68.64,61.13,,76.15,
-paraphrase-multilingual-mpnet-BASE-v2,,,69.1,65.68,,72.53,
-sentence-t5-LARGE,,,64.04,57.88,,70.21,
-sentence-t5-xl,,,66.22,61.64,,70.79,
-paraphrase-spanish-distilroberta,,,69.34,63.98,,74.7,
-sentence_similarity_spanish_es,,,68.5,61.77,,75.22,
-paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,,,68.62,64.0,,73.25,
-mstsb-paraphrase-multilingual-mpnet-BASE-v2,,,69.39,64.48,,74.29,
-multilingual-e5-BASE-b16-e10,,,71.97,65.09,,78.86,
-multilingual-e5-LARGE-stsb-tuned-b32-e10,,,72.73,66.19,,79.27,
-multilingual-e5-LARGE-stsb-tuned-b64-e10,,,71.83,64.58,,79.08,
-LaBSE,,,66.99,61.97,,72.01,
-multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,,,71.88,64.39,,79.38,
-bge-BASE-tuned-b16-e10,,,59.69,50.83,,68.55,
-multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,,,72.22,64.93,,79.5,
-bge-LARGE-tuned-b16-e10,,,61.5,51.67,,71.34,
-multilingual-e5-LARGE-stsb-tuned-b16-e15,,,73.38,67.56,,79.19,
-text2vec-base-multilingual,,,66.04,58.64,,73.43,
-paraphrase-multilingual-MiniLM-L12-v2,,,66.08,61.1,,71.06,
-text2vec-stsb-tuned-b16-e5,,,66.0,57.19,,74.81,
-paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,,,68.21,60.41,,76.01,
-multilingual-e5-LARGE-tuned-double-dataset-b16-e5,,,71.19,62.65,,79.74,
-multilingual-e5-LARGE-stsb-tuned-b16-e5,,,72.9,66.33,,79.47,
-multilingual-e5-LARGE-stsb-tuned-b16-e10,,,73.03,66.77,,79.29,
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,,,71.84,64.23,,79.44,
-multilingual-e5-LARGE-double-finetuning-b16-e5,,,71.53,63.55,,79.5,
+bge-large-en-v1.5,,,45.25,52.27,34.55,68.77,25.41
+e5-large-v2,,,45.87,56.15,30.97,71.76,24.6
+multilingual-e5-base,,,51.14,63.88,32.28,77.53,30.85
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,,,50.87,62.65,33.34,79.74,27.75
+multilingual-e5-large,,,52.58,66.6,34.37,77.83,31.5
+paraphrase-multilingual-mpnet-base-v2,,,50.47,65.68,35.9,72.53,27.78
+paraphrase-spanish-distilroberta,,,50.6,63.98,34.82,74.7,28.92
+retromae_es_67000,,,40.79,58.27,19.57,67.64,17.67
+RetroMAE-finetuned-stsb_multi_es_aug_gpt3.5-turbo_v2,,,47.71,59.63,32.17,74.3,24.73
+sentence_similarity_spanish_es,,,47.22,61.77,28.58,75.22,23.3
+sentence-t5-xl,,,47.42,61.64,32.34,70.79,24.91
+bert-base-spanish-wwm-uncased,,,37.28,57.35,21.2,57.38,13.17
+bertin-roberta-base-spanish,,,22.13,28.57,4.72,47.67,7.56
+roberta-base-bne,,,37.25,56.27,21.44,59.49,11.79

data/retrieval.csv

@@ -0,0 +1,15 @@
+Model name,Average,MTEB ArguAna,MTEB SCIDOCS
+bge-large-en-v1.5,25.41,41.27,9.54
+e5-large-v2,24.6,37.67,11.52
+multilingual-e5-base,30.85,50.36,11.34
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,27.75,41.45,14.05
+multilingual-e5-large,31.5,51.06,11.93
+paraphrase-multilingual-mpnet-base-v2,27.78,43.66,11.89
+paraphrase-spanish-distilroberta,28.92,46.06,11.77
+retromae_es_67000,17.67,28.78,6.56
+RetroMAE-finetuned-stsb_multi_es_aug_gpt3.5-turbo_v2,24.73,39.47,9.99
+sentence_similarity_spanish_es,23.3,40.07,6.52
+sentence-t5-xl,24.91,38.55,11.27
+bert-base-spanish-wwm-uncased,13.17,23.93,2.42
+bertin-roberta-base-spanish,7.56,14.84,0.29
+roberta-base-bne,11.79,21.24,2.33

data/sts.csv

@@ -1,30 +1,15 @@
 Model name,Average,MTEB STS17 (es-es),MTEB STS22 (es)
-multilingual-e5-LARGE,77.82,87.42,68.23
-bge-SMALL-en-v1.5,66.6,77.73,55.47
-multilingual-e5-BASE,77.52,87.26,67.79
-multilingual-e5-SMALL,76.15,85.27,67.04
-paraphrase-multilingual-mpnet-BASE-v2,72.52,85.14,59.91
-sentence-t5-LARGE,70.21,82.74,57.68
-sentence-t5-xl,70.78,83.42,58.16
+bge-large-en-v1.5,68.77,79.62,57.92
+e5-large-v2,71.76,82.16,61.36
+multilingual-e5-base,77.53,87.26,67.79
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,79.74,88.98,70.5
+multilingual-e5-large,77.83,87.42,68.23
+paraphrase-multilingual-mpnet-base-v2,72.53,85.14,59.91
 paraphrase-spanish-distilroberta,74.7,85.79,63.61
+retromae_es_67000,67.64,73.01,62.26
+RetroMAE-finetuned-stsb_multi_es_aug_gpt3.5-turbo_v2,74.3,84.77,63.83
 sentence_similarity_spanish_es,75.22,85.37,65.07
-paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,73.24,86.89,59.6
-mstsb-paraphrase-multilingual-mpnet-BASE-v2,74.28,88.22,60.36
-multilingual-e5-BASE-b16-e10,78.86,87.51,70.21
-multilingual-e5-LARGE-stsb-tuned-b32-e10,79.27,88.1,70.44
-multilingual-e5-LARGE-stsb-tuned-b64-e10,79.08,88.03,70.12
-LaBSE,72.01,80.83,63.18
-multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,79.38,87.77,70.99
-bge-BASE-tuned-b16-e10,68.55,82.01,55.08
-multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,79.5,88.66,70.35
-bge-LARGE-tuned-b16-e10,71.34,81.57,61.12
-multilingual-e5-LARGE-stsb-tuned-b16-e15,79.19,89.04,69.34
-text2vec-base-multilingual,73.43,87.91,58.94
-paraphrase-multilingual-MiniLM-L12-v2,71.06,85.56,56.56
-text2vec-stsb-tuned-b16-e5,74.81,88.28,61.34
-paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,76.01,88.03,63.99
-multilingual-e5-LARGE-tuned-double-dataset-b16-e5,79.74,88.98,70.5
-multilingual-e5-LARGE-stsb-tuned-b16-e5,79.47,88.72,70.21
-multilingual-e5-LARGE-stsb-tuned-b16-e10,79.29,89.1,69.48
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,79.44,88.78,70.11
-multilingual-e5-LARGE-double-finetuning-b16-e5,79.5,88.63,70.37
+sentence-t5-xl,70.79,83.42,58.16
+bert-base-spanish-wwm-uncased,57.38,63.61,51.14
+bertin-roberta-base-spanish,47.67,41.87,53.46
+roberta-base-bne,59.49,70.75,48.24