Added clustering task

add_new_model/add_new_model.py

@@ -18,7 +18,7 @@ def add_model():
     
     """
     # Initialize an empty DataFrame
-    df = pd.DataFrame(columns=['dataset_name', 'Accuracy', 'Spearman', "Category"])
+    df = pd.DataFrame(columns=['dataset_name', 'Accuracy', 'Spearman','V_measure', 'Category'])
 
     metadata_archive = 'mteb_metadata.yaml'
 
@@ -46,17 +46,27 @@ def add_model():
                             spearman = next((metric.get('value') for metric in results_list[i].get('metrics', []) if metric.get('type') == 'cos_sim_spearman'), None)
                             row['Spearman'] = spearman
                             row["Category"] = "STS"
-                        
+                        elif task_name == "Clustering":
+                            v_measure =  next((metric.get('value') for metric in results_list[i].get('metrics', []) if metric.get('type') == 'v_measure'), None)
+                            row['V_measure'] = v_measure
+                            row['Category'] = "Clustering"
                         # Append the row to the DataFrame using pd.concat
                         new_df = pd.DataFrame([row])
                         df = pd.concat([df, new_df], ignore_index=True)
 
+    print(df)
+    
+
     df['Accuracy'] = pd.to_numeric(df['Accuracy'], errors='coerce')
     classification_average = round(df.loc[df['Category'] == 'Classification', 'Accuracy'].mean(),2)
 
     df['Spearman'] = pd.to_numeric(df['Spearman'], errors='coerce')
     sts_spearman_average = round(df.loc[df['Category'] == 'STS', 'Spearman'].mean(),2)
 
+    df['V_measure'] = pd.to_numeric(df['V_measure'], errors='coerce')
+    clustering_v_measure_average = round(df.loc[df['Category'] == 'Clustering', 'V_measure'].mean(),2)
+
+    
     ## CLASSIFICATION
     classification_dataframe = pd.read_csv('../data/classification.csv')
     classification_df = df[df['Category']== 'Classification']
@@ -84,12 +94,26 @@ def add_model():
     sts_dataframe = pd.concat([sts_dataframe,new_row_df],ignore_index=True)
     sts_dataframe.to_csv('../data/sts.csv',index=False)
 
+    ## Clustering 
+    clustering_dataframe = pd.read_csv("../data/clustering.csv")
+    clustering_df = df[df['Category']=='Clustering']
+    new_row_data = {'Model name': model_name, 'Average': clustering_v_measure_average}
+    for index, row in clustering_df.iterrows():
+        column_name = row['dataset_name']
+        v_measure_value = row['V_measure']
+        new_row_data[column_name] = round(v_measure_value,2)
+
+    new_row_df = pd.DataFrame(new_row_data,index = [0])
+    clustering_dataframe = pd.concat([clustering_dataframe,new_row_df],ignore_index=True)
+    clustering_dataframe.to_csv('../data/clustering.csv',index=False)
+
+
     ## GENERAL
     general_dataframe = pd.read_csv("../data/general.csv")
 
     average = round(np.mean([classification_average,sts_spearman_average]),2)
     ## TODO: solucionar la meta-data como Model Size o Embedding Dimensions. 
-    new_instance = {'Model name':model_name, 'Model Size (GB)': None, 'Embedding Dimensions': None, 'Average':average, 'Classification Average': classification_average, 'Clustering Average': None, 'STS Average': sts_spearman_average, 'Retrieval Average': None}
+    new_instance = {'Model name':model_name, 'Model Size (GB)': None, 'Embedding Dimensions': None, 'Average':average, 'Classification Average': classification_average, 'Clustering Average': clustering_v_measure_average, 'STS Average': sts_spearman_average, 'Retrieval Average': None}
     new_row_df = pd.DataFrame(new_instance, index=[0])
     general_dataframe = pd.concat([general_dataframe, new_row_df], ignore_index=True)
     general_dataframe.to_csv("../data/general.csv",index=False)
@@ -191,17 +215,17 @@ def results_to_yaml(results_folder):
 
 
 def main():
-    
     if args.execute_eval:
         output_folder = evaluate(args.model_id)
-        results_to_yaml(output_folder)
+        #results_to_yaml(output_folder)
         add_model()
     else:
-        if args.output_folder == None:
+        if args.output_folder == None and args.already_yaml == False:
             raise ValueError("You must indicate where your results are located")
         else: 
-            results_to_yaml(args.output_folder)
+            #results_to_yaml(args.output_folder)
             add_model()
+            print('Model added')
 
 
 if __name__ == "__main__":
@@ -210,6 +234,7 @@ if __name__ == "__main__":
     parser.add_argument("--model_id", type=str, required=True, help="HuggingFace model path that you want to evaluate.")
     parser.add_argument("--execute_eval",type=bool, default=False, help="Select if you want to execute evaluation.")
     parser.add_argument("--output_folder", type=str, help = "Select the folder in which the results are stored.")
+    parser.add_argument("--already_yaml",default=False, help="Select if you already have the yaml file.")
     args = parser.parse_args()
     main()
 

data/clustering.csv

@@ -0,0 +1,3 @@
+Model name,Average,MTEB BiorxivClusteringS2S_Spanish,MTEB RedditClusteringSpanish
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,33.86,26.64,41.07
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,33.86,26.64,41.07