Clean some invalid tasks and columns for when loading the leaderboard and using the refresh button

app.py

@@ -242,8 +242,12 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
     df = df.groupby("Model", as_index=False).first()
     # Put 'Model' column first
     cols = sorted(list(df.columns))
+    base_columns = ["Model", "Model Size (Million Parameters)", "Memory Usage (GB, fp32)", "Embedding Dimensions", "Max Tokens"]
+    if len(datasets) > 0:
+        #filter invalid columns
+        cols = [col for col in cols if col in base_columns + datasets]
     i = 0
-    for column in ["Model", "Model Size (Million Parameters)", "Memory Usage (GB, fp32)", "Embedding Dimensions", "Max Tokens"]:
+    for column in base_columns:
         if column in cols:
             cols.insert(i, cols.pop(cols.index(column)))
             i += 1
@@ -310,6 +314,7 @@ for board, board_config in BOARDS_CONFIG.items():
     else:
         for task_category, task_category_list in board_config["tasks"].items():
             data_task_category = get_mteb_data(tasks=[task_category], datasets=task_category_list, refresh=False)
+            data_task_category.drop(columns=["Embedding Dimensions", "Max Tokens"], inplace=True)
             boards_data[board]["data_tasks"][task_category] = data_task_category
             all_data_tasks.append(data_task_category)
 
@@ -364,6 +369,13 @@ Each inner tab can have the following keys:
 - refresh: The function to refresh the leaderboard
 """
 
+def get_refresh_function(task_category, task_list):
+    def _refresh():
+        data_task_category = get_mteb_data(tasks=[task_category], datasets=task_list)
+        data_task_category.drop(columns=["Embedding Dimensions", "Max Tokens"], inplace=True)
+        return data_task_category
+    return _refresh
+
 data = {
     "Overall": {"metric": "Various, refer to task tabs", "data": []}
 }
@@ -399,7 +411,7 @@ for board, board_config in BOARDS_CONFIG.items():
             "language_long": board_config["language_long"],
             "description": f"**{task_category} {board_pretty_name}** {task_icon}{board_icon}",
             "data": boards_data[board]["data_tasks"][task_category],
-            "refresh": partial(get_mteb_data, tasks=[task_category], datasets=task_category_list),
+            "refresh": get_refresh_function(task_category, task_category_list),
             "credits": credits,
         })