testing now

src/leaderboard/read_evals.py

@@ -16,7 +16,7 @@ from src.submission.check_validity import is_model_on_hub
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
-    eval_name: str # org_model_precision (uid)
+    eval_name: str # org_model_track (uid)
     full_model: str # org/model (path on hub)
     org: str 
     model: str
@@ -33,6 +33,7 @@ class EvalResult:
 
         config = data.get("config")
         track = data.get("track")
+        self.track = track
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
@@ -45,7 +46,7 @@ class EvalResult:
             org = org_and_model[0]
             model = org_and_model[1]
         full_model = "/".join(org_and_model)
-        eval_name = full_model + f" ({track})"
+        eval_name = "_".join(org_and_model) + f"_{track}"
 
         still_on_hub, _, model_config = is_model_on_hub(
             full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
@@ -88,14 +89,14 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.track)
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.org}/{self.model}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -108,13 +109,17 @@ class EvalResult:
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
+        if self.track.lower() == "multimodal":
+            taskset = TasksMultimodal
+        else:
+            taskset = Tasks
+        for task in taskset:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
+def get_request_file_for_model(requests_path, model_name, track):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,
@@ -122,7 +127,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
+    # Select correct request file (track)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
@@ -130,7 +135,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
+                and req_content["track"] == track
             ):
                 request_file = tmp_request_file
     return request_file
@@ -175,4 +180,4 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         except KeyError:  # not all eval values present
             continue
 
-    return results
+    return results