add-support-for-new-vidore-result-format

data/model_handler.py

@@ -1,12 +1,15 @@
 import json
 import os
-from typing import Dict
-from huggingface_hub import HfApi, hf_hub_download, metadata_load
+from typing import Any, Dict, Tuple
+
 import pandas as pd
-from .dataset_handler import get_datasets_nickname, VIDORE_DATASETS_KEYWORDS
+from huggingface_hub import HfApi, hf_hub_download, metadata_load
+
+from .dataset_handler import VIDORE_DATASETS_KEYWORDS, get_datasets_nickname
 
 BLOCKLIST = ["impactframes"]
 
+
 class ModelHandler:
     def __init__(self, model_infos_path="model_infos.json"):
         self.api = HfApi()
@@ -23,26 +26,40 @@ class ModelHandler:
         with open(self.model_infos_path, "w") as f:
             json.dump(self.model_infos, f)
 
+    def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
+        return "metadata" in results and "metrics" in results
+
+    @staticmethod
+    def convert_new_vidore_results_format(results: Dict[str, Any]) -> Tuple[str, str, Dict[str, Any]]:
+        if "metadata" not in results:
+            raise KeyError("results does not contain a 'metadata' key.")
+        if "metrics" not in results:
+            raise KeyError("results does not contain a 'metrics' key.")
+
+        metadata = results["metadata"]
+        metrics = results["metrics"]
+
+        return metadata["timestamp"], metadata["vidore_benchmark_hash"], metrics
+
     def get_vidore_data(self, metric="ndcg_at_5"):
         models = self.api.list_models(filter="vidore")
         repositories = [model.modelId for model in models]  # type: ignore
 
         for repo_id in repositories:
-            org_name = repo_id.split('/')[0]
+            org_name = repo_id.split("/")[0]
             if org_name in BLOCKLIST:
                 continue
-            
-            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith('_metrics.json') or f == 'results.json']
 
-            
+            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith("_metrics.json") or f == "results.json"]
+
             if len(files) == 0:
                 continue
             else:
                 for file in files:
-                    if file.endswith('results.json'):
-                        model_name = repo_id.replace('/', '_')
+                    if file.endswith("results.json"):
+                        model_name = repo_id.replace("/", "_")
                     else:
-                        model_name = file.split('_metrics.json')[0]
+                        model_name = file.split("_metrics.json")[0]
 
                     if model_name not in self.model_infos:
                         readme_path = hf_hub_download(repo_id, filename="README.md")
@@ -53,15 +70,15 @@ class ModelHandler:
                             with open(result_path) as f:
                                 results = json.load(f)
 
-                            for dataset in results:
-                                results[dataset] = {key: value for key, value in results[dataset].items()}
+                            if self._are_results_in_new_vidore_format(results):
+                                timestamp, vidore_hash, results = self.convert_new_vidore_results_format(results)
 
                             self.model_infos[model_name] = {"meta": meta, "results": results}
                         except Exception as e:
                             print(f"Error loading {model_name} - {e}")
                             continue
 
-        #self._save_model_infos()
+        # self._save_model_infos()
 
         model_res = {}
         if len(self.model_infos) > 0:
@@ -69,7 +86,7 @@ class ModelHandler:
                 res = self.model_infos[model]["results"]
                 dataset_res = {}
                 for dataset in res.keys():
-                    #for each keyword check if it is in the dataset name if not continue
+                    # for each keyword check if it is in the dataset name if not continue
                     if not any(keyword in dataset for keyword in VIDORE_DATASETS_KEYWORDS):
                         print(f"{dataset} not found in ViDoRe datasets. Skipping ...")
                         continue
@@ -77,9 +94,9 @@ class ModelHandler:
                     dataset_nickname = get_datasets_nickname(dataset)
                     dataset_res[dataset_nickname] = res[dataset][metric]
                 model_res[model] = dataset_res
-                
+
             df = pd.DataFrame(model_res).T
-            
+
             return df
         return pd.DataFrame()
 
@@ -104,7 +121,7 @@ class ModelHandler:
             df.insert(len(df.columns) - len(cols_to_rank), "Average", df[cols_to_rank].mean(axis=1, skipna=False))
             df.sort_values("Average", ascending=False, inplace=True)
         df.insert(0, "Rank", list(range(1, len(df) + 1)))
-        #multiply values by 100 if they are floats and round to 1 decimal place
+        # multiply values by 100 if they are floats and round to 1 decimal place
         for col in df.columns:
             if df[col].dtype == "float64":
                 df[col] = df[col].apply(lambda x: round(x * 100, 1))

ruff.toml

@@ -0,0 +1,7 @@
+line-length = 120
+
+[lint]
+select = ["E", "F", "W", "I", "N"]
+
+[lint.per-file-ignores]
+"__init__.py" = ["F401"]