fix eval_name for non main revision models

.gitignore

@@ -4,7 +4,7 @@ __pycache__/
 .ipynb_checkpoints
 *ipynb
 .vscode/
-run_dot_env.sh
+*.sh
 hub/
 modules/
 original_results/

src/leaderboard/read_evals.py

@@ -58,19 +58,24 @@ class EvalResult:
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
         num_params = round(config.get("model_num_parameters", 0) / 1_000_000_000, 2)
+        revision = config.get("model_revision", "main")
 
         # Get model and org
         org_and_model = config.get("model_name")
         org_and_model = org_and_model.split("/", 1)
 
+        prefix = f"{precision.value.name}"
+        if revision != "main":
+           prefix = f"{revision}_{prefix}"
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}_{prefix}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}_{prefix}"
+        
         full_model = "/".join(org_and_model)
 
         # Extract results available in this file (some results are split in several files)
@@ -123,7 +128,7 @@ class EvalResult:
             results=results,
             precision=precision, 
             model_sha=config.get("model_sha", ""),
-            revision=config.get("model_revision", "main"),
+            revision=revision,
             json_filename=json_filename,
             eval_time=config.get("total_evaluation_time_seconds", 0.0),
             num_params=num_params
@@ -146,7 +151,7 @@ class EvalResult:
             self.main_language = request.get("main_language", "?")
         except Exception as e:
             self.status = "FAILED"
-            print(f"Could not find request file for {self.org}/{self.model}, precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.org}/{self.model}, precision {self.precision.value.name}, revision {self.revision}")
 
     def update_with_dynamic_file_dict(self, file_dict):
         self.license = file_dict.get("license", "?")
@@ -219,7 +224,7 @@ def get_request_file_for_model(requests_path, model_name, precision, revision):
 
     if revision is None or revision == "":
         revision = "main"
-
+        
     # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)

src/submission/submit.py

@@ -60,7 +60,7 @@ def add_new_eval(
         return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
 
     # Does the model actually exist?
-    if revision == "":
+    if revision == "" or revision is None:
         revision = "main"
 
     # Is the model on the hub?
@@ -150,7 +150,7 @@ def add_new_eval(
         out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
     else:
         rev_name = revision
-        if "rev_name" > 30:
+        if len(rev_name) > 12:
             rev_name = rev_name[:7]
         out_path = f"{OUT_DIR}/{model_path}_eval_request_{rev_name}_{private}_{precision}_{weight_type}.json"
 

upload_initial_queue.py

@@ -13,6 +13,8 @@ with open('initial_queue.jsonl', 'r', encoding='utf-8') as outfile:
         if not line.startswith('//'):
             data = json.loads(line)
             model_id = f'{data["model"]}_eval_request_False_{data["precision"]}_{data["weight_type"]}'
+            if data["revision"] != "main":
+                model_id = f'{data["model"]}_eval_request_{data["revision"]}_False_{data["precision"]}_{data["weight_type"]}'
             model_path = os.path.join(EVAL_REQUESTS_PATH, model_id + '.json')
             if not os.path.exists(model_path):
                 print(f"Adding new eval {model_id}")
@@ -24,7 +26,8 @@ with open('initial_queue.jsonl', 'r', encoding='utf-8') as outfile:
                     private=False,
                     weight_type=data["weight_type"],
                     model_type=data["model_type"],
-                    source="script"
+                    main_language=data["main_language"],
+                    source="manual"
                 )
                 print(message)