solve submission issue (#11)

- fix a bug (0e4197b4226198a6f4400b6a6d6b8f6d0ed6505f)
- Solve submission issue (99ee183e6eaa42fbbb1a5c2cbd66d1f4011e1267)
- fix submit issue (e2cfb382e91631ba85d0f20e0307e2915acdfe30)

src/backend/run_eval_suite.py

@@ -12,7 +12,7 @@ from src.backend.tasks.cnndm.task_v2 import CNNDMv2
 from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
 
 from src.backend.huggingface_generate_until import HFLMwithChatTemplate
-from src.backend.moe_infinity import MoEHFLM  # MoEInfinity
+from src.backend.moe_infinity import MoEHFLM
 
 def run_evaluation(
     eval_request: EvalRequest,

src/submission/check_validity.py

@@ -130,7 +130,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -113,13 +113,14 @@ def add_new_eval(
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if f"{model}_{revision}_{precision}_{inference_framework}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+    # out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}_{inference_framework}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))