Update app.py

app.py

@@ -106,10 +106,12 @@ def evaluate(
     max_as_limit: int = 30 * 1024,
     max_data_limit: int = 30 * 1024,
     max_stack_limit: int = 10,
+    calibrated: bool = True,
     check_gt_only: bool = False,
     no_gt: bool = False,
+    selective_evaluate: str = "",
 ):
-    pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
+    passk = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
     if parallel < 1:
         n_workers = max(1, multiprocessing.cpu_count() // 2)
     else:
@@ -121,6 +123,14 @@ def evaluate(
     extra = subset + "_" if subset != "full" else ""
 
     problems = get_bigcodebench(subset=subset)
+    
+    # Add selective evaluation logic
+    if selective_evaluate:
+        selected_ids = ["BigCodeBench/" + id for id in sorted(set(selective_evaluate.split(",")))]
+        problems = {k: v for k, v in problems.items() if k in selected_ids}
+        if not problems:
+            raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset")
+
     dataset_hash = get_bigcodebench_hash(subset=subset)
     
     if not no_gt:
@@ -156,7 +166,7 @@ def evaluate(
                     if "solution" in sample
                     else problems[task_id]["complete_prompt"] + sample["completion"]
                 )
-                if "sanitized_calibrated" in samples:
+                if calibrated:
                     solution = problems[task_id]["code_prompt"] + "\n    pass\n" + solution
                 remainings.add(sample["_identifier"])
                 args = (
@@ -213,7 +223,7 @@ def evaluate(
 
         pass_at_k.update({
             f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
-            for k in pass_k
+            for k in passk
             if total.min() >= k
         })
 
@@ -223,7 +233,7 @@ def evaluate(
     pass_at_k["model"] = os.path.basename(samples).split("--bigcodebench-")[0]
     pass_at_k["split"] = split
     pass_at_k["subset"] = subset
-    pass_at_k["calibrated"] = "sanitized_calibrated" in samples
+    pass_at_k["calibrated"] = calibrated
     pass_at_k["gt_pass_rate"] = gt_pass_rate
     pass_at_k["failed_tasks"] = failed_tasks
     
@@ -243,8 +253,10 @@ interface = gr.Interface(
         gr.Slider(1, 100 * 1024, step=1024, label="Max AS Limit", value=30 * 1024),
         gr.Slider(1, 100 * 1024, step=1024, label="Max Data Limit", value=30 * 1024),
         gr.Slider(1, 100, step=1, label="Max Stack Limit", value=10),
+        gr.Checkbox(label="Calibrated", value=True),
         gr.Checkbox(label="Check GT Only"),
         gr.Checkbox(label="No GT"),
+        gr.Textbox(label="Selective Evaluated Task IDs (comma-separated, e.g. '0,1,2')", value=""),
     ],
     outputs=[
         gr.JSON(label="Results"),
@@ -271,8 +283,14 @@ def restart_space():
 
 
 # if __name__ == "__main__":
-preload_gt()
+while True:
+    try:
+        preload_gt()
+        break
+    except:
+        continue
+
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", hours=5)  # Restart every 5hs
+scheduler.add_job(restart_space, "interval", hours=1)  # Restart every 2hs
 scheduler.start()
 interface.launch(show_error=True)