update

backend-cli.py

@@ -74,7 +74,7 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
     results = run_evaluation(eval_request=eval_request, task_names=[task.benchmark], num_fewshot=task.num_fewshot,
-                             batch_size=1, device=DEVICE, use_cache=False, limit=LIMIT)
+                             batch_size=1, device=DEVICE, use_cache=None, limit=LIMIT)
 
     dumped = json.dumps(results, indent=2)
     print(dumped)

src/backend/run_eval_suite.py

@@ -6,7 +6,7 @@ import logging
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 
-def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=False, limit=None) -> dict:
+def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None) -> dict:
     if limit:
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")