update

backend-cli.py

@@ -74,7 +74,7 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
     results = run_evaluation(eval_request=eval_request, task_names=[task.benchmark], num_fewshot=task.num_fewshot,
-                             batch_size=1, device=DEVICE, no_cache=True, limit=LIMIT)
+                             batch_size=1, device=DEVICE, use_cache=False, limit=LIMIT)
 
     dumped = json.dumps(results, indent=2)
     print(dumped)

src/backend/run_eval_suite.py

@@ -6,7 +6,7 @@ import logging
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 
-def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, no_cache=True, limit=None) -> dict:
+def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=False, limit=None) -> dict:
     if limit:
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
@@ -17,7 +17,7 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     results = evaluator.simple_evaluate(model="hf-causal-experimental",  # "hf-causal"
                                         model_args=eval_request.get_model_args(),
                                         tasks=task_names, num_fewshot=num_fewshot,
-                                        batch_size=batch_size, device=device, no_cache=no_cache,
+                                        batch_size=batch_size, device=device, ise_cache=use_cache,
                                         limit=limit, write_out=True, output_base_path="logs")
 
     results["config"]["model_dtype"] = eval_request.precision