update

fix-requests-cli.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import os
+import fnmatch
+
+import json
+from huggingface_hub import HfApi
+
+
+def find_json_files(directory):
+    matches = []
+    for root, dirnames, filenames in os.walk(directory):
+        for filename in fnmatch.filter(filenames, '*.json'):
+            matches.append(os.path.join(root, filename))
+    return matches
+
+
+directory_path = '/Users/pasquale/workspace/eval/requests'
+json_files = find_json_files(directory_path)
+
+api = HfApi()
+model_lst = api.list_models()
+
+model_lst = [m for m in model_lst]
+
+id_to_model = {m.id: m for m in model_lst}
+
+for path in json_files:
+    with open(path, 'r') as file:
+        data = json.load(file)
+        breakpoint()

src/backend/run_eval_suite.py

@@ -1,5 +1,5 @@
 from lm_eval import tasks, evaluator, utils
-from lm_eval.tasks import initialize_tasks
+from lm_eval.tasks import initialize_tasks, include_task_folder
 
 from src.backend.manage_requests import EvalRequest
 
@@ -12,7 +12,9 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     if limit:
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
+    include_task_folder("src/backend/tasks/")
     initialize_tasks('INFO')
+
     task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
 
     print(f"Selected Tasks: {task_names}")