add local run mode

app.py

@@ -1,7 +1,3 @@
-# import os
-# os.environ['CURL_CA_BUNDLE'] = ''
-#
-#
 import subprocess
 
 subprocess.run(["python", "scripts/fix_harness_import.py"])
@@ -33,7 +29,7 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, DEVICE, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, DEVICE, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, RUN_MODE
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
@@ -46,21 +42,22 @@ def launch_backend():
     _ = subprocess.run(["python", "main_backend.py"])
 
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
+if RUN_MODE != "LOCAL":
+    try:
+        print(f"Downloading {EVAL_REQUESTS_PATH}")
+        snapshot_download(
+            repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+            token=TOKEN
+        )
+    except Exception:
+        restart_space()
+    try:
+        print(f"Downloading {EVAL_RESULTS_PATH}")
+        snapshot_download(
+            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        )
+    except Exception:
+        restart_space()
 
 _ = subprocess.run(["python", "main_backend.py"])
 
@@ -359,4 +356,4 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.add_job(launch_backend, "interval", seconds=100) # will only allow one job to be run at the same time
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(share=True)

main_backend.py

@@ -1,18 +1,22 @@
 import logging
+import os
 import pprint
 
 from huggingface_hub import snapshot_download
 import subprocess
+
 subprocess.run(["python", "scripts/fix_harness_import.py"])
 
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 from src.backend.run_eval_suite import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request, EvalRequest
 from src.backend.sort_queue import sort_models_by_priority
 
-from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
+from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, \
+    LIMIT, TOKEN, RUN_MODE
 from src.about import Tasks, NUM_FEWSHOT
+
 TASKS_HARNESS = [task.value.benchmark for task in Tasks]
 
 logging.basicConfig(level=logging.ERROR)
@@ -23,9 +27,11 @@ RUNNING_STATUS = "RUNNING"
 FINISHED_STATUS = "FINISHED"
 FAILED_STATUS = "FAILED"
 
+
 # TODO: uncomment
-snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
-snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
+if RUN_MODE != "LOCAL":
+    snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
 
 def run_auto_eval():
     current_pending_status = [PENDING_STATUS]
@@ -33,21 +39,33 @@ def run_auto_eval():
     # pull the eval dataset from the hub and parse any eval requests
     # check completed evals and set them to finished
 
-    check_completed_evals(
-        api=API,
-        checked_status=RUNNING_STATUS,
-        completed_status=FINISHED_STATUS,
-        failed_status=FAILED_STATUS,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-        hf_repo_results=RESULTS_REPO,
-        local_dir_results=EVAL_RESULTS_PATH_BACKEND
-    )
-
-    # Get all eval request that are PENDING, if you want to run other evals, change this parameter
-    eval_requests = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
-    # Sort the evals by priority (first submitted first run)
-    eval_requests = sort_models_by_priority(api=API, models=eval_requests)
+    if RUN_MODE != "LOCAL":
+        check_completed_evals(
+            api=API,
+            checked_status=RUNNING_STATUS,
+            completed_status=FINISHED_STATUS,
+            failed_status=FAILED_STATUS,
+            hf_repo=QUEUE_REPO,
+            local_dir=EVAL_REQUESTS_PATH_BACKEND,
+            hf_repo_results=RESULTS_REPO,
+            local_dir_results=EVAL_RESULTS_PATH_BACKEND
+        )
+        # Get all eval request that are PENDING, if you want to run other evals, change this parameter
+        eval_requests = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO,
+                                          local_dir=EVAL_REQUESTS_PATH_BACKEND)
+        # Sort the evals by priority (first submitted first run)
+        eval_requests = sort_models_by_priority(api=API, models=eval_requests)
+
+    else:
+        local_model_name = os.getenv("LOCAL_MODEL_NAME", "hf-internal-testing/tiny-random-gpt2")
+        sample_request = {
+            "model": local_model_name, "json_filepath": "", "base_model": "", "revision": "main",
+            "private": False,
+            "precision": "bfloat16", "weight_type": "Original", "status": "PENDING",
+            "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0,
+            "params": 0.1, "license": "custom"
+        }
+        eval_requests = [EvalRequest(**sample_request)]
 
     print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
 
@@ -57,27 +75,28 @@ def run_auto_eval():
     eval_request = eval_requests[0]
     pp.pprint(eval_request)
 
-    set_eval_request(
-        api=API,
-        eval_request=eval_request,
-        set_to_status=RUNNING_STATUS,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-    )
+    if RUN_MODE != "LOCAL":
+        set_eval_request(
+            api=API,
+            eval_request=eval_request,
+            set_to_status=RUNNING_STATUS,
+            hf_repo=QUEUE_REPO,
+            local_dir=EVAL_REQUESTS_PATH_BACKEND,
+        )
 
     run_evaluation(
-        eval_request=eval_request, 
-        task_names=TASKS_HARNESS, 
-        num_fewshot=NUM_FEWSHOT, 
+        eval_request=eval_request,
+        task_names=TASKS_HARNESS,
+        num_fewshot=NUM_FEWSHOT,
         local_dir=EVAL_RESULTS_PATH_BACKEND,
         results_repo=RESULTS_REPO,
-        batch_size=1, 
-        device=DEVICE, 
-        no_cache=True, 
+        batch_size=1,
+        device=DEVICE,
+        no_cache=True,
         limit=LIMIT
-        )
+    )
     logging.info("Shopping finished")
 
 
 if __name__ == "__main__":
-    run_auto_eval()
+    run_auto_eval()

scripts/create_request_file.py

@@ -7,7 +7,7 @@ from datetime import datetime, timezone
 import click
 from colorama import Fore
 from huggingface_hub import HfApi, snapshot_download
-from src.envs import TOKEN, EVAL_REQUESTS_PATH, QUEUE_REPO
+from src.envs import TOKEN, EVAL_REQUESTS_PATH, QUEUE_REPO, RUN_MODE
 
 precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ", "float32")
 model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
@@ -34,6 +34,7 @@ def get_model_size(model_info, precision: str):
 def main():
     api = HfApi()
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
     snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", token=TOKEN)
 
     model_name = click.prompt("Enter model name")
@@ -90,13 +91,14 @@ def main():
         with open(out_path, "w") as f:
             f.write(json.dumps(eval_entry))
 
-        api.upload_file(
-            path_or_fileobj=out_path,
-            path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1],
-            repo_id=QUEUE_REPO,
-            repo_type="dataset",
-            commit_message=f"Add {model_name} to eval queue",
-        )
+        if RUN_MODE != "LOCAL":
+            api.upload_file(
+                path_or_fileobj=out_path,
+                path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1],
+                repo_id=QUEUE_REPO,
+                repo_type="dataset",
+                commit_message=f"Add {model_name} to eval queue",
+            )
     else:
         click.echo("aborting...")
 

src/about.py

@@ -16,6 +16,8 @@ class Tasks(Enum):
     # task1 = Task("logiqa", "acc_norm", "LogiQA")
     response_subjective_score = Task("response_subjective_score", "response_subjective_score", "Subjective Response Score")
     product_id_precision_score = Task("product_id_precision_score", "product_id_precision_score", "Product ID Precision Score")
+    mmlu_anatomy = Task("mmlu_flan_n_shot_generative_anatomy", "mmlu_flan_n_shot_generative_anatomy", "MMLU (anatomy)")
+    mmly_astronomy = Task("mmlu_flan_n_shot_generative_astronomy", "mmlu_flan_n_shot_generative_astronomy", "MMLU (astronomy)")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/backend/run_eval_suite.py

@@ -4,7 +4,7 @@ import logging
 from datetime import datetime
 from lm_eval import tasks, evaluator, utils
 
-from src.envs import RESULTS_REPO, API
+from src.envs import RESULTS_REPO, API, RUN_MODE
 from src.backend.manage_requests import EvalRequest
 from src.backend.lamini_evaluator import LaminiEvaluator
 from src.backend.harness_evaluator import HarnessEvaluator
@@ -19,20 +19,20 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
             "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
         )
 
-
-
     lamini_evaluator = LaminiEvaluator(eval_request.model, eval_request.revision, eval_request.precision,
                                        batch_size, device, no_cache, limit, write_out=True,
                                        output_base_path='logs')
     lamini_results = lamini_evaluator.evaluate()
 
     ## task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
+    task_names = ["mmlu_flan_n_shot_generative_anatomy", "mmlu_flan_n_shot_generative_astronomy"]
     print(f"Selected Tasks: {task_names}")
     harness_evaluator = HarnessEvaluator(eval_request.model, eval_request.revision, eval_request.precision,
-                                       batch_size, device, no_cache, limit, write_out=True,
-                                       output_base_path='logs')
+                                         batch_size, device, no_cache, limit, write_out=True,
+                                         output_base_path='logs')
     results = harness_evaluator.evaluate(task_names)
     results_trimmed = {
+        "config": results["config"],
         "results": {
             "mmlu_flan_n_shot_generative_astronomy": results["results"]["mmlu_flan_n_shot_generative_astronomy"],
             "mmlu_flan_n_shot_generative_anatomy": results["results"]["mmlu_flan_n_shot_generative_anatomy"],
@@ -40,29 +40,36 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
             "product_id_precision_score": lamini_results["results"]["product_id_precision_score"],
         }
     }
-    results["config"]["model_dtype"] = eval_request.precision
-    results["config"]["model_name"] = eval_request.model
-    results["config"]["model_sha"] = eval_request.revision
+    results_trimmed["config"]["model_dtype"] = eval_request.precision
+    results_trimmed["config"]["model_name"] = eval_request.model
+    results_trimmed["config"]["model_sha"] = eval_request.revision
 
     output = json.dumps(results_trimmed, indent=4)
     print("output:", output)
 
-    # output_path = os.path.join(local_dir, *eval_request.model.split("/"))
-    # os.makedirs(output_path)
-    output_path = os.path.join(local_dir, *eval_request.model.split("/"),
-                               f"results_{datetime.now()}.json")
-    os.makedirs(os.path.dirname(output_path), exist_ok=True)
-    print(f"output_path: {output_path}")
-    with open(output_path, "w") as f:
-        f.write(output)
+    if RUN_MODE == "LOCAL":
+        output_path = os.path.join(local_dir, *eval_request.model.split("/"))
+        os.makedirs(output_path)
+        print(f"output_path: {output_path}, exists: {os.path.exists(output_path)}")
+        with open(f"{output_path}/results_{datetime.now()}.json", "w") as f:
+            f.write(output)
+    else:
+        output_path = os.path.join(local_dir, *eval_request.model.split("/"),
+                                   f"results_{datetime.now()}.json")
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        print(f"output_path: {output_path}, exists: {os.path.exists(output_path)}")
+
+        with open(output_path, "w") as f:
+            f.write(output)
 
     # TODO: uncomment
-    API.upload_file(
-        path_or_fileobj=output_path,
-        path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",
-        repo_id=results_repo,
-        repo_type="dataset",
-    )
-    print(f"upload to {results_repo} complete")
+    if RUN_MODE != "LOCAL":
+        API.upload_file(
+            path_or_fileobj=output_path,
+            path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",
+            repo_id=results_repo,
+            repo_type="dataset",
+        )
+        print(f"upload to {results_repo} complete")
 
     return results

src/leaderboard/read_evals.py

@@ -3,7 +3,7 @@ import json
 import math
 import os
 from dataclasses import dataclass
-from src.envs import EVAL_RESULTS_PATH_BACKEND
+from src.envs import EVAL_RESULTS_PATH_BACKEND, RUN_MODE, EVAL_REQUESTS_PATH_BACKEND
 import dateutil
 import numpy as np
 
@@ -156,7 +156,9 @@ def get_request_file_for_model(requests_path, model_name, precision):
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     #TODO: comment
-    # results_path = EVAL_RESULTS_PATH_BACKEND
+    if RUN_MODE == "LOCAL":
+        results_path = EVAL_RESULTS_PATH_BACKEND
+        requests_path = EVAL_REQUESTS_PATH_BACKEND
     model_result_filepaths = []
     print("get_raw_eval_results - results_path", results_path)
 

start.sh

@@ -1,2 +1,18 @@
+#!/bin/bash
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+
+   KEY_LENGTH=${#KEY}
+   VALUE="${ARGUMENT:$KEY_LENGTH+1}"
+
+   export "$KEY"="$VALUE"
+done
+
+
+echo "Run mode is: $RUN_MODE"
+echo "Model passed is: $LOCAL_MODEL_NAME"
+
 docker buildx build --platform=linux/amd64 -t ldr .
-docker run -it --rm -p 7860:7860 --platform=linux/amd64 -e TOKEN=$TOKEN ldr python app.py
+docker run -it --rm -p 7860:7860 --platform=linux/amd64 -e TOKEN=$TOKEN -e RUN_MODE=$RUN_MODE -e LOCAL_MODEL_NAME=$LOCAL_MODEL_NAME ldr python app.py