import argparse import logging import pprint from huggingface_hub import snapshot_download import src.backend.run_eval_suite as run_eval_suite import src.backend.manage_requests as manage_requests import src.backend.sort_queue as sort_queue import src.envs as envs logging.basicConfig(level=logging.ERROR) pp = pprint.PrettyPrinter(width=80) PENDING_STATUS = "PENDING" RUNNING_STATUS = "RUNNING" FINISHED_STATUS = "FINISHED" FAILED_STATUS = "FAILED" snapshot_download(repo_id=envs.RESULTS_REPO, revision="main", local_dir=envs.EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60) snapshot_download(repo_id=envs.QUEUE_REPO, revision="main", local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60) def run_auto_eval(args): if not args.reproduce: current_pending_status = [PENDING_STATUS] manage_requests.check_completed_evals( api=envs.API, checked_status=RUNNING_STATUS, completed_status=FINISHED_STATUS, failed_status=FAILED_STATUS, hf_repo=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, hf_repo_results=envs.RESULTS_REPO, local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND ) logging.info("Checked completed evals") eval_requests = manage_requests.get_eval_requests(job_status=current_pending_status, hf_repo=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH_BACKEND) logging.info("Got eval requests") eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests) logging.info("Sorted eval requests") print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests") if len(eval_requests) == 0: print("No eval requests found. Exiting.") return eval_request = eval_requests[0] pp.pprint(eval_request) manage_requests.set_eval_request( api=envs.API, eval_request=eval_request, new_status=RUNNING_STATUS, hf_repo=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH_BACKEND ) logging.info("Set eval request to running, now running eval") run_eval_suite.run_evaluation( eval_request=eval_request, local_dir=envs.EVAL_RESULTS_PATH_BACKEND, results_repo=envs.RESULTS_REPO, batch_size=1, device=envs.DEVICE, no_cache=True, ) logging.info("Eval finished, now setting status to finished") else: eval_request = manage_requests.EvalRequest( model=args.model, status=PENDING_STATUS, precision=args.precision ) pp.pprint(eval_request) logging.info("Running reproducibility eval") run_eval_suite.run_evaluation( eval_request=eval_request, local_dir=envs.EVAL_RESULTS_PATH_BACKEND, results_repo=envs.RESULTS_REPO, batch_size=1, device=envs.DEVICE, ) logging.info("Reproducibility eval finished") def main(): parser = argparse.ArgumentParser(description="Run auto evaluation with optional reproducibility feature") # Optional arguments parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results") parser.add_argument("--model", type=str, default=None, help="Your Model ID") parser.add_argument("--precision", type=str, default="float16", help="Precision of your model") args = parser.parse_args() run_auto_eval(args) if __name__ == "__main__": main()