| from glob import glob | |
| import pandas as pd | |
| from huggingface_hub import snapshot_download | |
| import json | |
| from tqdm.auto import tqdm | |
| import os | |
| import traceback | |
| from functions import pr_already_exists, commit | |
| QUEUE_REPO = "eduagarcia-temp/llm_pt_leaderboard_requests" | |
| EVAL_REQUESTS_PATH = "./eval-queue/" | |
| blacklist = ['PORTULAN', 'Weni', '22h', 't5'] | |
| def run_pr_worker(): | |
| snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30) | |
| for filepath in glob(os.path.join(EVAL_REQUESTS_PATH, '**/*.json'), recursive=True): | |
| with open(filepath, 'r') as f: | |
| model_data = json.load(f) | |
| if model_data['status'] != 'FINISHED': | |
| continue | |
| if 'main_language' not in model_data: | |
| continue | |
| if model_data['main_language'] != "Portuguese": | |
| continue | |
| if model_data['result_metrics_average'] < 0.25: | |
| continue | |
| has_blacklist = False | |
| for b in blacklist: | |
| if b in model_data['model']: | |
| has_blacklist = True | |
| if has_blacklist: | |
| continue | |
| try: | |
| if not pr_already_exists(model_data['model']): | |
| print(f"Opening PR for {model_data['model']}") | |
| commit(model_data['model'], check_if_pr_exists=True) | |
| except Exception as e: | |
| traceback.print_exc() | |
| print(f"Error on {model_data['model']}: {str(e)}") | |
| if __name__ == "__main__": | |
| run_pr_worker() | |