pr worker for portuguese models

.gitignore

@@ -0,0 +1,9 @@
+venv/
+__pycache__/
+.env*
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+downloads/

app.py

@@ -7,6 +7,7 @@ import threading
 import gradio as gr
 from gradio_space_ci import enable_space_ci
 from functions import commit
+from pr_worker import run_pr_worker
 
 enable_space_ci()
 
@@ -65,4 +66,6 @@ with gr.Blocks() as demo:
     submit_btn.click(commit, model_id, output)
 
 threading.Thread(target=refresh).start()
+threading.Thread(target=run_pr_worker).start()
+
 demo.launch()

functions.py

@@ -8,6 +8,7 @@ import gradio as gr
 from openllm import get_json_format_data, get_datas
 import pandas as pd
 import traceback
+from huggingface_hub import HfApi
 
 BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN')
 
@@ -15,6 +16,9 @@ data = get_json_format_data()
 finished_models = get_datas(data)
 df = pd.DataFrame(finished_models)
 
+source_name = "Open Portuguese LLM Leaderboard"
+default_pull_request_title = "Adding the Open Portuguese LLM Leaderboard Evaluation Results"
+
 desc = """
 This is an automated PR created with https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard
 
@@ -162,7 +166,7 @@ def get_edited_yaml_readme(repo, token: str | None):
   card = ModelCard.load(repo, token=token)
   results = search(df, repo)
 
-  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open Portuguese LLM Leaderboard", "source_url": get_query_url(repo)}
+  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": source_name, "source_url": get_query_url(repo)}
 
   tasks_results = get_task_summary(results)
 
@@ -176,9 +180,30 @@ def get_edited_yaml_readme(repo, token: str | None):
       card.data['eval_results'].append(cur_result)
 
   return str(card)
-    
 
-def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want
+def pr_already_exists(repo, token: str | None = None):
+  card = ModelCard.load(repo, token=token)
+  if 'eval_results' in card.data and card.data['eval_results']:
+    for x in card.data['eval_results']:
+      if x.source_name == source_name:
+        return True
+  if 'Open Portuguese LLM Leaderboard' in card.content:
+    return True
+  if 'Open PT LLM Leaderboard' in card.content:
+    return True
+
+  api = HfApi(token=token)
+  for x in api.get_repo_discussions(repo):
+    if x.title == default_pull_request_title:
+      return True
+    if x.author == "leaderboard-pt-pr-bot":
+      return True
+    if x.author == "eduagarcia" and x.is_pull_request:
+      return True
+  
+  return False
+
+def commit(repo, pr_number=None, message=default_pull_request_title, oauth_token: gr.OAuthToken | None = None, check_if_pr_exists=False): # specify pr number if you want to edit it, don't if you don't want
   if oauth_token is None:
     gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.")
     token = BOT_HF_TOKEN
@@ -192,6 +217,10 @@ def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_toke
         repo = RepoUrl(repo).repo_id
       except Exception:
         raise gr.Error(f"Not a valid repo id: {str(repo)}")
+      
+  if check_if_pr_exists:
+    if pr_already_exists(repo, token):
+      return "PR already exists"
     
   edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
 

pr_worker.py

@@ -0,0 +1,41 @@
+from glob import glob
+import pandas as pd
+from huggingface_hub import snapshot_download
+import json
+from tqdm.auto import tqdm
+import os
+import traceback
+from functions import pr_already_exists, commit
+
+
+QUEUE_REPO = "eduagarcia-temp/llm_pt_leaderboard_requests"
+EVAL_REQUESTS_PATH = "./eval-queue/"
+
+def run_pr_worker():
+    snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
+    for filepath in glob(os.path.join(EVAL_REQUESTS_PATH, '**/*.json'), recursive=True):
+        with open(filepath, 'r') as f:
+            model_data = json.load(f)
+        if model_data['status'] != 'FINISHED':
+            continue
+        if 'main_language' not in model_data:
+            continue
+        if model_data['main_language'] != "Portuguese":
+            continue
+        if model_data['result_metrics_average'] < 0.25:
+            continue
+        if 'ptpt' in model_data['model']:
+            continue
+        try:
+            if not pr_already_exists(model_data['model']):
+                print(f"Opening PR for {model_data['model']}")
+                commit(model_data['model'], check_if_pr_exists=True)
+        except Exception as e:
+            traceback.print_exc()
+            print(f"Error on {model_data['model']}: {str(e)}")
+
+
+if __name__ == "__main__":
+    run_pr_worker()
+        
+