import os os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py") from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login from huggingface_hub import ModelCardData, EvalResult, ModelCard from huggingface_hub.repocard_data import eval_results_to_model_index from huggingface_hub.repocard import RepoCard from openllm import get_json_format_data, get_datas from tqdm import tqdm import time import requests import pandas as pd from pytablewriter import MarkdownTableWriter import gradio as gr api = HfApi() fs = HfFileSystem() data = get_json_format_data() finished_models = get_datas(data) df = pd.DataFrame(finished_models) def search(df, value): result_df = df[df["Model"] == value] return result_df.iloc[0].to_dict() if not result_df.empty else None def get_details_url(repo): author, model = repo.split("/") return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}" def get_query_url(repo): return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}" desc = """ This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card. If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions """ def get_task_summary(results): return { "ARC": {"dataset_type":"ai2_arc", "dataset_name":"AI2 Reasoning Challenge (25-Shot)", "dataset_short_name": "ARC (25-shot)", "metric_type":"acc_norm", "metric_value":results["ARC"], "dataset_config":"ARC-Challenge", "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 25}, "metric_name":"normalized accuracy" }, "HellaSwag": {"dataset_type":"hellaswag", "dataset_name":"HellaSwag (10-Shot)", "dataset_short_name": "HellaSwag (10-shot)", "metric_type":"acc_norm", "metric_value":results["HellaSwag"], "dataset_config":None, "dataset_split":"validation", "dataset_revision":None, "dataset_args":{"num_few_shot": 10}, "metric_name":"normalized accuracy" }, "MMLU": { "dataset_type":"cais/mmlu", "dataset_name":"MMLU (5-Shot)", "dataset_short_name": "MMLU (5-Shot)", "metric_type":"acc", "metric_value":results["MMLU"], "dataset_config":"all", "dataset_split":"test", "dataset_revision":None, "dataset_args":{"num_few_shot": 5}, "metric_name":"accuracy" }, "TruthfulQA": { "dataset_type":"truthful_qa", "dataset_name":"TruthfulQA (0-shot)", "dataset_short_name": "TruthfulQA (0-shot)", "metric_type":"mc2", "metric_value":results["TruthfulQA"], "dataset_config":"multiple_choice", "dataset_split":"validation", "dataset_revision":None, "dataset_args":{"num_few_shot": 0}, "metric_name":None }, "Winogrande": { "dataset_type":"winogrande", "dataset_name":"Winogrande (5-shot)", "dataset_short_name": "Winogrande (5-shot)", "metric_type":"acc", "metric_value":results["Winogrande"], "dataset_config":"winogrande_xl", "dataset_split":"validation", "dataset_args":{"num_few_shot": 5}, "metric_name":"accuracy" }, "GSM8K": { "dataset_type":"gsm8k", "dataset_name":"GSM8k (5-shot)", "dataset_short_name": "GSM8k (5-shot)", "metric_type":"acc", "metric_value":results["GSM8K"], "dataset_config":"main", "dataset_split":"test", "dataset_args":{"num_few_shot": 5}, "metric_name":"accuracy" } } def get_eval_results(repo): results = search(df, repo) task_summary = get_task_summary(results) md_writer = MarkdownTableWriter() md_writer.headers = ["Metric", "Value"] md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_short_name"], v["metric_value"]] for v in task_summary.items()] text = f""" # [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) Detailed results can be found [here]({get_details_url(repo)}) {md_writer.dumps()} """ return text def get_edited_yaml_readme(repo): card = ModelCard.load(repo) results = search(df, repo) common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"} tasks_results = get_task_summary(results) if not card.data['eval_results']: # No results reported yet, we initialize the metadata card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()]) else: # We add the new evaluations for task in tasks_results.values(): cur_result = EvalResult(**task, **common) if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']): continue card.data['eval_results'].append(cur_result) return str(card) def commit(hf_token, repo, pr_number=None, message="Adding Evaluation Results"): # specify pr number if you want to edit it, don't if you don't want login(hf_token) edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True} try: try: # check if there is a readme already readme_text = get_edited_yaml_readme(repo) + get_eval_results(repo) except Exception as e: if "Repo card metadata block was not found." in str(e): # There is no readme readme_text = get_edited_yaml_readme(repo) else: print(f"Something went wrong: {e}") liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] commit = (create_commit(repo_id=repo, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url) return commit except Exception as e: if "Discussions are disabled for this repo" in str(e): return "Discussions disabled" elif "Cannot access gated repo" in str(e): return "Gated repo" elif "Repository Not Found" in str(e): return "Repository Not Found" else: return e gradio_title="🧐 Open LLM Leaderboard Results PR Opener" gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card. ## 💭 What Does This Tool Do: - This tool adds the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) result of your model at the end of your model card. - This tool also adds evaluation results as your model's metadata to showcase the evaluation results as a widget. ## 🛠️ Backend The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api). ## 🤝 Acknowledgements - Special thanks to Clémentine Fourrier (clefourrier) for her help and contributions to the code. - Special thanks to [Lucain Pouget (Wauplin)](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api) for assisting with the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api). """ demo = gr.Interface(title=gradio_title, description=gradio_desc, fn=commit, inputs=["text", "text"], outputs="text") demo.launch()