Weyaxi's picture
add title and descriptions
52101ad verified
raw
history blame
7.96 kB
import os
os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login
from huggingface_hub import ModelCardData, EvalResult, ModelCard
from huggingface_hub.repocard_data import eval_results_to_model_index
from huggingface_hub.repocard import RepoCard
from openllm import get_json_format_data, get_datas
from tqdm import tqdm
import time
import requests
import pandas as pd
from pytablewriter import MarkdownTableWriter
import gradio as gr
api = HfApi()
fs = HfFileSystem()
data = get_json_format_data()
finished_models = get_datas(data)
df = pd.DataFrame(finished_models)
def search(df, value):
result_df = df[df["Model"] == value]
return result_df.iloc[0].to_dict() if not result_df.empty else None
def get_details_url(repo):
author, model = repo.split("/")
return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
def get_query_url(repo):
return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
desc = """
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
"""
def get_task_summary(results):
return {
"ARC":
{"dataset_type":"ai2_arc",
"dataset_name":"AI2 Reasoning Challenge (25-Shot)",
"dataset_short_name": "ARC (25-shot)",
"metric_type":"acc_norm",
"metric_value":results["ARC"],
"dataset_config":"ARC-Challenge",
"dataset_split":"test",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 25},
"metric_name":"normalized accuracy"
},
"HellaSwag":
{"dataset_type":"hellaswag",
"dataset_name":"HellaSwag (10-Shot)",
"dataset_short_name": "HellaSwag (10-shot)",
"metric_type":"acc_norm",
"metric_value":results["HellaSwag"],
"dataset_config":None,
"dataset_split":"validation",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 10},
"metric_name":"normalized accuracy"
},
"MMLU":
{
"dataset_type":"cais/mmlu",
"dataset_name":"MMLU (5-Shot)",
"dataset_short_name": "MMLU (5-Shot)",
"metric_type":"acc",
"metric_value":results["MMLU"],
"dataset_config":"all",
"dataset_split":"test",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
},
"TruthfulQA":
{
"dataset_type":"truthful_qa",
"dataset_name":"TruthfulQA (0-shot)",
"dataset_short_name": "TruthfulQA (0-shot)",
"metric_type":"mc2",
"metric_value":results["TruthfulQA"],
"dataset_config":"multiple_choice",
"dataset_split":"validation",
"dataset_revision":None,
"dataset_args":{"num_few_shot": 0},
"metric_name":None
},
"Winogrande":
{
"dataset_type":"winogrande",
"dataset_name":"Winogrande (5-shot)",
"dataset_short_name": "Winogrande (5-shot)",
"metric_type":"acc",
"metric_value":results["Winogrande"],
"dataset_config":"winogrande_xl",
"dataset_split":"validation",
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
},
"GSM8K":
{
"dataset_type":"gsm8k",
"dataset_name":"GSM8k (5-shot)",
"dataset_short_name": "GSM8k (5-shot)",
"metric_type":"acc",
"metric_value":results["GSM8K"],
"dataset_config":"main",
"dataset_split":"test",
"dataset_args":{"num_few_shot": 5},
"metric_name":"accuracy"
}
}
def get_eval_results(repo):
results = search(df, repo)
task_summary = get_task_summary(results)
md_writer = MarkdownTableWriter()
md_writer.headers = ["Metric", "Value"]
md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_short_name"], v["metric_value"]] for v in task_summary.items()]
text = f"""
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here]({get_details_url(repo)})
{md_writer.dumps()}
"""
return text
def get_edited_yaml_readme(repo):
card = ModelCard.load(repo)
results = search(df, repo)
common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}
tasks_results = get_task_summary(results)
if not card.data['eval_results']: # No results reported yet, we initialize the metadata
card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
else: # We add the new evaluations
for task in tasks_results.values():
cur_result = EvalResult(**task, **common)
if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
continue
card.data['eval_results'].append(cur_result)
return str(card)
def commit(hf_token, repo, pr_number=None, message="Adding Evaluation Results"): # specify pr number if you want to edit it, don't if you don't want
login(hf_token)
edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
try:
try: # check if there is a readme already
readme_text = get_edited_yaml_readme(repo) + get_eval_results(repo)
except Exception as e:
if "Repo card metadata block was not found." in str(e): # There is no readme
readme_text = get_edited_yaml_readme(repo)
else:
print(f"Something went wrong: {e}")
liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
commit = (create_commit(repo_id=repo, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)
return commit
except Exception as e:
if "Discussions are disabled for this repo" in str(e):
return "Discussions disabled"
elif "Cannot access gated repo" in str(e):
return "Gated repo"
elif "Repository Not Found" in str(e):
return "Repository Not Found"
else:
return e
gradio_title="🧐 Open LLM Leaderboard Results PR Opener"
gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card.
## 💭 What Does This Tool Do:
- This tool adds the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) result of your model at the end of your model card.
- This tool also adds evaluation results as your model's metadata to showcase the evaluation results as a widget.
## 🛠️ Backend
The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
## 🤝 Acknowledgements
- Special thanks to Clémentine Fourrier (clefourrier) for her help and contributions to the code.
- Special thanks to [Lucain Pouget (Wauplin)](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api) for assisting with the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
"""
demo = gr.Interface(title=gradio_title, description=gradio_desc, fn=commit, inputs=["text", "text"], outputs="text")
demo.launch()