| import argparse | |
| import pandas as pd | |
| #Run ScandEval on model | |
| #scandeval --model-id KennethTM/bert-base-uncased-danish | |
| #Format results as markdown | |
| #python scandeval_summary.py --name scandeval_benchmark_results.jsonl | |
| def scandeval2md(file): | |
| df = pd.read_json(file, lines=True) | |
| df["total"] = [i["total"] for i in df["results"]] | |
| summary = [] | |
| for i in df["total"]: | |
| summary_list = [] | |
| for k, v in i.items(): | |
| summary_string = "" | |
| if "_se" in k: | |
| summary_string += f" (±{round(v, 2)})\n" | |
| else: | |
| summary_string += f'{k.replace("test_", "")} = {round(v, 2)}' | |
| summary_list.append(summary_string) | |
| summary.append("".join(summary_list).strip()) | |
| df["summary"] = summary | |
| md_table = df[["task", "dataset", "summary"]].rename(columns={"task": "Task", "dataset": "Dataset", "summary": "Score (±SE)"}).to_markdown(index=False) | |
| print(md_table) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description='Script for formatting ScandEval .jsonl result file as markdown to standard output') | |
| parser.add_argument('--name', help='ScandEval .jsonl result file') | |
| args = parser.parse_args() | |
| scandeval2md(args.name) | |