Spaces:
Running
Running
File size: 3,015 Bytes
9ceb843 0b8c16d 9ceb843 90eea3b 9ceb843 0b8c16d 9ceb843 ab74236 9ceb843 ab74236 9ceb843 ab74236 9ceb843 35e2ca1 9ceb843 56fcfaf ab74236 56fcfaf ab74236 9ceb843 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
import pandas as pd
from pathlib import Path
from datasets import load_dataset
import numpy as np
import os
import re
# From Open LLM Leaderboard
def model_hyperlink(link, model_name):
if model_name == "random":
return "random"
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def undo_hyperlink(html_string):
# Regex pattern to match content inside > and <
pattern = r'>[^<]+<'
match = re.search(pattern, html_string)
if match:
# Extract the matched text and remove leading '>' and trailing '<'
return match.group(0)[1:-1]
else:
return "No text found"
# Define a function to fetch and process data
def load_all_data(data_repo, subdir:str, subsubsets=False): # use HF api to pull the git repo
dir = Path(data_repo)
data_dir = dir / subdir
orgs = [d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d))]
# get all files within the sub folders orgs
models_results = []
for org in orgs:
org_dir = data_dir / org
files = [f for f in os.listdir(org_dir) if os.path.isfile(os.path.join(org_dir, f))]
for file in files:
if file.endswith(".json"):
models_results.append(org + "/" + file)
# create empty dataframe to add all data to
df = pd.DataFrame()
# load all json data in the list models_results one by one to avoid not having the same entries
for model in models_results:
model_data = load_dataset("json", data_files=data_repo + subdir+ "/" + model, split="train")
df2 = pd.DataFrame(model_data)
# add to df
df = pd.concat([df2, df])
# remove chat_template comlumn
df = df.drop(columns=["chat_template"])
# sort columns alphabetically
df = df.reindex(sorted(df.columns), axis=1)
# move column "model" to the front
cols = list(df.columns)
cols.insert(0, cols.pop(cols.index('model')))
df = df.loc[:, cols]
# select all columns except "model"
cols = df.columns.tolist()
cols.remove("model")
# round
df[cols] = df[cols].round(2)
avg = np.nanmean(df[cols].values,axis=1).round(2)
# add average column
df["average"] = avg
# apply model_hyperlink function to column "model"
df["model"] = df["model"].apply(lambda x: model_hyperlink(f"https://huggingface.co/{x}", x))
# move average column to the second
cols = list(df.columns)
cols.insert(1, cols.pop(cols.index('average')))
df = df.loc[:, cols]
# remove column xstest (outdated data)
# if xstest is a column
if "xstest" in df.columns:
df = df.drop(columns=["xstest"])
# remove column anthropic and summarize_prompted (outdated data)
if "anthropic" in df.columns:
df = df.drop(columns=["anthropic"])
if "summarize_prompted" in df.columns:
df = df.drop(columns=["summarize_prompted"])
return df
|