mlabonne commited on
Commit
3d13fcf
1 Parent(s): 083ca5d

Create yall.py

Browse files
Files changed (1) hide show
  1. yall.py +128 -0
yall.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gistyc
3
+ import requests
4
+ from dataclasses import dataclass
5
+ import re
6
+ import streamlit as st
7
+
8
+ @dataclass
9
+ class GistInfo:
10
+ gist_id: str
11
+ filename: str
12
+ url: str
13
+ model_name: str
14
+ model_id: str
15
+ model: str
16
+ agieval: float
17
+ gpt4all: float
18
+ truthfulqa: float
19
+ bigbench: float
20
+ average: float
21
+
22
+
23
+ def update_gist(content, gist_id, access_token):
24
+ """
25
+ Update the content of a GitHub Gist.
26
+
27
+ Args:
28
+ content (str): The new content of the gist.
29
+ gist_id (str): The ID of the gist to update.
30
+ access_token (str): GitHub personal access token with gist permissions.
31
+ """
32
+ api_url = f"https://api.github.com/gists/{gist_id}"
33
+ headers = {
34
+ "Authorization": f"token {access_token}",
35
+ "Accept": "application/vnd.github.v3+json"
36
+ }
37
+ data = {
38
+ "files": {
39
+ "YALL - Yet Another LLM Leaderboard.md": {
40
+ "content": content
41
+ }
42
+ }
43
+ }
44
+
45
+ response = requests.patch(api_url, json=data, headers=headers)
46
+
47
+ if response.status_code == 200:
48
+ print("Gist updated successfully.")
49
+ else:
50
+ print("Failed to update gist. Status code:", response.status_code)
51
+ print("Response:", response.json())
52
+
53
+
54
+ @st.cache_data
55
+ def create_yall():
56
+ # Get token
57
+ GITHUB_API_TOKEN = os.environ.get("github")
58
+
59
+ # Retrieve all gists
60
+ gist_api = gistyc.GISTyc(auth_token=GITHUB_API_TOKEN)
61
+ data = gist_api.get_gists()
62
+
63
+ # List to store the GistInfo objects
64
+ gist_infos = []
65
+
66
+ for data_dict in data:
67
+ if 'files' in data_dict and data_dict['files']:
68
+ file_info = next(iter(data_dict['files'].values()))
69
+ filename = file_info['filename']
70
+ if filename.endswith("-Nous.md"):
71
+ raw_url = file_info['raw_url']
72
+ response = requests.get(raw_url)
73
+ if response.status_code == 200:
74
+ if "Error: File does not exist" not in response.text:
75
+ # Parse the markdown table
76
+ lines = response.text.split('\n')
77
+ if len(lines) >= 3:
78
+ values = lines[2].split('|')[1:-1]
79
+
80
+ # Extract model name and model id using regular expression
81
+ model_match = re.search(r'\[([^\]]+)\]\(https://huggingface.co/([^/]+)/([^)]+)\)', values[0].strip())
82
+ if model_match:
83
+ model_name = model_match.group(1)
84
+ model_id = f"{model_match.group(2)}/{model_match.group(3)}"
85
+ print(values[0].strip())
86
+ print(model_name)
87
+ print(model_id)
88
+ print("=============")
89
+ else:
90
+ model_name = model_id = 'Unknown'
91
+
92
+
93
+ # Parse the markdown table
94
+ lines = response.text.split('\n')
95
+ if len(lines) >= 3:
96
+ values = lines[2].split('|')[1:-1]
97
+
98
+ # Create a GistInfo object and add it to the list
99
+ gist_info = GistInfo(
100
+ gist_id=data_dict['id'],
101
+ filename=filename,
102
+ url=data_dict['html_url'], # Assuming html_url is the URL of the gist
103
+ model_name=model_name,
104
+ model_id=model_id,
105
+ model=values[0].strip(),
106
+ agieval=float(values[1].strip()),
107
+ gpt4all=float(values[2].strip()),
108
+ truthfulqa=float(values[3].strip()),
109
+ bigbench=float(values[4].strip()),
110
+ average=float(values[5].strip()),
111
+ )
112
+ gist_infos.append(gist_info)
113
+
114
+ # Sort the list by average
115
+ gist_infos = sorted(gist_infos, key=lambda x: x.average, reverse=True)
116
+
117
+ # Create markdown table
118
+ markdown_table = "| Model | Average | AGIEval | GPT4All | TruthfulQA | Bigbench |\n"
119
+ markdown_table += "|---|---:|---:|---:|---:|---:|\n"
120
+
121
+ for gist in gist_infos:
122
+ model_link = f"[{gist.model_id}](https://huggingface.co/{gist.model_id})"
123
+ markdown_table += f"| {model_link} [📄]({gist.url}) | {gist.average} | {gist.agieval} | {gist.gpt4all} | {gist.truthfulqa} | {gist.bigbench} |\n"
124
+
125
+ # Update YALL's gist
126
+ update_gist(content=markdown_table, gist_id="90294929a2dbcb8877f9696f28105fdf", access_token=GITHUB_API_TOKEN)
127
+
128
+ return markdown_table