Spaces:
Running
Running
sherzod-hakimov
commited on
Commit
•
fa0181f
1
Parent(s):
295441e
adapted to the new changes in the benchmark list
Browse files- app.py +1 -1
- src/leaderboard_utils.py +40 -33
- src/plot_utils.py +5 -10
- src/version_utils.py +3 -11
app.py
CHANGED
@@ -128,7 +128,7 @@ with hf_app:
|
|
128 |
|
129 |
# Show information about the clemscore and last updated date below the table
|
130 |
gr.HTML(CLEMSCORE_TEXT)
|
131 |
-
gr.HTML(f"Last updated - {github_data['
|
132 |
|
133 |
# Add a dummy leaderboard to handle search queries in leaderboard_table
|
134 |
# This will show a temporary leaderboard based on the searched value
|
|
|
128 |
|
129 |
# Show information about the clemscore and last updated date below the table
|
130 |
gr.HTML(CLEMSCORE_TEXT)
|
131 |
+
gr.HTML(f"Last updated - {github_data['mm_date']}")
|
132 |
|
133 |
# Add a dummy leaderboard to handle search queries in leaderboard_table
|
134 |
# This will show a temporary leaderboard based on the searched value
|
src/leaderboard_utils.py
CHANGED
@@ -29,20 +29,11 @@ def get_github_data():
|
|
29 |
json_data = response.json()
|
30 |
versions = json_data['versions']
|
31 |
|
32 |
-
# Sort version names - latest first
|
33 |
version_names = sorted(
|
34 |
[ver['version'] for ver in versions],
|
35 |
-
key=lambda v:
|
36 |
reverse=True
|
37 |
-
)
|
38 |
-
print(f"Found {len(version_names)} versions from get_github_data(): {version_names}.")
|
39 |
-
|
40 |
-
# Get Last updated date of the latest version
|
41 |
-
latest_version = version_names[0]
|
42 |
-
latest_date = next(
|
43 |
-
ver['date'] for ver in versions if ver['version'] == latest_version
|
44 |
-
)
|
45 |
-
formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
|
46 |
|
47 |
# Get Leaderboard data - for text-only + multimodal
|
48 |
github_data = {}
|
@@ -51,34 +42,49 @@ def get_github_data():
|
|
51 |
text_dfs = []
|
52 |
mm_dfs = []
|
53 |
|
|
|
|
|
|
|
|
|
|
|
54 |
for version in version_names:
|
55 |
# Collect CSV data in descending order of clembench-runs versions
|
56 |
# Collect Text-only data
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
|
63 |
-
text_dfs.append(df)
|
64 |
-
else:
|
65 |
-
print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
|
66 |
-
|
67 |
-
# Collect Multimodal data
|
68 |
-
if float(version[1:]) >= 1.6:
|
69 |
-
mm_url = f"{base_repo}{version}_multimodal/results.csv"
|
70 |
-
mm_response = requests.get(mm_url)
|
71 |
-
if mm_response.status_code == 200:
|
72 |
-
df = pd.read_csv(StringIO(mm_response.text))
|
73 |
df = process_df(df)
|
74 |
-
df = df.sort_values(by=df.columns[1], ascending=False)
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
79 |
github_data["text"] = text_dfs
|
80 |
github_data["multimodal"] = mm_dfs
|
81 |
-
github_data["date"] =
|
|
|
82 |
|
83 |
return github_data
|
84 |
|
@@ -137,3 +143,4 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
|
137 |
filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
|
138 |
|
139 |
return filtered_df
|
|
|
|
29 |
json_data = response.json()
|
30 |
versions = json_data['versions']
|
31 |
|
|
|
32 |
version_names = sorted(
|
33 |
[ver['version'] for ver in versions],
|
34 |
+
key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),
|
35 |
reverse=True
|
36 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
|
38 |
# Get Leaderboard data - for text-only + multimodal
|
39 |
github_data = {}
|
|
|
42 |
text_dfs = []
|
43 |
mm_dfs = []
|
44 |
|
45 |
+
text_flag = True
|
46 |
+
text_date = ""
|
47 |
+
mm_flag = True
|
48 |
+
mm_date = ""
|
49 |
+
|
50 |
for version in version_names:
|
51 |
# Collect CSV data in descending order of clembench-runs versions
|
52 |
# Collect Text-only data
|
53 |
+
if len(version.split('_')) == 1:
|
54 |
+
text_url = f"{base_repo}{version}/results.csv"
|
55 |
+
csv_response = requests.get(text_url)
|
56 |
+
if csv_response.status_code == 200:
|
57 |
+
df = pd.read_csv(StringIO(csv_response.text))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
df = process_df(df)
|
59 |
+
df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
|
60 |
+
text_dfs.append(df)
|
61 |
+
if text_flag:
|
62 |
+
text_flag = False
|
63 |
+
text_date = next(ver['date'] for ver in versions if ver['version'] == version)
|
64 |
+
text_date = datetime.strptime(text_date, "%Y-%m-%d").strftime("%d %b %Y")
|
65 |
+
|
66 |
+
else:
|
67 |
+
print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
|
68 |
+
|
69 |
+
# Check if version ends with 'multimodal' before constructing the URL
|
70 |
+
mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
|
71 |
+
mm_url = f"{base_repo}{version}{mm_suffix}/results.csv"
|
72 |
+
mm_response = requests.get(mm_url)
|
73 |
+
if mm_response.status_code == 200:
|
74 |
+
df = pd.read_csv(StringIO(mm_response.text))
|
75 |
+
df = process_df(df)
|
76 |
+
df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
|
77 |
+
mm_dfs.append(df)
|
78 |
+
if mm_flag:
|
79 |
+
mm_flag = False
|
80 |
+
mm_date = next(ver['date'] for ver in versions if ver['version'] == version)
|
81 |
+
mm_date = datetime.strptime(mm_date, "%Y-%m-%d").strftime("%d %b %Y")
|
82 |
+
|
83 |
+
|
84 |
github_data["text"] = text_dfs
|
85 |
github_data["multimodal"] = mm_dfs
|
86 |
+
github_data["date"] = text_date
|
87 |
+
github_data["mm_date"] = mm_date
|
88 |
|
89 |
return github_data
|
90 |
|
|
|
143 |
filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
|
144 |
|
145 |
return filtered_df
|
146 |
+
|
src/plot_utils.py
CHANGED
@@ -127,27 +127,22 @@ def split_models(model_list: list):
|
|
127 |
"""
|
128 |
Split the models into open source and commercial
|
129 |
"""
|
130 |
-
|
131 |
open_models = []
|
132 |
commercial_models = []
|
133 |
-
|
134 |
-
|
135 |
# Load model registry data from main repo
|
136 |
model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
|
137 |
response = requests.get(model_registry_url)
|
138 |
|
139 |
if response.status_code == 200:
|
140 |
json_data = json.loads(response.text)
|
141 |
-
# Classify as Open or Commercial based on the defined backend in the model registry
|
142 |
-
backend_mapping = {}
|
143 |
|
144 |
for model_name in model_list:
|
145 |
-
model_prefix = model_name.split('-')[0] # Get the prefix part of the model name
|
146 |
for entry in json_data:
|
147 |
-
if entry["model_name"]
|
148 |
-
|
149 |
-
|
150 |
-
if
|
151 |
open_models.append(model_name)
|
152 |
else:
|
153 |
commercial_models.append(model_name)
|
|
|
127 |
"""
|
128 |
Split the models into open source and commercial
|
129 |
"""
|
|
|
130 |
open_models = []
|
131 |
commercial_models = []
|
132 |
+
|
|
|
133 |
# Load model registry data from main repo
|
134 |
model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
|
135 |
response = requests.get(model_registry_url)
|
136 |
|
137 |
if response.status_code == 200:
|
138 |
json_data = json.loads(response.text)
|
|
|
|
|
139 |
|
140 |
for model_name in model_list:
|
|
|
141 |
for entry in json_data:
|
142 |
+
if entry["model_name"] == model_name:
|
143 |
+
open_model = entry["open_weight"]
|
144 |
+
|
145 |
+
if open_model:
|
146 |
open_models.append(model_name)
|
147 |
else:
|
148 |
commercial_models.append(model_name)
|
src/version_utils.py
CHANGED
@@ -31,20 +31,18 @@ def get_versions_data():
|
|
31 |
json_data = response.json()
|
32 |
versions = json_data['versions']
|
33 |
|
34 |
-
# Sort version names - latest first
|
35 |
version_names = sorted(
|
36 |
[ver['version'] for ver in versions],
|
37 |
-
key=lambda v:
|
38 |
reverse=True
|
39 |
-
)
|
40 |
-
print(f"Found {len(version_names)} versions from get_versions_data(): {version_names}.")
|
41 |
|
42 |
# Get Last updated date of the latest version
|
43 |
latest_version = version_names[0]
|
44 |
latest_date = next(
|
45 |
ver['date'] for ver in versions if ver['version'] == latest_version
|
46 |
)
|
47 |
-
formatted_date = datetime.strptime(latest_date, "%Y
|
48 |
|
49 |
# Get Versions data
|
50 |
versions_data = {"latest": latest_version, "date": formatted_date}
|
@@ -64,8 +62,6 @@ def get_versions_data():
|
|
64 |
df = process_df(df)
|
65 |
df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
|
66 |
versions_data[version] = df
|
67 |
-
else:
|
68 |
-
print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {response.status_code}")
|
69 |
|
70 |
# Multimodal Data
|
71 |
mm_response = requests.get(mm_url)
|
@@ -74,8 +70,6 @@ def get_versions_data():
|
|
74 |
mm_df = process_df(mm_df)
|
75 |
mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False) # Sort by clemscore column
|
76 |
versions_data[version+"_multimodal"] = mm_df
|
77 |
-
else:
|
78 |
-
print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if multimodal results are not available for this version")
|
79 |
|
80 |
# Multimodal Data
|
81 |
q_response = requests.get(quant_url)
|
@@ -84,8 +78,6 @@ def get_versions_data():
|
|
84 |
q_df = process_df(q_df)
|
85 |
q_df = q_df.sort_values(by=q_df.columns[1], ascending=False) # Sort by clemscore column
|
86 |
versions_data[version + "_quantized"] = q_df
|
87 |
-
else:
|
88 |
-
print(f"Failed to read quantized leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if quantized results are not available for this version")
|
89 |
|
90 |
return versions_data
|
91 |
|
|
|
31 |
json_data = response.json()
|
32 |
versions = json_data['versions']
|
33 |
|
|
|
34 |
version_names = sorted(
|
35 |
[ver['version'] for ver in versions],
|
36 |
+
key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),
|
37 |
reverse=True
|
38 |
+
)
|
|
|
39 |
|
40 |
# Get Last updated date of the latest version
|
41 |
latest_version = version_names[0]
|
42 |
latest_date = next(
|
43 |
ver['date'] for ver in versions if ver['version'] == latest_version
|
44 |
)
|
45 |
+
formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")
|
46 |
|
47 |
# Get Versions data
|
48 |
versions_data = {"latest": latest_version, "date": formatted_date}
|
|
|
62 |
df = process_df(df)
|
63 |
df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
|
64 |
versions_data[version] = df
|
|
|
|
|
65 |
|
66 |
# Multimodal Data
|
67 |
mm_response = requests.get(mm_url)
|
|
|
70 |
mm_df = process_df(mm_df)
|
71 |
mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False) # Sort by clemscore column
|
72 |
versions_data[version+"_multimodal"] = mm_df
|
|
|
|
|
73 |
|
74 |
# Multimodal Data
|
75 |
q_response = requests.get(quant_url)
|
|
|
78 |
q_df = process_df(q_df)
|
79 |
q_df = q_df.sort_values(by=q_df.columns[1], ascending=False) # Sort by clemscore column
|
80 |
versions_data[version + "_quantized"] = q_df
|
|
|
|
|
81 |
|
82 |
return versions_data
|
83 |
|