sherzod-hakimov commited on
Commit
fa0181f
1 Parent(s): 295441e

adapted to the new changes in the benchmark list

Browse files
Files changed (4) hide show
  1. app.py +1 -1
  2. src/leaderboard_utils.py +40 -33
  3. src/plot_utils.py +5 -10
  4. src/version_utils.py +3 -11
app.py CHANGED
@@ -128,7 +128,7 @@ with hf_app:
128
 
129
  # Show information about the clemscore and last updated date below the table
130
  gr.HTML(CLEMSCORE_TEXT)
131
- gr.HTML(f"Last updated - {github_data['date']}")
132
 
133
  # Add a dummy leaderboard to handle search queries in leaderboard_table
134
  # This will show a temporary leaderboard based on the searched value
 
128
 
129
  # Show information about the clemscore and last updated date below the table
130
  gr.HTML(CLEMSCORE_TEXT)
131
+ gr.HTML(f"Last updated - {github_data['mm_date']}")
132
 
133
  # Add a dummy leaderboard to handle search queries in leaderboard_table
134
  # This will show a temporary leaderboard based on the searched value
src/leaderboard_utils.py CHANGED
@@ -29,20 +29,11 @@ def get_github_data():
29
  json_data = response.json()
30
  versions = json_data['versions']
31
 
32
- # Sort version names - latest first
33
  version_names = sorted(
34
  [ver['version'] for ver in versions],
35
- key=lambda v: float(v[1:]),
36
  reverse=True
37
- )
38
- print(f"Found {len(version_names)} versions from get_github_data(): {version_names}.")
39
-
40
- # Get Last updated date of the latest version
41
- latest_version = version_names[0]
42
- latest_date = next(
43
- ver['date'] for ver in versions if ver['version'] == latest_version
44
- )
45
- formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
46
 
47
  # Get Leaderboard data - for text-only + multimodal
48
  github_data = {}
@@ -51,34 +42,49 @@ def get_github_data():
51
  text_dfs = []
52
  mm_dfs = []
53
 
 
 
 
 
 
54
  for version in version_names:
55
  # Collect CSV data in descending order of clembench-runs versions
56
  # Collect Text-only data
57
- text_url = f"{base_repo}{version}/results.csv"
58
- csv_response = requests.get(text_url)
59
- if csv_response.status_code == 200:
60
- df = pd.read_csv(StringIO(csv_response.text))
61
- df = process_df(df)
62
- df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
63
- text_dfs.append(df)
64
- else:
65
- print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
66
-
67
- # Collect Multimodal data
68
- if float(version[1:]) >= 1.6:
69
- mm_url = f"{base_repo}{version}_multimodal/results.csv"
70
- mm_response = requests.get(mm_url)
71
- if mm_response.status_code == 200:
72
- df = pd.read_csv(StringIO(mm_response.text))
73
  df = process_df(df)
74
- df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
75
- mm_dfs.append(df)
76
- else:
77
- print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {csv_response.status_code}. Please ignore this message if multimodal results are not available for this version")
78
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  github_data["text"] = text_dfs
80
  github_data["multimodal"] = mm_dfs
81
- github_data["date"] = formatted_date
 
82
 
83
  return github_data
84
 
@@ -137,3 +143,4 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
137
  filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
138
 
139
  return filtered_df
 
 
29
  json_data = response.json()
30
  versions = json_data['versions']
31
 
 
32
  version_names = sorted(
33
  [ver['version'] for ver in versions],
34
+ key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),
35
  reverse=True
36
+ )
 
 
 
 
 
 
 
 
37
 
38
  # Get Leaderboard data - for text-only + multimodal
39
  github_data = {}
 
42
  text_dfs = []
43
  mm_dfs = []
44
 
45
+ text_flag = True
46
+ text_date = ""
47
+ mm_flag = True
48
+ mm_date = ""
49
+
50
  for version in version_names:
51
  # Collect CSV data in descending order of clembench-runs versions
52
  # Collect Text-only data
53
+ if len(version.split('_')) == 1:
54
+ text_url = f"{base_repo}{version}/results.csv"
55
+ csv_response = requests.get(text_url)
56
+ if csv_response.status_code == 200:
57
+ df = pd.read_csv(StringIO(csv_response.text))
 
 
 
 
 
 
 
 
 
 
 
58
  df = process_df(df)
59
+ df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
60
+ text_dfs.append(df)
61
+ if text_flag:
62
+ text_flag = False
63
+ text_date = next(ver['date'] for ver in versions if ver['version'] == version)
64
+ text_date = datetime.strptime(text_date, "%Y-%m-%d").strftime("%d %b %Y")
65
+
66
+ else:
67
+ print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
68
+
69
+ # Check if version ends with 'multimodal' before constructing the URL
70
+ mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
71
+ mm_url = f"{base_repo}{version}{mm_suffix}/results.csv"
72
+ mm_response = requests.get(mm_url)
73
+ if mm_response.status_code == 200:
74
+ df = pd.read_csv(StringIO(mm_response.text))
75
+ df = process_df(df)
76
+ df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
77
+ mm_dfs.append(df)
78
+ if mm_flag:
79
+ mm_flag = False
80
+ mm_date = next(ver['date'] for ver in versions if ver['version'] == version)
81
+ mm_date = datetime.strptime(mm_date, "%Y-%m-%d").strftime("%d %b %Y")
82
+
83
+
84
  github_data["text"] = text_dfs
85
  github_data["multimodal"] = mm_dfs
86
+ github_data["date"] = text_date
87
+ github_data["mm_date"] = mm_date
88
 
89
  return github_data
90
 
 
143
  filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
144
 
145
  return filtered_df
146
+
src/plot_utils.py CHANGED
@@ -127,27 +127,22 @@ def split_models(model_list: list):
127
  """
128
  Split the models into open source and commercial
129
  """
130
-
131
  open_models = []
132
  commercial_models = []
133
- open_backends = {"huggingface_local", "huggingface_multimodal", "openai_compatible"} # Define backends considered as open
134
-
135
  # Load model registry data from main repo
136
  model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
137
  response = requests.get(model_registry_url)
138
 
139
  if response.status_code == 200:
140
  json_data = json.loads(response.text)
141
- # Classify as Open or Commercial based on the defined backend in the model registry
142
- backend_mapping = {}
143
 
144
  for model_name in model_list:
145
- model_prefix = model_name.split('-')[0] # Get the prefix part of the model name
146
  for entry in json_data:
147
- if entry["model_name"].startswith(model_prefix):
148
- backend = entry["backend"]
149
- # Classify based on backend
150
- if backend in open_backends:
151
  open_models.append(model_name)
152
  else:
153
  commercial_models.append(model_name)
 
127
  """
128
  Split the models into open source and commercial
129
  """
 
130
  open_models = []
131
  commercial_models = []
132
+
 
133
  # Load model registry data from main repo
134
  model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
135
  response = requests.get(model_registry_url)
136
 
137
  if response.status_code == 200:
138
  json_data = json.loads(response.text)
 
 
139
 
140
  for model_name in model_list:
 
141
  for entry in json_data:
142
+ if entry["model_name"] == model_name:
143
+ open_model = entry["open_weight"]
144
+
145
+ if open_model:
146
  open_models.append(model_name)
147
  else:
148
  commercial_models.append(model_name)
src/version_utils.py CHANGED
@@ -31,20 +31,18 @@ def get_versions_data():
31
  json_data = response.json()
32
  versions = json_data['versions']
33
 
34
- # Sort version names - latest first
35
  version_names = sorted(
36
  [ver['version'] for ver in versions],
37
- key=lambda v: float(v[1:]),
38
  reverse=True
39
- )
40
- print(f"Found {len(version_names)} versions from get_versions_data(): {version_names}.")
41
 
42
  # Get Last updated date of the latest version
43
  latest_version = version_names[0]
44
  latest_date = next(
45
  ver['date'] for ver in versions if ver['version'] == latest_version
46
  )
47
- formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
48
 
49
  # Get Versions data
50
  versions_data = {"latest": latest_version, "date": formatted_date}
@@ -64,8 +62,6 @@ def get_versions_data():
64
  df = process_df(df)
65
  df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
66
  versions_data[version] = df
67
- else:
68
- print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {response.status_code}")
69
 
70
  # Multimodal Data
71
  mm_response = requests.get(mm_url)
@@ -74,8 +70,6 @@ def get_versions_data():
74
  mm_df = process_df(mm_df)
75
  mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False) # Sort by clemscore column
76
  versions_data[version+"_multimodal"] = mm_df
77
- else:
78
- print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if multimodal results are not available for this version")
79
 
80
  # Multimodal Data
81
  q_response = requests.get(quant_url)
@@ -84,8 +78,6 @@ def get_versions_data():
84
  q_df = process_df(q_df)
85
  q_df = q_df.sort_values(by=q_df.columns[1], ascending=False) # Sort by clemscore column
86
  versions_data[version + "_quantized"] = q_df
87
- else:
88
- print(f"Failed to read quantized leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if quantized results are not available for this version")
89
 
90
  return versions_data
91
 
 
31
  json_data = response.json()
32
  versions = json_data['versions']
33
 
 
34
  version_names = sorted(
35
  [ver['version'] for ver in versions],
36
+ key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),
37
  reverse=True
38
+ )
 
39
 
40
  # Get Last updated date of the latest version
41
  latest_version = version_names[0]
42
  latest_date = next(
43
  ver['date'] for ver in versions if ver['version'] == latest_version
44
  )
45
+ formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")
46
 
47
  # Get Versions data
48
  versions_data = {"latest": latest_version, "date": formatted_date}
 
62
  df = process_df(df)
63
  df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
64
  versions_data[version] = df
 
 
65
 
66
  # Multimodal Data
67
  mm_response = requests.get(mm_url)
 
70
  mm_df = process_df(mm_df)
71
  mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False) # Sort by clemscore column
72
  versions_data[version+"_multimodal"] = mm_df
 
 
73
 
74
  # Multimodal Data
75
  q_response = requests.get(quant_url)
 
78
  q_df = process_df(q_df)
79
  q_df = q_df.sort_values(by=q_df.columns[1], ascending=False) # Sort by clemscore column
80
  versions_data[version + "_quantized"] = q_df
 
 
81
 
82
  return versions_data
83