chrisjay commited on
Commit
6843958
1 Parent(s): d27283f

organized adding envs + added more envs

Browse files
Files changed (3) hide show
  1. .gitignore +1 -0
  2. app.py +60 -88
  3. utils.py +63 -0
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ __pycache__/*
app.py CHANGED
@@ -1,81 +1,42 @@
1
  import requests
2
  import pandas as pd
3
  from tqdm.auto import tqdm
4
-
5
  import gradio as gr
6
  from huggingface_hub import HfApi, hf_hub_download
7
  from huggingface_hub.repocard import metadata_load
8
 
9
 
10
- RL_ENVS = ['LunarLander-v2','CarRacing-v0','MountainCar-v0',
11
- 'BipedalWalker-v3','FrozenLake-v1','FrozenLake-v1-no_slippery',
12
- 'Taxi-v3','Cliffwalker-v0']
13
 
14
- with open('app.css','r') as f:
15
- BLOCK_CSS = f.read()
 
 
 
 
 
16
 
17
- LOADED_MODEL_IDS = {rl_env:[] for rl_env in RL_ENVS}
 
18
 
 
19
 
20
- # Based on Omar Sanseviero work
21
- # Make model clickable link
22
- def make_clickable_model(model_name):
23
- # remove user from model name
24
- model_name_show = ' '.join(model_name.split('/')[1:])
25
 
26
- link = "https://huggingface.co/" + model_name
27
- return f'<a target="_blank" href="{link}">{model_name_show}</a>'
28
 
29
- # Make user clickable link
30
- def make_clickable_user(user_id):
31
- link = "https://huggingface.co/" + user_id
32
- return f'<a target="_blank" href="{link}">{user_id}</a>'
33
-
34
- def get_model_ids(rl_env):
35
- api = HfApi()
36
- models = api.list_models(filter=rl_env)
37
- model_ids = [x.modelId for x in models]
38
- return model_ids
39
-
40
- def get_metadata(model_id):
41
- try:
42
- readme_path = hf_hub_download(model_id, filename="README.md")
43
- return metadata_load(readme_path)
44
- except requests.exceptions.HTTPError:
45
- # 404 README.md not found
46
- return None
47
-
48
- def parse_metrics_accuracy(meta):
49
- if "model-index" not in meta:
50
- return None
51
- result = meta["model-index"][0]["results"]
52
- metrics = result[0]["metrics"]
53
- accuracy = metrics[0]["value"]
54
- return accuracy
55
-
56
- # We keep the worst case episode
57
- def parse_rewards(accuracy):
58
- default_std = -1000
59
- default_reward=-1000
60
- if accuracy != None:
61
- parsed = accuracy.split(' +/- ')
62
- if len(parsed)>1:
63
- mean_reward = float(parsed[0])
64
- std_reward = float(parsed[1])
65
- else:
66
- mean_reward = default_std
67
- std_reward = default_reward
68
 
69
- else:
70
- mean_reward = default_std
71
- std_reward = default_reward
72
- return mean_reward, std_reward
73
 
74
  def get_data(rl_env):
75
  global LOADED_MODEL_IDS
76
  data = []
77
  model_ids = get_model_ids(rl_env)
78
- LOADED_MODEL_IDS[rl_env]+=model_ids
79
 
80
  for model_id in tqdm(model_ids):
81
  meta = get_metadata(model_id)
@@ -93,6 +54,47 @@ def get_data(rl_env):
93
  data.append(row)
94
  return pd.DataFrame.from_records(data)
95
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
 
97
 
98
  def update_data(rl_env):
@@ -149,27 +151,6 @@ def update_data_per_env(rl_env):
149
  return html,dataframe,dataframe.empty
150
 
151
 
152
- def get_data_per_env(rl_env):
153
- dataframe = get_data(rl_env)
154
- dataframe = dataframe.fillna("")
155
-
156
- if not dataframe.empty:
157
- # turn the model ids into clickable links
158
- dataframe["User"] = dataframe["User"].apply(make_clickable_user)
159
- dataframe["Model"] = dataframe["Model"].apply(make_clickable_model)
160
- dataframe = dataframe.sort_values(by=['Results'], ascending=False)
161
- if not 'Ranking' in dataframe.columns:
162
- dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)])
163
- else:
164
- dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)]
165
- table_html = dataframe.to_html(escape=False, index=False,justify = 'left')
166
- return table_html,dataframe,dataframe.empty
167
- else:
168
- html = """<div style="color: green">
169
- <p> ⌛ Please wait. Results will be out soon... </p>
170
- </div>
171
- """
172
- return html,dataframe,dataframe.empty
173
 
174
 
175
 
@@ -225,15 +206,6 @@ def reload_leaderboard(rl_env):
225
 
226
 
227
 
228
- RL_DETAILS ={'CarRacing-v0':{'title':" The Car Racing 🏎️ Leaderboard 🚀",'data':get_data_per_env('CarRacing-v0')},
229
- 'MountainCar-v0':{'title':"The Mountain Car ⛰️ 🚗 Leaderboard 🚀",'data':get_data_per_env('MountainCar-v0')},
230
- 'LunarLander-v2':{'title':"The Lunar Lander 🌕 Leaderboard 🚀",'data':get_data_per_env('LunarLander-v2')},
231
- 'BipedalWalker-v3':{'title':"The BipedalWalker Leaderboard 🚀",'data':get_data_per_env('BipedalWalker-v3')},
232
- 'FrozenLake-v1':{'title':"The FrozenLake Leaderboard 🚀",'data':get_data_per_env('FrozenLake-v1')},
233
- 'FrozenLake-v1-no_slippery':{'title':'The FrozenLake-v1-no_slippery Leaderboard 🚀','data':get_data_per_env('FrozenLake-v1-no_slippery')},
234
- 'Taxi-v3':{'title':'The Taxi-v3🚖 Leaderboard 🚀','data':get_data_per_env('Taxi-v3')},
235
- 'Cliffwalker-v0':{'title':'The Cliffwalker-v0 Leaderboard 🚀','data':get_data_per_env('Cliffwalker-v0')},
236
- }
237
 
238
 
239
 
 
1
  import requests
2
  import pandas as pd
3
  from tqdm.auto import tqdm
4
+ from utils import *
5
  import gradio as gr
6
  from huggingface_hub import HfApi, hf_hub_download
7
  from huggingface_hub.repocard import metadata_load
8
 
9
 
10
+ class DeepRL_Leaderboard:
11
+ def __init__(self) -> None:
12
+ self.leaderboard= {}
13
 
14
+ def add_leaderboad(self,id=None, title=None):
15
+ if id is not None and title is not None:
16
+ id = id.strip()
17
+ title = title.strip()
18
+ self.leaderboard.update({id:{'title':title,'data':get_data_per_env(id)}})
19
+ def get_data(self):
20
+ return self.leaderboard
21
 
22
+ def get_ids(self):
23
+ return list(self.leaderboard.keys())
24
 
25
+
26
 
27
+ # CSS file for the
28
+ with open('app.css','r') as f:
29
+ BLOCK_CSS = f.read()
 
 
30
 
 
 
31
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
 
33
+ LOADED_MODEL_IDS = {}
 
 
 
34
 
35
  def get_data(rl_env):
36
  global LOADED_MODEL_IDS
37
  data = []
38
  model_ids = get_model_ids(rl_env)
39
+ LOADED_MODEL_IDS[rl_env]=model_ids
40
 
41
  for model_id in tqdm(model_ids):
42
  meta = get_metadata(model_id)
 
54
  data.append(row)
55
  return pd.DataFrame.from_records(data)
56
 
57
+ def get_data_per_env(rl_env):
58
+ dataframe = get_data(rl_env)
59
+ dataframe = dataframe.fillna("")
60
+
61
+ if not dataframe.empty:
62
+ # turn the model ids into clickable links
63
+ dataframe["User"] = dataframe["User"].apply(make_clickable_user)
64
+ dataframe["Model"] = dataframe["Model"].apply(make_clickable_model)
65
+ dataframe = dataframe.sort_values(by=['Results'], ascending=False)
66
+ if not 'Ranking' in dataframe.columns:
67
+ dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)])
68
+ else:
69
+ dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)]
70
+ table_html = dataframe.to_html(escape=False, index=False,justify = 'left')
71
+ return table_html,dataframe,dataframe.empty
72
+ else:
73
+ html = """<div style="color: green">
74
+ <p> ⌛ Please wait. Results will be out soon... </p>
75
+ </div>
76
+ """
77
+ return html,dataframe,dataframe.empty
78
+
79
+
80
+
81
+ rl_leaderboard = DeepRL_Leaderboard()
82
+ rl_leaderboard.add_leaderboad('CarRacing-v0'," The Car Racing 🏎️ Leaderboard 🚀")
83
+ rl_leaderboard.add_leaderboad('MountainCar-v0',"The Mountain Car ⛰️ 🚗 Leaderboard 🚀")
84
+ rl_leaderboard.add_leaderboad('LunarLander-v2',"The Lunar Lander 🌕 Leaderboard 🚀")
85
+ rl_leaderboard.add_leaderboad('BipedalWalker-v3',"The BipedalWalker Leaderboard 🚀")
86
+ rl_leaderboard.add_leaderboad('Taxi-v3','The Taxi-v3🚖 Leaderboard 🚀')
87
+ rl_leaderboard.add_leaderboad('FrozenLake-v1-4x4-no_slippery','The FrozenLake-v1-4x4-no_slippery Leaderboard 🚀')
88
+ rl_leaderboard.add_leaderboad('FrozenLake-v1-8x8-no_slippery','The FrozenLake-v1-8x8-no_slippery Leaderboard 🚀')
89
+ rl_leaderboard.add_leaderboad('FrozenLake-v1-4x4','The FrozenLake-v1-4x4 Leaderboard 🚀')
90
+ rl_leaderboard.add_leaderboad('FrozenLake-v1-8x8','The FrozenLake-v1-8x8 Leaderboard 🚀')
91
+
92
+ RL_ENVS = rl_leaderboard.get_ids()
93
+ RL_DETAILS = rl_leaderboard.get_data()
94
+
95
+
96
+
97
+
98
 
99
 
100
  def update_data(rl_env):
 
151
  return html,dataframe,dataframe.empty
152
 
153
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
 
155
 
156
 
 
206
 
207
 
208
 
 
 
 
 
 
 
 
 
 
209
 
210
 
211
 
utils.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import requests
3
+ from tqdm.auto import tqdm
4
+ from huggingface_hub import HfApi, hf_hub_download
5
+ from huggingface_hub.repocard import metadata_load
6
+
7
+
8
+ # Based on Omar Sanseviero work
9
+ # Make model clickable link
10
+ def make_clickable_model(model_name):
11
+ # remove user from model name
12
+ model_name_show = ' '.join(model_name.split('/')[1:])
13
+
14
+ link = "https://huggingface.co/" + model_name
15
+ return f'<a target="_blank" href="{link}">{model_name_show}</a>'
16
+
17
+ # Make user clickable link
18
+ def make_clickable_user(user_id):
19
+ link = "https://huggingface.co/" + user_id
20
+ return f'<a target="_blank" href="{link}">{user_id}</a>'
21
+
22
+
23
+
24
+ def get_model_ids(rl_env):
25
+ api = HfApi()
26
+ models = api.list_models(filter=rl_env)
27
+ model_ids = [x.modelId for x in models]
28
+ return model_ids
29
+
30
+ def get_metadata(model_id):
31
+ try:
32
+ readme_path = hf_hub_download(model_id, filename="README.md")
33
+ return metadata_load(readme_path)
34
+ except requests.exceptions.HTTPError:
35
+ # 404 README.md not found
36
+ return None
37
+
38
+ def parse_metrics_accuracy(meta):
39
+ if "model-index" not in meta:
40
+ return None
41
+ result = meta["model-index"][0]["results"]
42
+ metrics = result[0]["metrics"]
43
+ accuracy = metrics[0]["value"]
44
+ return accuracy
45
+
46
+ # We keep the worst case episode
47
+ def parse_rewards(accuracy):
48
+ default_std = -1000
49
+ default_reward=-1000
50
+ if accuracy != None:
51
+ parsed = accuracy.split(' +/- ')
52
+ if len(parsed)>1:
53
+ mean_reward = float(parsed[0])
54
+ std_reward = float(parsed[1])
55
+ else:
56
+ mean_reward = default_std
57
+ std_reward = default_reward
58
+
59
+ else:
60
+ mean_reward = default_std
61
+ std_reward = default_reward
62
+ return mean_reward, std_reward
63
+