File size: 6,553 Bytes
059d8f0 fb67b80 059d8f0 fb67b80 059d8f0 fb67b80 059d8f0 7fa09dc 059d8f0 7fa09dc 059d8f0 7fa09dc 059d8f0 fb67b80 7fa09dc fb67b80 7fa09dc 059d8f0 fb67b80 7fa09dc fb67b80 059d8f0 7fa09dc 059d8f0 fb67b80 059d8f0 fb67b80 059d8f0 fb67b80 059d8f0 fb67b80 059d8f0 7fa09dc 059d8f0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
import requests
import pandas as pd
from tqdm.auto import tqdm
import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
# Based on Omar Sanseviero work
# Make model clickable link
def make_clickable_model(model_name):
link = "https://huggingface.co/" + model_name
return f'<a style="text-decoration: underline; color: #1f3b54 " target="_blank" href="{link}">{model_name}</a>'
# Make user clickable link
def make_clickable_user(user_id):
link = "https://huggingface.co/" + user_id
return f'<a style="text-decoration: underline; color: #1f3b54 " target="_blank" href="{link}">{user_id}</a>'
def get_model_ids(rl_env):
api = HfApi()
models = api.list_models(filter=rl_env)
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md")
return metadata_load(readme_path)
except requests.exceptions.HTTPError:
# 404 README.md not found
return None
def parse_metrics_accuracy(meta):
if "model-index" not in meta:
return None
result = meta["model-index"][0]["results"]
metrics = result[0]["metrics"]
accuracy = metrics[0]["value"]
#print("ACCURACY", accuracy)
return accuracy
# We keep the worst case episode
def parse_rewards(accuracy):
if accuracy != None:
parsed = accuracy.split(' +/- ')
mean_reward = float(parsed[0])
std_reward = float(parsed[1])
else:
mean_reward = -1000
std_reward = -1000
return mean_reward, std_reward
def get_data(rl_env):
data = []
model_ids = get_model_ids(rl_env)
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
user_id = model_id.split('/')[0]
row = {}
row["User"] = user_id
row["Model"] = model_id
accuracy = parse_metrics_accuracy(meta)
#print("RETURNED ACCURACY", accuracy)
mean_reward, std_reward = parse_rewards(accuracy)
#print("MEAN REWARD", mean_reward)
row["Results"] = mean_reward - std_reward
row["Mean Reward"] = mean_reward
row["Std Reward"] = std_reward
data.append(row)
return pd.DataFrame.from_records(data)
def get_data_per_env(rl_env):
dataframe = get_data(rl_env)
dataframe = dataframe.fillna("")
if not dataframe.empty:
# turn the model ids into clickable links
dataframe["User"] = dataframe["User"].apply(make_clickable_user)
dataframe["Model"] = dataframe["Model"].apply(make_clickable_model)
dataframe = dataframe.sort_values(by=['Results'], ascending=False)
table_html = dataframe.to_html(escape=False, index=False)
table_html = table_html.replace("<table>", '<table style="width: 100%; margin: auto; border:0.5px solid; border-spacing: 7px 0px">') # center-align the headers
table_html = table_html.replace("<thead>", '<thead align="left">') # left-align the headers
table_html = "<div style='text-align: left ; width:100%'>"+table_html+"</div>"
return table_html,dataframe,dataframe.empty
else:
html = """<div style="color: green">
<p> ⌛ Please wait. Results will be out soon... </p>
</div>
"""
return html,dataframe,dataframe.empty
RL_ENVS = ['LunarLander-v2','CarRacing-v0','MountainCar-v0']
RL_DETAILS ={'CarRacing-v0':{'title':" The Car Racing 🏎️ Leaderboard 🚀",'data':get_data_per_env('CarRacing-v0')},
'MountainCar-v0':{'title':"The Mountain Car ⛰️ 🚗 Leaderboard 🚀",'data':get_data_per_env('MountainCar-v0')},
'LunarLander-v2':{'title':" The Lunar Lander 🌕 Leaderboard 🚀",'data':get_data_per_env('LunarLander-v2')}
}
def reload_leaderboard(rl_env):
#import pdb;pdb.set_trace()
global RL_DETAILS
RL_DETAILS[rl_env]['data'] = get_data_per_env(rl_env)
data_html,data_dataframe,is_empty = RL_DETAILS[rl_env]['data']
if not is_empty:
markdown = """
# {name_leaderboard}
This is a leaderboard of **{len_dataframe}** agents playing {env_name} 👩🚀.
We use lower bound result to sort the models: mean_reward - std_reward.
You can click on the model's name to be redirected to its model card which includes documentation.
You want to try your model? Read this [Unit 1](https://github.com/huggingface/deep-rl-class/blob/Unit1/unit1/README.md) of Deep Reinforcement Learning Class.
""".format(len_dataframe = len(data_dataframe),env_name = rl_env,name_leaderboard = RL_DETAILS[rl_env]['title'])
else:
markdown = """
# {name_leaderboard}
""".format(name_leaderboard = RL_DETAILS[rl_env]['title'])
return markdown,data_html
block = gr.Blocks()
with block:
with gr.Tabs():
for rl_env in RL_ENVS:
with gr.TabItem(rl_env):
data_html,data_dataframe,is_empty = RL_DETAILS[rl_env]['data']
if not is_empty:
markdown = """
# {name_leaderboard}
This is a leaderboard of **{len_dataframe}** agents playing {env_name} 👩🚀.
We use lower bound result to sort the models: mean_reward - std_reward.
You can click on the model's name to be redirected to its model card which includes documentation.
You want to try your model? Read this [Unit 1](https://github.com/huggingface/deep-rl-class/blob/Unit1/unit1/README.md) of Deep Reinforcement Learning Class.
""".format(len_dataframe = len(data_dataframe),env_name = rl_env,name_leaderboard = RL_DETAILS[rl_env]['title'])
else:
markdown = """
# {name_leaderboard}
""".format(name_leaderboard = RL_DETAILS[rl_env]['title'])
reload = gr.Button('Reload Leaderboard')
#env_state = gr.State(rl_env)
output_markdown = gr.Markdown(markdown)
output_html = gr.HTML(data_html)
reload.click(reload_leaderboard,inputs=[rl_env],outputs=[output_markdown,output_html])
block.launch()
|