|
import requests |
|
import pandas as pd |
|
from tqdm.auto import tqdm |
|
from utils import * |
|
import gradio as gr |
|
from huggingface_hub import HfApi, hf_hub_download |
|
from huggingface_hub.repocard import metadata_load |
|
|
|
|
|
class DeepRL_Leaderboard: |
|
def __init__(self) -> None: |
|
self.leaderboard= {} |
|
|
|
def add_leaderboad(self,id=None, title=None): |
|
if id is not None and title is not None: |
|
id = id.strip() |
|
title = title.strip() |
|
self.leaderboard.update({id:{'title':title,'data':get_data_per_env(id)}}) |
|
def get_data(self): |
|
return self.leaderboard |
|
|
|
def get_ids(self): |
|
return list(self.leaderboard.keys()) |
|
|
|
|
|
|
|
|
|
with open('app.css','r') as f: |
|
BLOCK_CSS = f.read() |
|
|
|
|
|
|
|
LOADED_MODEL_IDS = {} |
|
|
|
def get_data(rl_env): |
|
global LOADED_MODEL_IDS |
|
data = [] |
|
model_ids = get_model_ids(rl_env) |
|
LOADED_MODEL_IDS[rl_env]=model_ids |
|
|
|
for model_id in tqdm(model_ids): |
|
meta = get_metadata(model_id) |
|
if meta is None: |
|
continue |
|
user_id = model_id.split('/')[0] |
|
row = {} |
|
row["User"] = user_id |
|
row["Model"] = model_id |
|
accuracy = parse_metrics_accuracy(meta) |
|
mean_reward, std_reward = parse_rewards(accuracy) |
|
mean_reward = mean_reward if not pd.isna(mean_reward) else 0 |
|
std_reward = std_reward if not pd.isna(std_reward) else 0 |
|
row["Results"] = mean_reward - std_reward |
|
row["Mean Reward"] = mean_reward |
|
row["Std Reward"] = std_reward |
|
data.append(row) |
|
return pd.DataFrame.from_records(data) |
|
|
|
def get_data_per_env(rl_env): |
|
dataframe = get_data(rl_env) |
|
dataframe = dataframe.fillna("") |
|
|
|
if not dataframe.empty: |
|
|
|
dataframe["User"] = dataframe["User"].apply(make_clickable_user) |
|
dataframe["Model"] = dataframe["Model"].apply(make_clickable_model) |
|
dataframe = dataframe.sort_values(by=['Results'], ascending=False) |
|
if not 'Ranking' in dataframe.columns: |
|
dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)]) |
|
else: |
|
dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)] |
|
table_html = dataframe.to_html(escape=False, index=False,justify = 'left') |
|
return table_html,dataframe,dataframe.empty |
|
else: |
|
html = """<div style="color: green"> |
|
<p> ⌛ Please wait. Results will be out soon... </p> |
|
</div> |
|
""" |
|
return html,dataframe,dataframe.empty |
|
|
|
|
|
|
|
rl_leaderboard = DeepRL_Leaderboard() |
|
rl_leaderboard.add_leaderboad('CarRacing-v0'," The Car Racing 🏎️ Leaderboard 🚀") |
|
rl_leaderboard.add_leaderboad('MountainCar-v0',"The Mountain Car ⛰️ 🚗 Leaderboard 🚀") |
|
rl_leaderboard.add_leaderboad('LunarLander-v2',"The Lunar Lander 🌕 Leaderboard 🚀") |
|
rl_leaderboard.add_leaderboad('BipedalWalker-v3',"The BipedalWalker Leaderboard 🚀") |
|
rl_leaderboard.add_leaderboad('Taxi-v3','The Taxi-v3🚖 Leaderboard 🚀') |
|
rl_leaderboard.add_leaderboad('FrozenLake-v1-4x4-no_slippery','The FrozenLake-v1-4x4-no_slippery Leaderboard 🚀') |
|
rl_leaderboard.add_leaderboad('FrozenLake-v1-8x8-no_slippery','The FrozenLake-v1-8x8-no_slippery Leaderboard 🚀') |
|
rl_leaderboard.add_leaderboad('FrozenLake-v1-4x4','The FrozenLake-v1-4x4 Leaderboard 🚀') |
|
rl_leaderboard.add_leaderboad('FrozenLake-v1-8x8','The FrozenLake-v1-8x8 Leaderboard 🚀') |
|
|
|
RL_ENVS = rl_leaderboard.get_ids() |
|
RL_DETAILS = rl_leaderboard.get_data() |
|
|
|
|
|
|
|
|
|
|
|
|
|
def update_data(rl_env): |
|
global LOADED_MODEL_IDS |
|
data = [] |
|
model_ids = [x for x in get_model_ids(rl_env) if x not in LOADED_MODEL_IDS[rl_env]] |
|
LOADED_MODEL_IDS[rl_env]+=model_ids |
|
|
|
for model_id in tqdm(model_ids): |
|
meta = get_metadata(model_id) |
|
if meta is None: |
|
continue |
|
user_id = model_id.split('/')[0] |
|
row = {} |
|
row["User"] = user_id |
|
row["Model"] = model_id |
|
accuracy = parse_metrics_accuracy(meta) |
|
mean_reward, std_reward = parse_rewards(accuracy) |
|
mean_reward = mean_reward if not pd.isna(mean_reward) else 0 |
|
std_reward = std_reward if not pd.isna(std_reward) else 0 |
|
|
|
row["Results"] = mean_reward - std_reward |
|
row["Mean Reward"] = mean_reward |
|
row["Std Reward"] = std_reward |
|
data.append(row) |
|
return pd.DataFrame.from_records(data) |
|
|
|
|
|
|
|
def update_data_per_env(rl_env): |
|
global RL_DETAILS |
|
|
|
_,old_dataframe,_ = RL_DETAILS[rl_env]['data'] |
|
new_dataframe = update_data(rl_env) |
|
|
|
new_dataframe = new_dataframe.fillna("") |
|
if not new_dataframe.empty: |
|
new_dataframe["User"] = new_dataframe["User"].apply(make_clickable_user) |
|
new_dataframe["Model"] = new_dataframe["Model"].apply(make_clickable_model) |
|
|
|
dataframe = pd.concat([old_dataframe,new_dataframe]) |
|
|
|
if not dataframe.empty: |
|
|
|
dataframe = dataframe.sort_values(by=['Results'], ascending=False) |
|
if not 'Ranking' in dataframe.columns: |
|
dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)]) |
|
else: |
|
dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)] |
|
table_html = dataframe.to_html(escape=False, index=False,justify = 'left') |
|
return table_html,dataframe,dataframe.empty |
|
else: |
|
html = """<div style="color: green"> |
|
<p> ⌛ Please wait. Results will be out soon... </p> |
|
</div> |
|
""" |
|
return html,dataframe,dataframe.empty |
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_info_display(len_dataframe,env_name,name_leaderboard,is_empty): |
|
if not is_empty: |
|
markdown = """ |
|
<div class='infoPoint'> |
|
<h1> {name_leaderboard} </h1> |
|
<br> |
|
<p> This is a leaderboard of <b>{len_dataframe}</b> agents playing {env_name} 👩🚀. </p> |
|
<br> |
|
<p> We use lower bound result to sort the models: mean_reward - std_reward. </p> |
|
<br> |
|
<p> You can click on the model's name to be redirected to its model card which includes documentation. </p> |
|
<br> |
|
<p> You want to try your model? Read this <a href="https://github.com/huggingface/deep-rl-class/blob/Unit1/unit1/README.md" target="_blank">Unit 1</a> of Deep Reinforcement Learning Class. |
|
</p> |
|
</div> |
|
""".format(len_dataframe = len_dataframe,env_name = env_name,name_leaderboard = name_leaderboard) |
|
|
|
else: |
|
markdown = """ |
|
<div class='infoPoint'> |
|
<h1> {name_leaderboard} </h1> |
|
<br> |
|
</div> |
|
""".format(name_leaderboard = name_leaderboard) |
|
return markdown |
|
|
|
def reload_all_data(): |
|
|
|
global RL_DETAILS,RL_ENVS |
|
|
|
for rl_env in RL_ENVS: |
|
RL_DETAILS[rl_env]['data'] = update_data_per_env(rl_env) |
|
|
|
html = """<div style="color: green"> |
|
<p> ✅ Leaderboard updated! Click `Reload Leaderboard` to see the current leaderboard.</p> |
|
</div> |
|
""" |
|
return html |
|
|
|
|
|
def reload_leaderboard(rl_env): |
|
global RL_DETAILS |
|
|
|
data_html,data_dataframe,is_empty = RL_DETAILS[rl_env]['data'] |
|
|
|
markdown = get_info_display(len(data_dataframe),rl_env,RL_DETAILS[rl_env]['title'],is_empty) |
|
|
|
return markdown,data_html |
|
|
|
|
|
|
|
|
|
|
|
|
|
block = gr.Blocks(css=BLOCK_CSS) |
|
with block: |
|
notification = gr.HTML("""<div style="color: green"> |
|
<p> ⌛ Updating leaderboard... </p> |
|
</div> |
|
""") |
|
block.load(reload_all_data,[],[notification]) |
|
|
|
with gr.Tabs(): |
|
for rl_env in RL_ENVS: |
|
with gr.TabItem(rl_env) as rl_tab: |
|
data_html,data_dataframe,is_empty = RL_DETAILS[rl_env]['data'] |
|
markdown = get_info_display(len(data_dataframe),rl_env,RL_DETAILS[rl_env]['title'],is_empty) |
|
env_state =gr.Variable(default_value=rl_env) |
|
output_markdown = gr.HTML(markdown) |
|
reload = gr.Button('Reload Leaderboard') |
|
|
|
output_html = gr.HTML(data_html) |
|
|
|
reload.click(reload_leaderboard,inputs=[env_state],outputs=[output_markdown,output_html]) |
|
rl_tab.select(reload_leaderboard,inputs=[env_state],outputs=[output_markdown,output_html]) |
|
|
|
block.launch() |
|
|