Spaces:
Runtime error
Runtime error
| import requests | |
| import pandas as pd | |
| from tqdm.auto import tqdm | |
| import gradio as gr | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from huggingface_hub.repocard import metadata_load | |
| def make_clickable_model(model_name): | |
| # remove user from model name | |
| model_name_show = ' '.join(model_name.split('/')[1:]) | |
| link = "https://huggingface.co/" + model_name | |
| return f'<a target="_blank" href="{link}">{model_name_show}</a>' | |
| # Make user clickable link | |
| def make_clickable_user(user_id): | |
| link = "https://huggingface.co/" + user_id | |
| return f'<a target="_blank" href="{link}">{user_id}</a>' | |
| def get_model_ids(assignment): | |
| api = HfApi() | |
| models = api.list_models(author="Classroom-workshop", search=assignment) | |
| model_ids = [x.modelId for x in models] | |
| print(model_ids) | |
| return model_ids | |
| def get_metadata(model_id): | |
| try: | |
| readme_path = hf_hub_download(model_id, filename="README.md") | |
| return metadata_load(readme_path) | |
| except requests.exceptions.HTTPError: | |
| # 404 README.md not found | |
| return None | |
| def parse_metrics_accuracy(meta): | |
| if "model-index" not in meta: | |
| return None | |
| result = meta["model-index"][0]["results"] | |
| metrics = result[0]["metrics"] | |
| accuracy = metrics[0]["value"] | |
| return accuracy | |
| # We keep the worst case episode | |
| def parse_rewards(accuracy): | |
| default_std = -1000 | |
| default_reward=-1000 | |
| if accuracy != None: | |
| parsed = accuracy.split(' +/- ') | |
| if len(parsed)>1: | |
| mean_reward = float(parsed[0]) | |
| std_reward = float(parsed[1]) | |
| else: | |
| mean_reward = float(default_std) | |
| std_reward = float(default_reward) | |
| else: | |
| mean_reward = float(default_std) | |
| std_reward = float(default_reward) | |
| return mean_reward, std_reward | |
| class Leaderboard: | |
| def __init__(self) -> None: | |
| self.leaderboard= {} | |
| def add_leaderboard(self,id=None, title=None): | |
| if id is not None and title is not None: | |
| id = id.strip() | |
| title = title.strip() | |
| self.leaderboard.update({id:{'title':title,'data':get_data_per_env(id)}}) | |
| def get_data(self): | |
| return self.leaderboard | |
| def get_ids(self): | |
| return list(self.leaderboard.keys()) | |
| # CSS file for the | |
| with open('app.css','r') as f: | |
| BLOCK_CSS = f.read() | |
| LOADED_MODEL_IDS = {} | |
| def get_data(rl_env): | |
| global LOADED_MODEL_IDS | |
| data = [] | |
| model_ids = get_model_ids(rl_env) | |
| LOADED_MODEL_IDS[rl_env]=model_ids | |
| for model_id in tqdm(model_ids): | |
| meta = get_metadata(model_id) | |
| if meta is None: | |
| continue | |
| user_id = model_id.split('/')[0] | |
| row = {} | |
| row["User"] = user_id | |
| row["Model"] = model_id | |
| metric = parse_metrics_accuracy(meta) | |
| row["Result"] = metric | |
| data.append(row) | |
| return pd.DataFrame.from_records(data) | |
| def get_data_per_env(assignment): | |
| dataframe = get_data(assignment) | |
| dataframe = dataframe.fillna("") | |
| if not dataframe.empty: | |
| # turn the model ids into clickable links | |
| dataframe["User"] = dataframe["User"].apply(make_clickable_user) | |
| dataframe["Model"] = dataframe["Model"].apply(make_clickable_model) | |
| print(dataframe) | |
| dataframe = dataframe.sort_values(by=['Result'], ascending=False) | |
| if not 'Ranking' in dataframe.columns: | |
| dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)]) | |
| else: | |
| dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)] | |
| table_html = dataframe.to_html(escape=False, index=False,justify = 'left') | |
| return table_html,dataframe,dataframe.empty | |
| else: | |
| html = """<div style="color: green"> | |
| <p> ⌛ Please wait. Results will be out soon... </p> | |
| </div> | |
| """ | |
| return html,dataframe,dataframe.empty | |
| leaderboard = Leaderboard() | |
| leaderboard.add_leaderboard('assignment1'," Automatic Speech Recognition") | |
| leaderboard.add_leaderboard('assignment2',"RL Agent for Moon landing") | |
| ASSIGNMENTS = leaderboard.get_ids() | |
| DETAILS = leaderboard.get_data() | |
| def update_data(rl_env): | |
| global LOADED_MODEL_IDS | |
| data = [] | |
| model_ids = [x for x in get_model_ids(rl_env) if x not in LOADED_MODEL_IDS[rl_env]] | |
| LOADED_MODEL_IDS[rl_env]+=model_ids | |
| for model_id in tqdm(model_ids): | |
| meta = get_metadata(model_id) | |
| if meta is None: | |
| continue | |
| user_id = model_id.split('/')[0] | |
| row = {} | |
| row["User"] = user_id | |
| row["Model"] = model_id | |
| accuracy = parse_metrics_accuracy(meta) | |
| row["Accuracy"] = accuracy | |
| data.append(row) | |
| return pd.DataFrame.from_records(data) | |
| def update_data_per_env(rl_env): | |
| global DETAILS | |
| _,old_dataframe,_ = DETAILS[rl_env]['data'] | |
| new_dataframe = update_data(rl_env) | |
| new_dataframe = new_dataframe.fillna("") | |
| if not new_dataframe.empty: | |
| new_dataframe["User"] = new_dataframe["User"].apply(make_clickable_user) | |
| new_dataframe["Model"] = new_dataframe["Model"].apply(make_clickable_model) | |
| dataframe = pd.concat([old_dataframe,new_dataframe]) | |
| if not dataframe.empty: | |
| dataframe = dataframe.sort_values(by=['Results'], ascending=False) | |
| if not 'Ranking' in dataframe.columns: | |
| dataframe.insert(0, 'Ranking', [i for i in range(1,len(dataframe)+1)]) | |
| else: | |
| dataframe['Ranking'] = [i for i in range(1,len(dataframe)+1)] | |
| table_html = dataframe.to_html(escape=False, index=False,justify = 'left') | |
| return table_html,dataframe,dataframe.empty | |
| else: | |
| html = """<div style="color: green"> | |
| <p> ⌛ Please wait. Results will be out soon... </p> | |
| </div> | |
| """ | |
| return html,dataframe,dataframe.empty | |
| def get_info_display(len_dataframe,env_name,name_leaderboard,is_empty): | |
| if not is_empty: | |
| markdown = """ | |
| <div class='infoPoint'> | |
| <h1> {name_leaderboard} </h1> | |
| <br> | |
| <p> This is a leaderboard of <b>{len_dataframe}</b> assignments for assignment {env_name} 👩🚀. </p> | |
| <br> | |
| </div> | |
| """.format(len_dataframe = len_dataframe,env_name = env_name,name_leaderboard = name_leaderboard) | |
| else: | |
| markdown = """ | |
| <div class='infoPoint'> | |
| <h1> {name_leaderboard} </h1> | |
| <br> | |
| </div> | |
| """.format(name_leaderboard = name_leaderboard) | |
| return markdown | |
| def reload_all_data(): | |
| global DETAILS, ASSIGNMENTS | |
| for assignment in ASSIGNMENTS: | |
| DETAILS[assignment]['data'] = update_data_per_env(assignment) | |
| html = """<div style="color: green"> | |
| <p> ✅ Leaderboard updated! Click `Reload Leaderboard` to see the current leaderboard.</p> | |
| </div> | |
| """ | |
| return html | |
| def reload_leaderboard(rl_env): | |
| global DETAILS | |
| data_html,data_dataframe,is_empty = DETAILS[rl_env]['data'] | |
| markdown = get_info_display(len(data_dataframe),rl_env, DETAILS[rl_env]['title'],is_empty) | |
| return markdown,data_html | |
| block = gr.Blocks(css=BLOCK_CSS) | |
| with block: | |
| notification = gr.HTML("""<div style="color: green"> | |
| <p> ⌛ Updating leaderboard... </p> | |
| </div> | |
| """) | |
| block.load(reload_all_data,[],[notification]) | |
| with gr.Tabs(): | |
| for rl_env in ASSIGNMENTS: | |
| with gr.TabItem(rl_env) as rl_tab: | |
| data_html,data_dataframe,is_empty = DETAILS[rl_env]['data'] | |
| markdown = get_info_display(len(data_dataframe),rl_env,DETAILS[rl_env]['title'],is_empty) | |
| env_state =gr.Variable(default_value=rl_env) | |
| output_markdown = gr.HTML(markdown) | |
| reload = gr.Button('Reload Leaderboard') | |
| output_html = gr.HTML(data_html) | |
| reload.click(reload_leaderboard,inputs=[env_state],outputs=[output_markdown,output_html]) | |
| rl_tab.select(reload_leaderboard,inputs=[env_state],outputs=[output_markdown,output_html]) | |
| block.launch() |