Spaces:
Runtime error
Runtime error
File size: 6,639 Bytes
3c478c7 2103a11 2820ed5 2103a11 8142abe 3c478c7 7a48d5e 3c478c7 74d31de 3c478c7 7a48d5e 3c478c7 74d31de 3c478c7 7a48d5e 3c478c7 74d31de 3c478c7 7a48d5e 3c478c7 74d31de 3c478c7 7a48d5e 3c478c7 74d31de 3c478c7 2103a11 3c478c7 2103a11 3c478c7 e5ce742 3c478c7 7393cd1 7f9264a 3c478c7 e5ce742 3c478c7 7393cd1 3c478c7 e5ce742 7393cd1 e5ce742 3c478c7 e5ce742 3c478c7 e5ce742 3c478c7 e5ce742 3c478c7 2103a11 103916a 3c478c7 f67c6da 8899d2c a3a931f 40567ae a3a931f 40567ae c72fcf9 8899d2c c72fcf9 40567ae c72fcf9 40567ae 3c478c7 1e32f27 3c478c7 401b6e4 2c2b21c 74d31de 7a48d5e 401b6e4 74d31de 401b6e4 7393cd1 20a0793 5b0c5d4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 |
import os
import json
import requests
import gradio as gr
import pandas as pd
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from huggingface_hub.repocard import metadata_load
from apscheduler.schedulers.background import BackgroundScheduler
from tqdm.contrib.concurrent import thread_map
from utils import make_clickable_model, make_clickable_user
DATASET_REPO_URL = (
"https://huggingface.co/datasets/hivex-research/hivex-leaderboard-data"
)
DATASET_REPO_ID = "hivex-research/hivex-leaderboard-data"
HF_TOKEN = os.environ.get("HF_TOKEN")
block = gr.Blocks()
api = HfApi(token=HF_TOKEN)
# .tab-buttons button {
# font-size: 20px;
# }
custom_css = """
/* Full width space */
.gradio-container {
max-width: 95%!important;
}
"""
hivex_envs = [
{
"title": "Wind Farm Control",
"hivex_env": "hivex-wind-farm-control",
"task_count": 2,
},
{
"title": "Wildfire Resource Management",
"hivex_env": "hivex-wildfire-resource-management",
"task_count": 3,
},
{
"title": "Drone-Based Reforestation",
"hivex_env": "hivex-drone-based-reforestation",
"task_count": 7,
},
{
"title": "Ocean Plastic Collection",
"hivex_env": "hivex-ocean-plastic-collection",
"task_count": 4,
},
{
"title": "Aerial Wildfire Suppression",
"hivex_env": "hivex-aerial-wildfire-suppression",
"task_count": 9,
},
]
def restart():
print("RESTART")
api.restart_space(repo_id="hivex-research/hivex-leaderboard")
def download_leaderboard_dataset():
path = snapshot_download(repo_id=DATASET_REPO_ID, repo_type="dataset")
return path
def get_total_models():
total_models = 0
for hivex_env in hivex_envs:
model_ids = get_model_ids(hivex_env["hivex_env"])
total_models += len(model_ids)
return total_models
def get_model_ids(hivex_env):
api = HfApi()
models = api.list_models(filter=hivex_env)
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md", etag_timeout=180)
return metadata_load(readme_path)
except requests.exceptions.HTTPError:
# 404 README.md not found
return None
def update_leaderboard_dataset_parallel(hivex_env, path):
# Get model ids associated with hivex_env
model_ids = get_model_ids(hivex_env)
def process_model(model_id):
meta = get_metadata(model_id)
# LOADED_MODEL_METADATA[model_id] = meta if meta is not None else ''
if meta is None:
return None
user_id = model_id.split("/")[0]
row = {}
row["User"] = user_id
row["Model"] = model_id
results = meta["model-index"][0]["results"][0]
row["Task"] = results["task"]["task-id"]
results_metrics = results["metrics"]
for result in results_metrics:
row[result["name"]] = float(result["value"].split("+/-")[0].strip())
return row
data = list(thread_map(process_model, model_ids, desc="Processing models"))
# Filter out None results (models with no metadata)
data = [row for row in data if row is not None]
# ranked_dataframe = rank_dataframe(pd.DataFrame.from_records(data))
ranked_dataframe = pd.DataFrame.from_records(data)
new_history = ranked_dataframe
file_path = path + "/" + hivex_env + ".csv"
new_history.to_csv(file_path, index=False)
return ranked_dataframe
def run_update_dataset():
path_ = download_leaderboard_dataset()
for i in range(0, len(hivex_envs)):
hivex_env = hivex_envs[i]
update_leaderboard_dataset_parallel(hivex_env["hivex_env"], path_)
api.upload_folder(
folder_path=path_,
repo_id="hivex-research/hivex-leaderboard-data",
repo_type="dataset",
commit_message="Update dataset",
)
def get_data(rl_env, task, path) -> pd.DataFrame:
"""
Get data from rl_env, filter by the given task, and drop the Task column.
:return: filtered data as a pandas DataFrame without the Task column
"""
csv_path = path + "/" + rl_env + ".csv"
data = pd.read_csv(csv_path)
# Filter the data to only include rows where the "Task" column matches the given task
filtered_data = data[data["Task"] == task]
# Drop the "Task" column
filtered_data = filtered_data.drop(columns=["Task"])
# Convert User and Model columns to clickable links
for index, row in filtered_data.iterrows():
user_id = row["User"]
filtered_data.loc[index, "User"] = make_clickable_user(user_id)
model_id = row["Model"]
filtered_data.loc[index, "Model"] = make_clickable_model(model_id)
return filtered_data
run_update_dataset()
block = gr.Blocks(css=custom_css)
with block:
with gr.Row(elem_id="header-row"):
# TITLE IMAGE
gr.HTML(
"""
<div align="centre">
<div style="width: 50%;">
<img
src="https://huggingface.co/spaces/hivex-research/hivex-leaderboard/resolve/main/hivex_logo.png"
alt="hivex logo"
/><h1>HIVEX-Leaderboard</h1>
</div>
</div>
"""
)
with gr.Row(elem_id="header-row"):
gr.HTML(f"<h1>HIVEX-Leaderboard</h1>")
with gr.Row(elem_id="header-row"):
gr.HTML(f"<p>Total models: {get_total_models()}</p>")
with gr.Row(elem_id="header-row"):
gr.HTML(f"<p>To get started, please check out <a href='https://github.com/hivex-research/hivex'>our GitHub repository</a>.</p>")
path_ = download_leaderboard_dataset()
# gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
# ENVIRONMENT TABS
with gr.Tabs() as tabs: # elem_classes="tab-buttons"
for i in range(0, len(hivex_envs)):
hivex_env = hivex_envs[i]
with gr.Tab(hivex_env["title"]) as env_tabs:
# TASK TABS
for j in range(0, hivex_env["task_count"]):
task = "Task " + str(j + 1)
with gr.TabItem(f"Task {j}"):
with gr.Row():
gr_dataframe = gr.components.Dataframe(value=get_data(hivex_env["hivex_env"], j, path_), headers=["User", "Model"], datatype=["markdown", "markdown"], row_count=(100, 'fixed'))
scheduler = BackgroundScheduler()
scheduler.add_job(restart, "interval", seconds=86400)
scheduler.start()
block.launch()
|