iqm

app.py

@@ -1,3 +1,4 @@
+import glob
 import json
 import os
 
@@ -6,12 +7,12 @@ import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
+from rliable import library as rly
+from rliable import metrics
 
 from src.backend import backend_routine
-import glob
 from src.logging import configure_root_logger, setup_logger
 
-
 configure_root_logger()
 logger = setup_logger(__name__)
 
@@ -119,6 +120,14 @@ ALL_ENV_IDS = {
 }
 
 
+def iqm(x):
+    score_dict = {"a": np.expand_dims(np.array(x), 1)}
+    aggregate_func = lambda x: np.array([metrics.aggregate_iqm(x)])
+    aggregate_scores, aggregate_score_cis = rly.get_interval_estimates(score_dict, aggregate_func, reps=50000)
+    iqm, low, high = aggregate_scores["a"][0], aggregate_score_cis["a"][0][0], aggregate_score_cis["a"][1][0]
+    return iqm
+
+
 def get_leaderboard_df():
     dir_path = API.snapshot_download(repo_id=RESULTS_REPO, repo_type="dataset")
     pattern = os.path.join(dir_path, "**", "results_*.json")
@@ -134,7 +143,7 @@ def get_leaderboard_df():
             env_ids = list(report["results"].keys())
             assert len(env_ids) == 1, "Only one environment supported for the moment"
             row["env_id"] = env_ids[0]
-            row["mean_episodic_return"] = np.mean(report["results"][env_ids[0]]["episodic_returns"])
+            row["iqm_episodic_return"] = iqm(report["results"][env_ids[0]]["episodic_returns"])
         data.append(row)
 
     df = pd.DataFrame(data)  # create DataFrame
@@ -144,7 +153,7 @@ def get_leaderboard_df():
 
 def select_env(df: pd.DataFrame, env_id: str):
     df = df[df["env_id"] == env_id]
-    df = df.sort_values("mean_episodic_return", ascending=False)
+    df = df.sort_values("iqm_episodic_return", ascending=False)
     df["ranking"] = np.arange(1, len(df) + 1)
     return df
 
@@ -159,7 +168,7 @@ def format_df(df: pd.DataFrame):
         df.loc[index, "model_id"] = f"[{model_id}](https://huggingface.co/{user_id}/{model_id})"
 
     # Keep only the relevant columns
-    df = df[["ranking", "user_id", "model_id", "mean_episodic_return"]]
+    df = df[["ranking", "user_id", "model_id", "iqm_episodic_return"]]
     return df.values.tolist()
 
 
@@ -201,9 +210,11 @@ def refresh_winners():
             # # Or in HTML:
             # outputs.append(f'<h3>🏆 <a href="https://huggingface.co/{model}">{model}</a> 🏆</h3>')
         else:
-            outputs.append(f"""## {env_id}
+            outputs.append(
+                f"""## {env_id}
 
-### 🤷‍♂️ No winner yet""")
+### 🤷‍♂️ No winner yet"""
+            )
     return outputs
 
 
@@ -318,7 +329,7 @@ with gr.Blocks(css=css) as demo:
                                 with gr.Column(scale=3):
                                     # Display the leaderboard
                                     gr_df = gr.components.Dataframe(
-                                        headers=["🏆", "🧑 User", "🤖 Model id", "📊 Mean episodic return"],
+                                        headers=["🏆", "🧑 User", "🤖 Model id", "📊 IQM episodic return"],
                                         datatype=["number", "markdown", "markdown", "number"],
                                         row_count=(20, "fixed"),
                                     )

requirements.txt

@@ -12,6 +12,7 @@ numpy==1.24.2
 pandas==2.0.0
 python-dateutil==2.8.2
 requests==2.28.2
+rliable==1.0.8
 torch==2.2.2
 tqdm==4.65.0