include FPS and license columns + text with FPS and hardware info

app.py

@@ -6,7 +6,8 @@ from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
 from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
 from datetime import datetime, timezone
 
-LAST_UPDATED = "Aug 13th 2023"
+LAST_UPDATED = "September 7h 2023"
+GPU_MODEL = "NVIDIA Tesla M60"
 
 column_names = {"AP-IoU=0.50:0.95-area=all-maxDets=100": "AP",
                  "AP-IoU=0.50-area=all-maxDets=100": "AP@.50",
@@ -19,7 +20,10 @@ column_names = {"AP-IoU=0.50:0.95-area=all-maxDets=100": "AP",
                  "AR-IoU=0.50:0.95-area=all-maxDets=100": "AR100",
                  "AR-IoU=0.50:0.95-area=small-maxDets=100": "AR-S",
                  "AR-IoU=0.50:0.95-area=medium-maxDets=100": "AR-M",
-                 "AR-IoU=0.50:0.95-area=large-maxDets=100": "AR-L"}
+                 "AR-IoU=0.50:0.95-area=large-maxDets=100": "AR-L",
+                 "estimated_fps": "FPS(*)",
+                 "hub_license": "hub license",
+                 }
 
 eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
 
@@ -30,18 +34,29 @@ if not csv_results.exists():
 original_df = pd.read_csv(csv_results)
 
 # Formats the columns
-def formatter(x):
+def decimal_formatter(x):
+    x = "{:.2f}".format(x)
+    return x
+
+def perc_formatter(x):
     x = "{:.2%}".format(x)
     while len(x) < 6:
         x = f"0{x}"
     return x
 
+# Drop columns not specified in dictionary
+cols_to_drop = [col for col in original_df.columns if col not in column_names]
+original_df.drop(cols_to_drop, axis=1, inplace=True)
+
 for col in original_df.columns:
     if col == "model":
         original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+    elif col == "estimated_fps":
+        original_df[col] = original_df[col].apply(decimal_formatter) # For decimal values
+    elif col == "hub_license":
+        continue
     else:
-        original_df[col] = original_df[col].apply(formatter) # For % values
-        # original_df[col] = original_df[col].multiply(100).round(2) 
+        original_df[col] = original_df[col].apply(perc_formatter) # For % values
  
 original_df.rename(columns=column_names, inplace=True)
 
@@ -133,8 +148,9 @@ with gr.Blocks() as demo:
                                       [model_name_textbox, chb_coco2017], 
                                       mdw_submission_result)
 
+    gr.Markdown(f"(*) FPS was measured using *{GPU_MODEL}* processing 1 image per batch. Refer to the 📈 \"Metrics\" tab for further details.", elem_classes="markdown-text")
     gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
-    
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             gr.Textbox(

constants.py

@@ -58,6 +58,13 @@ There are plenty of variations of these metrics, depending on the IoU threshold,
 - **AR-M (ARmedium)**: AR for medium objects: 322 < area < 962.
 - **AR-L (ARlarge)**: AR for large objects: area > 962.
 
+## Frames Per Second (FPS)
+We measure the frames per second (FPS) for each model by looking at the average time it takes across the whole dataset. This includes the pre and post processing steps.   
+
+The hardware we use definitely plays a role in these numbers. You can see which hardware we used in the results table. 📈
+
+Because each model has its own specific needs when it comes to batch size and memory requirements, we decided to test them with just 1 image per batch. One thing to keep in mind: this test setup might not fully reflect real-world scenarios. Typically, more images are processed together to get things moving faster. 🚀"
+
 ## How to reproduce our results
 
 To compute these metrics, various tools employ different methods. For this leaderboard's evaluation, we utilize the COCO evaluation approach, which can be found in the [COCO evaluation toolkit](https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py).

init.py

@@ -2,7 +2,9 @@ import os
 from constants import EVAL_REQUESTS_PATH
 from pathlib import Path
 from huggingface_hub import HfApi, Repository
+from dotenv import load_dotenv
 
+load_dotenv()
 TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
 QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
 QUEUE_PATH = os.environ.get("QUEUE_PATH", None)