fix display

backend-cli.py

@@ -12,7 +12,6 @@ from src.backend.run_eval_suite import run_evaluation
 from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
 from src.backend.sort_queue import sort_models_by_priority
 from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
-LIMIT=2
 from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
@@ -150,10 +149,10 @@ def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
         else:
             raise
 
-    print("RESULTS", results)
+    # print("RESULTS", results)
 
     dumped = json.dumps(results, indent=2, default=lambda o: "<not serializable>")
-    print(dumped)
+    # print(dumped)
 
     output_path = os.path.join(
         EVAL_RESULTS_PATH_BACKEND, *eval_request.model.split("/"), f"results_{datetime.now()}.json"

src/backend/tasks/measurement_task_utils.py

@@ -8,21 +8,17 @@ def process_results_decorator(func):
     def wrapper(self, doc, results, *args, **kwargs):
         # We process the results here
         processed_results = [r[0] for r in results]
-
-        # end_to_end_time = end_to_end_time / batch_size
-        # prefilling_time = prefilling_time / batch_size
-        # token_per_sec = output_length / (decoding_time / batch_size)
         
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
-        token_per_sec = sum([r[3] for r in results]) / len(results)
-        print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, token_per_sec: {token_per_sec}")
+        decoding_throughput = sum([r[3] for r in results]) / len(results)
+        print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
 
         # Now call the original process_results with the processed results
         result_dict = func(self, doc, processed_results, *args, **kwargs)
         result_dict["end_to_end_time"] = end_to_end_time
         result_dict["prefilling_time"] = prefilling_time
-        result_dict["token_per_sec"] = token_per_sec
+        result_dict["decoding_throughput"] = decoding_throughput
         return result_dict
     return wrapper
 
@@ -33,7 +29,7 @@ def aggregation_decorator(func):
         aggregation_list = func(self, *args, **kwargs)
         aggregation_list["end_to_end_time"] = mean
         aggregation_list["prefilling_time"] = mean
-        aggregation_list["token_per_sec"] = mean
+        aggregation_list["decoding_throughput"] = mean
         return aggregation_list
     return wrapper
 
@@ -44,7 +40,7 @@ def higher_is_better_decorator(func):
         higher_is_better_dict = func(self, *args, **kwargs)
         higher_is_better_dict["end_to_end_time"] = False
         higher_is_better_dict["prefilling_time"] = False
-        higher_is_better_dict["token_per_sec"] = True
+        higher_is_better_dict["decoding_throughput"] = True
         return higher_is_better_dict
     return wrapper
 

src/display/utils.py

@@ -73,12 +73,12 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 # Inference framework
 auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent("Inference framework", "str", True)])
 
-# System performance metrics
-auto_eval_column_dict.append(["prefilling_time", ColumnContent, ColumnContent("Prefilling time (s)", "number", True)])
-auto_eval_column_dict.append(["token_per_second", ColumnContent, ColumnContent("Tokens/s", "number", True)])
-
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    # System performance metrics
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} End-to-end time (s)", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} Prefilling time (s)", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} Decoding throughput (tok/s)", "number", True)])
 
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])

src/leaderboard/read_evals.py

@@ -107,7 +107,10 @@ class EvalResult:
                         multiplier = 1.0
                     if "squad" in benchmark:
                         multiplier = 1.0
-
+                    if "time" in metric:
+                        multiplier = 1.0
+                    if "throughput" in metric:
+                        multiplier = 1.0
                     # print('RESULTS', data['results'])
                     # print('XXX', benchmark, metric, value, multiplier)
                     results[benchmark][metric] = value * multiplier

src/populate.py

@@ -30,7 +30,8 @@ def get_leaderboard_df(
                 raw_data[result_idx], requests_path_open_llm
             )
 
-    all_data_json_ = [v.to_dict() for v in raw_data if v.is_complete()]
+    # all_data_json_ = [v.to_dict() for v in raw_data if v.is_complete()]
+    all_data_json_ = [v.to_dict() for v in raw_data] # include incomplete evals
 
     name_to_bm_map = {}
 
@@ -45,15 +46,22 @@ def get_leaderboard_df(
         name_to_bm_map[name] = bm
 
     # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
+    system_metrics_to_name_map = {
+        "end_to_end_time": "End-to-end time (s)",
+        "prefilling_time": "Prefilling time (s)",
+        "decoding_throughput": "Decoding throughput (tok/s)",
+    }
 
     all_data_json = []
     for entry in all_data_json_:
         new_entry = copy.deepcopy(entry)
-
         for k, v in entry.items():
             if k in name_to_bm_map:
                 benchmark, metric = name_to_bm_map[k]
                 new_entry[k] = entry[k][metric]
+                for sys_metric, metric_namne in system_metrics_to_name_map.items():
+                    if sys_metric in entry[k]:
+                        new_entry[f"{k} {metric_namne}"] = entry[k][sys_metric]
 
         all_data_json += [new_entry]
 
@@ -69,10 +77,10 @@ def get_leaderboard_df(
             df[col] = np.nan
 
     if not df.empty:
-        df = df[cols].round(decimals=2)
+        df = df.round(decimals=2)
 
         # filter out if any of the benchmarks have not been produced
-        df = df[has_no_nan_values(df, benchmark_cols)]
+        # df = df[has_no_nan_values(df, benchmark_cols)]
 
     return raw_data, df