add debug info

backend-cli.py

@@ -172,6 +172,7 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
                 results['results'][task_name][f"{key},none"] = value
 
         results['results'][task_name]['batch_size,none'] = batch_size
+        results['results'][task_name]['precision,none'] = eval_request.precision
     print(f"gpu_stats_list: {gpu_stats_list}")
     print("GPU Usage:", gpu_info)
 

src/display/utils.py

@@ -15,11 +15,11 @@ MULTIPLE_CHOICEs = ["mmlu"]
 
 GPU_TEMP = 'Temp(C)'
 GPU_Power = 'Power(W)'
-GPU_Mem = 'Mem(M)'
+GPU_Mem = 'Mem(G)'
 GPU_Name = "GPU"
 GPU_Util = 'Util(%)'
 BATCH_SIZE = 'bs'
-
+PRECISION = "Precision"
 system_metrics_to_name_map = {
     "end_to_end_time": f"{E2Es}",
     "prefilling_time": f"{PREs}",
@@ -32,6 +32,7 @@ gpu_metrics_to_name_map = {
     GPU_Power: GPU_Power,
     GPU_Mem: GPU_Mem,
     "batch_size": BATCH_SIZE,
+    "precision": PRECISION,
     GPU_Name: GPU_Name,
 }
 
@@ -105,6 +106,7 @@ for task in Tasks:
     # System performance metrics
     auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
     auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True)])
+    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True)])
     auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
     auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True)])
     auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])

src/leaderboard/read_evals.py

@@ -106,6 +106,9 @@ class EvalResult:
                     if "GPU" in metric:
                         results[benchmark][metric] = value
                         continue
+                    if "precision" in metric:
+                        results[benchmark][metric] = value
+                        continue
 
                     if "rouge" in metric and "truthful" not in benchmark:
                         multiplier = 1.0

src/utils.py

@@ -74,13 +74,13 @@ def parse_nvidia_smi():
                 gpu_info.update({
                     GPU_TEMP: temp,
                     GPU_Power: power_usage,
-                    GPU_Mem: mem_usage,
+                    GPU_Mem: round(mem_usage / 1024, 2),
                     GPU_Util: gpu_util
                 })
 
             if len(gpu_info) >= 4:
                 gpu_stats.append(gpu_info)
-    print(f"len(gpu_stats): {len(gpu_stats)}")
+    print(f"gpu_stats: {gpu_stats}")
     gpu_name = f"{len(gpu_stats)}x{gpu_name}"
     gpu_stats_total = {
                         GPU_TEMP: 0,
@@ -94,7 +94,7 @@ def parse_nvidia_smi():
         gpu_stats_total[GPU_Power] += gpu_stat[GPU_Power]
         gpu_stats_total[GPU_Mem] += gpu_stat[GPU_Mem]
         gpu_stats_total[GPU_Util] += gpu_stat[GPU_Util]
-
+    gpu_stats_total[GPU_Mem] = gpu_stats_total[GPU_Mem] # G
     gpu_stats_total[GPU_TEMP] /= len(gpu_stats)
     gpu_stats_total[GPU_Power] /= len(gpu_stats)
     gpu_stats_total[GPU_Util] /= len(gpu_stats)