add gpu info

backend-cli.py

@@ -6,6 +6,7 @@ import argparse
 
 import socket
 import random
+import threading
 from datetime import datetime
 
 from src.backend.run_eval_suite import run_evaluation
@@ -16,7 +17,7 @@ from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
 from src.envs import QUEUE_REPO, RESULTS_REPO, API
-from src.utils import my_snapshot_download
+from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -123,7 +124,16 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 
 def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
-    batch_size = 4
+    batch_size = eval_request.batch_size
+
+    init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
+    # if init_gpu_info['Mem(M)'] > 500:
+    #     assert False, f"This machine is not empty: {init_gpu_info}"
+    gpu_stats_list = []
+    stop_event = threading.Event()
+    monitor_thread = threading.Thread(target=monitor_gpus, args=(stop_event, 5, gpu_stats_list))
+    monitor_thread.start()
+
     try:
         results = run_evaluation(
             eval_request=eval_request,
@@ -150,6 +160,14 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
             raise
 
     # print("RESULTS", results)
+    stop_event.set()
+    monitor_thread.join()
+    gpu_info = analyze_gpu_stats(gpu_stats_list)
+    for task_name in results['results'].keys():
+        for key, value in gpu_info.items():
+            results['results'][task_name][f"{key},none"] = int(value)
+
+    print("GPU Usage:", gpu_info)
 
     dumped = json.dumps(results, indent=2, default=lambda o: "<not serializable>")
     # print(dumped)
@@ -409,23 +427,27 @@ if __name__ == "__main__":
     local_debug = args.debug
     # debug specific task by ping
     if local_debug:
-        debug_model_names = [args.model]  # Use model from arguments
-        debug_task_name = args.task  # Use task from arguments
+        # debug_model_names = [args.model]  # Use model from arguments
+        # debug_task_name = [args.task]  # Use task from arguments
+        debug_model_names = ["mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x7B-v0.1"]  # Use model from arguments
+        debug_task_name = ['mmlu', 'selfcheckgpt']  # Use task from arguments
+        precisions = ['float16', 'float16', '8bit']
         task_lst = TASKS_HARNESS.copy()
-        for task in task_lst:
-            for debug_model_name in debug_model_names:
-                task_name = task.benchmark
-                if task_name != debug_task_name:
-                    continue
-                eval_request = EvalRequest(
-                    model=debug_model_name, 
-                    private=False, 
-                    status="", 
-                    json_filepath="", 
-                    precision=args.precision,  # Use precision from arguments
-                    inference_framework=args.inference_framework  # Use inference framework from arguments
-                )
-                results = process_evaluation(task, eval_request, limit=args.limit)
+        for precision in precisions:
+            for task in task_lst:
+                for debug_model_name in debug_model_names:
+                    task_name = task.benchmark
+                    if task_name not in debug_task_name:
+                        continue
+                    eval_request = EvalRequest(
+                        model=debug_model_name,
+                        private=False,
+                        status="",
+                        json_filepath="",
+                        precision=args.precision,  # Use precision from arguments
+                        inference_framework=args.inference_framework  # Use inference framework from arguments
+                    )
+                    results = process_evaluation(task, eval_request, limit=args.limit)
     else:
         while True:
             res = False

src/backend/hflm_with_measurement.py

@@ -57,12 +57,12 @@ class StopWatch(TextStreamer):
             self.start_decoding = time()
         self.decoding_iterations += 1
         return
-    
+
     def end(self):
         if self.decoding_time is None and self.start_decoding is not None:
             self.decoding_time = time() - self.start_decoding
         return
-    
+
 
 class HFLMWithMeasurement(HFLM):
     def __init__(self, **kwargs):
@@ -287,7 +287,7 @@ class HFLMWithMeasurement(HFLM):
         pbar.close()
 
         return re_ord.get_original(res)
-    
+
     def _model_generate(self, context, max_length, stop, **generation_kwargs):
         # temperature = 0.0 if not set
         # if do_sample is false and temp==0.0:
@@ -318,7 +318,7 @@ class HFLMWithMeasurement(HFLM):
             **generation_kwargs,
         )
         end = time()
-        
+
         batch_size = context.shape[0]
         output_length = stop_watch.decoding_iterations
 
@@ -403,7 +403,7 @@ class HFLMWithMeasurement(HFLM):
                     f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}"
                 )
             # add EOS token to stop sequences
-            eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False)
+            eos = self.tok_decode(self.eot_token_id)
             if not until:
                 until = [eos]
             else:

src/backend/manage_requests.py

@@ -27,10 +27,11 @@ class EvalRequest:
     likes: Optional[int] = 0
     params: Optional[int] = None
     license: Optional[str] = ""
+    batch_size: Optional[int] = 1
 
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"
-
+        model_args += ",trust_remote_code=True"
         if self.precision in ["float16", "float32", "bfloat16"]:
             model_args += f",dtype={self.precision}"
             # Quantized models need some added config, the install of bits and bytes, etc
@@ -44,7 +45,6 @@ class EvalRequest:
             pass
         elif self.precision == "8bit":
             model_args += ",load_in_8bit=True"
-            model_args += ",trust_remote_code=True"
         else:
             raise Exception(f"Unknown precision {self.precision}.")
         return model_args

src/backend/run_eval_suite.py

@@ -13,7 +13,7 @@ orig_higher_is_better = ConfigurableTask.higher_is_better
 def process_results_decorator(func):
     def wrapper(self, doc, results, *args, **kwargs):
         processed_results = [r[0] for r in results]
-        
+
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
         decoding_throughput = sum([r[3] for r in results]) / len(results)

src/display/utils.py

@@ -13,6 +13,11 @@ TS = "T/s" #Decoding throughput (tok/s)
 InFrame = "Method" #"Inference framework"
 MULTIPLE_CHOICEs = ["mmlu"]
 
+GPU_TEMP = 'Temp(C)'
+GPU_Power = 'Power(W)'
+GPU_Mem = 'Mem(M)'
+GPU_Util = 'Util(%)'
+
 @dataclass
 class Task:
     benchmark: str
@@ -81,11 +86,16 @@ auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnConten
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
-    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{E2Es}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
     if task.value.benchmark in MULTIPLE_CHOICEs:
         continue
-    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{PREs}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name}-{TS}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True)])
+
+    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_power", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Power}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_temp", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_TEMP}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])
 
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])

src/populate.py

@@ -12,7 +12,7 @@ from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_
 
 from src.backend.envs import Tasks as BackendTasks
 from src.display.utils import Tasks
-from src.display.utils import E2Es, PREs, TS
+from src.display.utils import E2Es, PREs, TS, GPU_Mem, GPU_Power, GPU_TEMP, GPU_Util
 
 def get_leaderboard_df(
     results_path: str,
@@ -52,6 +52,13 @@ def get_leaderboard_df(
         "decoding_throughput": f"{TS}",
     }
 
+    gpu_metrics_to_name_map = {
+        GPU_Util: GPU_Util,
+        GPU_TEMP: GPU_TEMP,
+        GPU_Power: GPU_Power,
+        GPU_Mem: GPU_Mem
+    }
+
     all_data_json = []
     for entry in all_data_json_:
         new_entry = copy.deepcopy(entry)
@@ -63,6 +70,10 @@ def get_leaderboard_df(
                     if sys_metric in entry[k]:
                         new_entry[f"{k} {metric_namne}"] = entry[k][sys_metric]
 
+                for gpu_metric, metric_namne in gpu_metrics_to_name_map.items():
+                    if gpu_metric in entry[k]:
+                        new_entry[f"{k} {metric_namne}"] = entry[k][gpu_metric]
+
         all_data_json += [new_entry]
 
     # all_data_json.append(baseline_row)

src/utils.py

@@ -1,6 +1,12 @@
 import pandas as pd
 from huggingface_hub import snapshot_download
-
+import subprocess
+import re
+try:
+    from src.display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util
+except:
+    print("local debug: from display.utils")
+    from display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util
 
 def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
     for i in range(10):
@@ -32,3 +38,61 @@ def get_dataset_summary_table(file_path):
     df = df[["Category", "Benchmark", "Data Split", "Data Size", "Language"]]
 
     return df
+
+def parse_nvidia_smi():
+    # Execute the nvidia-smi command
+    result = subprocess.run(['nvidia-smi'], capture_output=True, text=True)
+    output = result.stdout.strip()
+
+    # Initialize data storage
+    gpu_stats = []
+
+    # Regex to extract the relevant data for each GPU
+    gpu_info_pattern = re.compile(r'(\d+)C\s+P\d+\s+(\d+)W / \d+W\s+\|\s+(\d+)MiB / \d+MiB\s+\|\s+(\d+)%')
+    lines = output.split('\n')
+
+    for line in lines:
+        match = gpu_info_pattern.search(line)
+        if match:
+            temp, power_usage, mem_usage, gpu_util = map(int, match.groups())
+            gpu_stats.append({
+                GPU_TEMP: temp,
+                GPU_Power: power_usage,
+                GPU_Mem: mem_usage,
+                GPU_Util: gpu_util
+            })
+
+    gpu_stats_total = {
+                        GPU_TEMP: 0,
+                        GPU_Power: 0,
+                        GPU_Mem: 0,
+                        GPU_Util: 0
+                    }
+    for gpu_stat in gpu_stats:
+        gpu_stats_total[GPU_TEMP] += gpu_stat[GPU_TEMP]
+        gpu_stats_total[GPU_Power] += gpu_stat[GPU_Power]
+        gpu_stats_total[GPU_Mem] += gpu_stat[GPU_Mem]
+        gpu_stats_total[GPU_Util] += gpu_stat[GPU_Util]
+
+    gpu_stats_total[GPU_TEMP] /= len(gpu_stats)
+    gpu_stats_total[GPU_Power] /= len(gpu_stats)
+    gpu_stats_total[GPU_Util] /= len(gpu_stats)
+
+    return [gpu_stats_total]
+
+def monitor_gpus(stop_event, interval, stats_list):
+    while not stop_event.is_set():
+        gpu_stats = parse_nvidia_smi()
+        if gpu_stats:
+            stats_list.extend(gpu_stats)
+        stop_event.wait(interval)
+
+def analyze_gpu_stats(stats_list):
+    if not stats_list:
+        return None
+    avg_stats = {key: sum(d[key] for d in stats_list) / len(stats_list) for key in stats_list[0]}
+    return avg_stats
+
+
+if __name__ == "__main__":
+    print(analyze_gpu_stats(parse_nvidia_smi()))