Repository: NVIDIA/TensorRT-LLM
Pull request #3189: tests: change qa perf test to trtllm-bench
Author: login: ruodil
Created: 2025-04-01T06:33:54Z | Merged: 2025-04-17T01:53:33Z



Diff:
diff --git a/tests/integration/defs/perf/model_yaml_config.py b/tests/integration/defs/perf/model_yaml_config.py
new file mode 100644
index 000000000000..47a3604bfabc
--- /dev/null
+++ b/tests/integration/defs/perf/model_yaml_config.py
@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -*- coding: utf-8 -*-
+"""
+Model yaml config for trtllm-bench perf tests
+"""
+
+
+def get_model_yaml_config(model_label: str) -> dict:
+    """
+        Return the yaml config corresponding to the model label.
+        Args:
+            model_label: model label from self._config.to_string()
+        Returns:
+            dict: yaml config
+        """
+    base_config = {
+        'enable_attention_dp': True,
+        'pytorch_backend_config': {
+            'enable_overlap_scheduler': True,
+            'print_iter_log': True,
+            'use_cuda_graph': True,
+            'cuda_graph_batch_sizes': [1, 512]
+        }
+    }
+    model_configs = {
+        'deepseek_r1-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-quant:fp8-reqs:10-ep:4-gpus:8':
+        {
+            'pytorch_backend_config': {
+                'enable_overlap_scheduler': True,
+                'use_cuda_graph': True,
+            },
+            'speculative_config': {
+                'decoding_type': 'MTP',
+                'num_nextn_predict_layers': 3
+            }
+        },
+        'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-quant:nvfp4-reqs:10-ep:4-tp:8-gpus:8':
+        {
+            'pytorch_backend_config': {
+                'enable_overlap_scheduler': True,
+                'use_cuda_graph': True,
+            },
+            'speculative_config': {
+                'decoding_type': 'MTP',
+                'num_nextn_predict_layers': 3
+            }
+        },
+        'deepseek_r1-bench-pytorch-float16-maxbs:128-maxnt:1127-input_output_len:1000,2000-quant:fp8-reqs:5120-con:1024-ep:8-gpus:8':
+        {
+            'pytorch_backend_config': {
+                'cuda_graph_batch_sizes': [128]
+            },
+        },
+        'deepseek_r1-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-quant:nvfp4-reqs:49152-con:3072-ep:8-gpus:8':
+        {
+            'pytorch_backend_config': {
+                'cuda_graph_padding_enabled': True,
+                'cuda_graph_batch_sizes':
+                [1, 2, 4, 8, 16, 32, 64, 128, 256, 384]
+            },
+        },
+        'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-quant:nvfp4-reqs:49152-con:3072-ep:8-gpus:8':
+        {
+            'pytorch_backend_config': {
+                'cuda_graph_padding_enabled': True,
+                'cuda_graph_batch_sizes':
+                [1, 2, 4, 8, 16, 32, 64, 128, 256, 384]
+            },
+        }
+    }
+    # get model name from model_label
+    model_name = next(
+        (key for key in model_configs if key in model_label.lower()), None)
+    if model_name:
+        base_config.update(model_configs[model_name])
+
+    return base_config
diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py
index a2244b8aa0f2..1f833f7a3937 100644
--- a/tests/integration/defs/perf/test_perf.py
+++ b/tests/integration/defs/perf/test_perf.py
@@ -27,6 +27,7 @@
                                        print_warning)
 
 from ..conftest import get_llm_root, llm_models_root, trt_environment
+from .model_yaml_config import get_model_yaml_config
 from .utils import (AbstractPerfScriptTestClass, PerfBenchScriptTestCmds,
                     PerfMetricType, PerfScriptTestCmds, generate_test_nodes)
 
@@ -38,21 +39,43 @@
 
 # Model PATH of local dir synced from internal LLM models repo
 MODEL_PATH_DICT = {
-    "llama_v2_7b": "llama-models-v2/llama-v2-7b-hf",
-    "llama_v2_13b": "llama-models-v2/llama-v2-13b-hf",
-    "llama_v2_70b": "llama-models-v2/llama-v2-70b-hf",
-    "llama_v3_8b": "llama-models-v3/8B",
+    "llama_v2_7b": "llama-models-v2/llama-v2-7b-hf",  # not safetensors repo
+    "llama_v2_13b": "llama-models-v2/llama-v2-13b-hf",  # not safetensors repo
+    "llama_v2_70b": "llama-models-v2/llama-v2-70b-hf",  # not safetensors repo
     "llama_v3.1_8b": "llama-3.1-model/Meta-Llama-3.1-8B",
-    "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8",
+    "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct",
     "llama_v3.1_70b": "llama-3.1-model/Meta-Llama-3.1-70B",
     "llama_v3.1_70b_instruct": "llama-3.1-model/Meta-Llama-3.1-70B-Instruct",
+    "llama_v3.2_11b": "llama-3.2-models/Llama-3.2-11B-Vision",
+    # "llama_30b": "llama-models/llama-30b-hf",
     "mixtral_8x7b_v0.1": "Mixtral-8x7B-v0.1",
     "mixtral_8x7b_v0.1_instruct": "Mixtral-8x7B-Instruct-v0.1",
+    "mixtral_8x22b_v0.1": "Mixtral-8x22B-v0.1",
     "mistral_7b_v0.1": "mistral-7b-v0.1",
     "deepseek_r1": "DeepSeek-R1/DeepSeek-R1",
     "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4",
     "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8",
     "deepseek_v3_lite_nvfp4": "DeepSeek-V3-Lite/nvfp4_moe_only",
+    "qwen2_7b_instruct": "Qwen2-7B-Instruct",
+    "qwen_14b_chat": "Qwen-14B-Chat",
+    "starcoder2_3b": "starcoder2-3b",
+    "starcoder_15b": "starcoder2-15b",
+    "t5": "t5-small",  # not supported for trtllm-bench build config
+    "flan_t5_base":
+    "flan-t5-small",  # not supported for trtllm-bench build config
+    "flan_t5_large":
+    "flan-t5-xl",  # not supported for trtllm-bench build config
+    "whisper_large_v3":
+    "whisper-models/large-v3",  # not supported for trtllm-bench tokenizer
+    "bart_large_cnn": "bart-large-cnn",  # not safetensors repo
+    "mbart_large_50_many_to_one_mmt": "mbart-large-50-many-to-one-mmt",
+    "mamba_130m": "mamba/mamba-130m-hf",
+    "mamba_370m": "mamba/mamba-370m-hf",
+    "mamba_2.8b": "mamba/mamba-2.8b-hf",
+    "gpt_20b": "gpt-neox-20b",
+    "gpt_350m_moe": "gpt2-medium",
+    "phi_3_mini_4k_instruct": "Phi-3/Phi-3-mini-4k-instruct",
+    "phi_3_mini_128k_instruct": "Phi-3/Phi-3-mini-128k-instruct",
 }
 # Model PATH of HuggingFace
 HF_MODEL_PATH = {
@@ -70,6 +93,7 @@
     "mixtral_8x7b_v0.1_hf": "mistralai/Mixtral-8x7B-v0.1",
     "mixtral_8x7b_v0.1_instruct_hf": "mistralai/Mixtral-8x7B-Instruct-v0.1",
     "mistral_7b_v0.1_hf": "mistralai/Mistral-7B-v0.1",
+    "flan_t5_base_hf": "google/flan-t5-small",
 }
 LORA_MODEL_PATH = {
     "llama_v2_13b": "llama-models-v2/chinese-llama-2-lora-13b",
@@ -156,9 +180,9 @@ def import_allowed_perf_config():
     PerfMetricType.SEQ_THROUGHPUT:
     re.compile(r"Request Throughput \(req\/sec\):\s+([\d\.]+)"),
     PerfMetricType.FIRST_TOKEN_TIME:
-    re.compile(r"Average time-to-first-token \[TTFT\]\(ms\):\s+([\d\.]+)"),
+    re.compile(r"Average time-to-first-token \[TTFT\] \(ms\):\s+([\d\.]+)"),
     PerfMetricType.OUTPUT_TOKEN_TIME:
-    re.compile(r"Average time-per-output-token \[TPOT\]\(ms\):\s+([\d\.]+)"),
+    re.compile(r"Average time-per-output-token \[TPOT\] \(ms\):\s+([\d\.]+)"),
 }
 # (Relative threshold, Absolute threshold) for all metric types
 PERF_METRIC_THRESHOLD = {
@@ -259,7 +283,8 @@ def __init__(
         backend: str = "",
         mode: str = "plugin",
         data_type: str = "float16",
-        max_batch_size: int = 0,
+        max_batch_size: int = 2048,
+        max_num_tokens: int = 8192,
         gpu_weights_percent: float = -1,
         batch_sizes: List[int] = [0],
         input_lens: List[int] = [8],
@@ -294,6 +319,8 @@ def __init__(
         self.gpu_weights_percent = gpu_weights_percent
         # Max Batch Size to build TRT engine with.
         self.max_batch_size = max_batch_size
+        # Max number of tokens to build TRT engine with.
+        self.max_num_tokens = max_num_tokens
         # List of batch sizes to run benchmark with.
         self.batch_sizes = batch_sizes
         # List of input lens to run benchmark with.
@@ -360,8 +387,10 @@ def to_string(self,
             entries.append(f"mp")
 
         # Add Max batch size.
-        if self.max_batch_size > 0:
-            entries.append(f"maxbs:{self.max_batch_size}")
+        entries.append(f"maxbs:{self.max_batch_size}")
+
+        # Add Max number of tokens.
+        entries.append(f"maxnt:{self.max_num_tokens}")
 
         if self.build_only:
             entries.append(f"build_only")
@@ -467,6 +496,9 @@ def load_from_str(self, test_param_labels) -> None:
         if labels[0].startswith("maxbs"):
             self.max_batch_size = int(labels.pop(0).replace("maxbs:", ""))
 
+        if labels[0].startswith("maxnt"):
+            self.max_num_tokens = int(labels.pop(0).replace("maxnt:", ""))
+
         if labels[0] == "build_only":
             self.build_only = True
             labels.pop(0)
@@ -601,21 +633,20 @@ def validate(self):
         if self.gpu_weights_percent != -1:
             assert 0 <= self.gpu_weights_percent <= 1, f"Invalid gpu_weights_percent: {self.gpu_weights_percent}!"
         if not self.build_only:
-            if self.runtime != "cppmanager":
+            if self.runtime != "cppmanager" and self.runtime != "bench":
+                print(f"runtime: {self.runtime}")
                 # Validate max batch size.
                 if self.max_batch_size > 0:
                     assert max(
                         self.batch_sizes
                     ) <= self.max_batch_size, f"Batch Size larger than Max Batch Size!"
-                if self.runtime != "bench":
                     # Validate bs, seq lens, and num_beams.
                     assert len(
                         self.batch_sizes
                     ) > 0 and self.batch_sizes[0] > 0, f"Empty batch sizes!"
                 assert self.static_batching == "", f"Static Batching only valid for gptManagerBenchmark!"
                 assert self.api == "", f"API Type only valid for gptManagerBenchmark!"
-                if self.runtime != "bench":
-                    assert self.streaming == "", f"Streaming only valid for gptManagerBenchmark and trtllm-bench!"
+                assert self.streaming == "", f"Streaming only valid for gptManagerBenchmark and trtllm-bench!"
 
             assert len(self.input_lens) > 0, f"Empty input_lens!"
             if self.is_bert_like():
@@ -629,6 +660,10 @@ def validate(self):
 
             # BERT with small BS is very unstable. Try to avoid it.
             if self.is_bert_like():
+                if self.runtime == "trtllm-bench":
+                    self.batch_sizes[
+                        0] = self.max_batch_size if self.max_batch_size > 0 else 1
+                    print(f"batch_sizes: {self.batch_sizes}")
                 assert all(
                     [b >= 32 for b in self.batch_sizes]
                 ), f"BERT with small BS is very unstable! Please increase to at least 32."
@@ -801,7 +836,8 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
             self._build_script, f"--output_dir={engine_dir}",
             f"--checkpoint_dir={checkpoint_dir}",
             f"--workers={self._config.tp_size}",
-            f"--use_paged_context_fmha=enable", f"--monitor_memory"
+            f"--use_paged_context_fmha=enable", f"--monitor_memory",
+            f"--max_batch_size={self._config.max_batch_size}"
         ]
         # For Multiple Profiles
         if self._config.multiple_profiles:
@@ -814,8 +850,6 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
         gpu_percent = self._config.gpu_weights_percent
         if gpu_percent != -1:
             build_cmd += [f"--weight_streaming"]
-        if self._config.max_batch_size > 0:
-            build_cmd.append(f"--max_batch_size={self._config.max_batch_size}")
         # For engine inspector
         build_cmd.append("--profiling_verbosity=layer_names_only")
         if self._config.num_loras > 0:
@@ -868,12 +902,12 @@ def get_trtllm_bench_build_command(self, engine_dir) -> list:
             model_name = model_name + "_hf"
         hf_model_name = HF_MODEL_PATH.get(model_name, "")
         build_cmd = [
-            self._build_script, f"--workspace={engine_dir}",
-            f"--model={hf_model_name}", f"--model_path={model_dir}", "build",
-            f"--dataset={dataset_path}"
+            self._build_script, f"--log_level=info",
+            f"--workspace={engine_dir}", f"--model={hf_model_name}",
+            f"--model_path={model_dir}", "build", f"--dataset={dataset_path}",
+            f"--tp_size={self._config.tp_size}",
+            f"--pp_size={self._config.pp_size}"
         ]
-        if self._config.max_batch_size > 0:
-            build_cmd.append(f"--max_batch_size={self._config.max_batch_size}")
         max_seq_len = max(self._config.input_lens) + max(
             self._config.output_lens)
         build_cmd.append(f"--max_seq_len={max_seq_len}")
@@ -1062,6 +1096,7 @@ def get_trtllm_bench_command(self, engine_dir):
         model_dir = self.get_trtllm_bench_model()
         model_name = self._config.model_name
         dataset_path = os.path.join(engine_dir, "synthetic_data.json")
+        report_path = os.path.join(engine_dir, "report.json")
         if not model_name.endswith("_hf"):
             model_name = model_name + "_hf"
         hf_model_name = HF_MODEL_PATH.get(model_name, "")
@@ -1073,13 +1108,16 @@ def get_trtllm_bench_command(self, engine_dir):
             f"--model_path={model_dir}",
             "throughput",
             f"--dataset={dataset_path}",
+            f"--max_batch_size={self._config.max_batch_size}",
+            f"--max_num_tokens={self._config.max_num_tokens}",
+            f"--report_json={report_path}",
         ]
         if self._config.backend != "pytorch":
             benchmark_cmd += [f"--engine_dir={engine_dir}"]
         else:
             benchmark_cmd += ["--backend=pytorch"]
-        if self._config.max_batch_size > 0:
-            benchmark_cmd += [f"--max_batch_size={self._config.max_batch_size}"]
+        if self._config.num_reqs > 0:
+            benchmark_cmd += [f"--num_requests={self._config.num_reqs}"]
         if self._config.concurrency != -1:
             benchmark_cmd += [f"--concurrency={self._config.concurrency}"]
         if self._config.ep_size != None:
@@ -1093,15 +1131,7 @@ def get_trtllm_bench_command(self, engine_dir):
         #use default yaml config
         if self._config.backend == "pytorch":
             import yaml
-            config = {
-                'enable_attention_dp': True,
-                'pytorch_backend_config': {
-                    'enable_overlap_scheduler': True,
-                    'print_iter_log': True,
-                    'use_cuda_graph': True,
-                    'cuda_graph_batch_sizes': [1, 512]
-                }
-            }
+            config = get_model_yaml_config(self._config.to_string())
             with open('extra-llm-api-config.yml', 'w') as f:
                 yaml.dump(config, f, default_flow_style=False)
             benchmark_cmd += [
@@ -1288,6 +1318,10 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float:
         if len(metric_values) == 0:
             if self._build_script == "trtllm-build" and metric.metric_type == PerfMetricType.ENGINE_SIZE:
                 metric_values = [0.0]
+            elif self._build_script == "trtllm-bench" and self._config.num_gpus > 1 and metric.metric_type == PerfMetricType.BUILD_TIME:
+                print_info("skip building process for multi-gpu test"
+                           )  #https://nvbugspro.nvidia.com/bug/5210111
+                metric_values = [0.0]
             else:
                 raise RuntimeError(
                     f"Cannot find perf result for {metric_name} from perf script logs!"
@@ -1318,7 +1352,7 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float:
                     f"Combining up enc builder_perf {enc_metrics} and dec builder_perf {dec_metrics} to {metric_values}."
                 )
             # For other models, builder metric should equal # gpus.
-            elif self._build_script != "trtllm-build":
+            elif self._build_script != "trtllm-build" and self._build_script != "trtllm-bench":
                 assert len(
                     metric_values
                 ) == num_gpus, f"num of metrics: {len(metric_values)} should match num_gpus: {num_gpus}"
diff --git a/tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml b/tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml
index 3d1509a25386..c953dd8a9e43 100644
--- a/tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml
+++ b/tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml
@@ -26,9 +26,6 @@ trt_llm_integration_perf_test:
   - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-ootb_except_mha-float16-input_output_len:128,128+512,32]
   - perf/test_perf.py::test_perf[gptneox_20b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
   - perf/test_perf.py::test_perf[gptneox_20b-cppmanager-ootb_except_mha-float16-maxbs:1-input_output_len:128,128+512,32-reqs:10]
-  - perf/test_perf.py::test_perf[falcon_rw_1b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[falcon_7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[baichuan2_7b_chat-cppmanager-ootb_except_mha-float16-maxbs:1-input_output_len:128,128-reqs:10]
   - perf/test_perf.py::test_perf[internlm_chat_7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
   - perf/test_perf.py::test_perf[internlm_chat_7b-cppmanager-ootb_except_mha-float16-maxbs:128-input_output_len:128,128]
   - perf/test_perf.py::test_perf[qwen2_7b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
@@ -84,8 +81,6 @@ trt_llm_integration_perf_test:
   - perf/test_perf.py::test_perf[llama_30b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:2]
   - perf/test_perf.py::test_perf[llama_30b-cppmanager-ootb_except_mha-float16-input_output_len:128,128+512,32-gpus:2]
   - perf/test_perf.py::test_perf[llama_30b-cppmanager-ootb_except_mha-float16-gwp:0.5-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[baichuan2_13b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:2]
-  - perf/test_perf.py::test_perf[baichuan2_13b_chat-cppmanager-ootb_except_mha-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
 
 # Tests for systems with 2+ GPUs and high memory
 - condition:
@@ -118,8 +113,6 @@ trt_llm_integration_perf_test:
       system_gpu_count:
         gte: 4
   tests:
-  - perf/test_perf.py::test_perf[opt_66b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:4]
-  - perf/test_perf.py::test_perf[falcon_40b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:4]
   - perf/test_perf.py::test_perf[internlm_chat_20b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:4]
   - perf/test_perf.py::test_perf[internlm_chat_20b-cppmanager-ootb_except_mha-float16-maxbs:64-input_output_len:128,128+512,32-gpus:4]
   - perf/test_perf.py::test_perf[qwen_14b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:4]
@@ -135,14 +128,10 @@ trt_llm_integration_perf_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[bloom_176b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:8]
-  - perf/test_perf.py::test_perf[bloom_176b-cppmanager-ootb_except_mha-float16-maxbs:1-input_output_len:128,128+512,32-reqs:10-gpus:8]
-  - perf/test_perf.py::test_perf[falcon_180b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:8]
   - perf/test_perf.py::test_perf[gpt_175b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128+512,32-reqs:10-gpus:8]
   - perf/test_perf.py::test_perf[gpt_175b-cppmanager-ootb_except_mha-float16-maxbs:1-input_output_len:128,128+512,32-reqs:10-gpus:8]
   - perf/test_perf.py::test_perf[llama_70b_long_context-cppmanager-exe-plugin_ifb-float16-input_output_len:2000,200-reqs:64-gpus:8]
   - perf/test_perf.py::test_perf[llama_70b_long_generation-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-gpus:8]
-  - perf/test_perf.py::test_perf[opt_66b-cppmanager-ootb_except_mha-float16-maxbs:8-input_output_len:128,128+512,32-reqs:80-gpus:8]
 
 # FP8 tests for systems with 8+ GPUs
 - condition:
@@ -152,4 +141,4 @@ trt_llm_integration_perf_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[falcon_180b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128+512,32-quant:fp8-reqs:10-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:128,128+512,32-quant:fp8-gpus:8]
diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml b/tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml
index a0b629093e46..67a272ef43c2 100644
--- a/tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml
+++ b/tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml
@@ -5,13 +5,16 @@ trt_llm_release_perf_cluster_test:
       system_gpu_count:
         gte: 1
   tests:
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128+512,32-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:nvfp4-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v2_13b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-loras:8-gpus:2]
-  - perf/test_perf.py::test_perf[t5_large-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[mistral_7b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000+500,2000-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:512,32-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:nvfp4-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:256-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:256-input_output_len:512,32-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v2_13b-bench-float16-input_output_len:128,128-loras:8-gpus:2]
+  - perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:1000,1000-quant:fp8]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:500,2000-quant:fp8]
   - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
 
 # Tests for systems with 4+ GPUs
@@ -20,11 +23,10 @@ trt_llm_release_perf_cluster_test:
       system_gpu_count:
         gte: 4
   tests:
-  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:512,512-quant:fp8-tp:4]
-  - perf/test_perf.py::test_perf[baichuan2_13b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:2]
-  - perf/test_perf.py::test_perf[qwen_14b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:4]
-  - perf/test_perf.py::test_perf[starcoder_15.5b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,200-gpus:4]
-  - perf/test_perf.py::test_perf[falcon_40b-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-bench-float16-input_output_len:512,512-quant:fp8-tp:4]
+  - perf/test_perf.py::test_perf[qwen_14b_chat-bench-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[qwen_14b_chat-bench-float16-input_output_len:512,32-gpus:4]
+  - perf/test_perf.py::test_perf[starcoder_15b-bench-float16-input_output_len:512,200-gpus:4]
 
 # Tests for systems with 8+ GPUs
 - condition:
@@ -32,10 +34,9 @@ trt_llm_release_perf_cluster_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:512,512-quant:fp8-tp:8]
-  - perf/test_perf.py::test_perf[bloom_176b-cppmanager-exe-plugin_ifb-float16-maxbs:8-input_output_len:128,128+512,32-reqs:80-gpus:8]
-  - perf/test_perf.py::test_perf[falcon_180b-cppmanager-exe-plugin_ifb-float16-maxbs:8-input_output_len:128,128+512,32-reqs:80-gpus:8]
-  - perf/test_perf.py::test_perf[gpt_175b-cppmanager-exe-plugin_ifb-float16-maxbs:8-input_output_len:128,128+512,32-reqs:80-gpus:8]
+  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-bench-float16-input_output_len:512,512-quant:fp8-tp:8]
+  - perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:128,128-reqs:80-gpus:8]
+  - perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:512,32-reqs:80-gpus:8]
 
 # GB chip specific tests with high memory
 - condition:
@@ -50,3 +51,5 @@ trt_llm_release_perf_cluster_test:
   tests:
   - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:512-input_output_len:128,128-quant:fp8-ep:8-tp:8-gpus:8]
   - perf/test_perf.py::test_perf[deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:512-input_output_len:128,128-quant:nvfp4-ep:8-tp:8-gpus:8]
+  - perf/test_perf.py::test_perf[deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:1-input_output_len:1000,2000-quant:nvfp4-reqs:10-ep:4-tp:8-gpus:8] #min latency test
+  - perf/test_perf.py::test_perf[deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-quant:nvfp4-reqs:49152-con:3072-ep:8-tp:8-gpus:8] #max throughput test
diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml b/tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml
index a0439a74d185..525e8dd23f1b 100644
--- a/tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml
+++ b/tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml
@@ -5,29 +5,26 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 1
   tests:
-  # E2E gptSessionBenchmark
-  - perf/test_perf.py::test_perf[gpt_350m_moe-cpp-plugin-float16-maxbs:64-bs:64-input_output_len:128,128]
+  # E2E trtllm-bench
+  - perf/test_perf.py::test_perf[gpt_350m_moe-bench-float16-maxbs:64-input_output_len:128,128]
 
   # E2E BERT
-  - perf/test_perf.py::test_perf[bert_large-cpp-plugin-float16-bs:32-input_len:128+512]
-  - perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512]
+  - perf/test_perf.py::test_perf[bert_large-bench-float16-maxbs:32-input_len:128+512]
+  - perf/test_perf.py::test_perf[roberta_base-bench-float16-maxbs:32-input_len:128+512]
 
   # Common models for all GPUs
-  - perf/test_perf.py::test_perf[starcoder2_3b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:512,200-reqs:10]
-  - perf/test_perf.py::test_perf[mamba_130m-cppmanager-exe-plugin-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[mamba_2.8b-cppmanager-exe-plugin-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[starcoder2_3b-bench-float16-maxbs:1-input_output_len:512,200-reqs:10]
+  - perf/test_perf.py::test_perf[mamba_130m-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[mamba_2.8b-bench-float16-input_output_len:128,128]
 
   # E2E ENC-DEC
-  - perf/test_perf.py::test_perf[t5_base-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[t5_large-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[t5_3b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[flan_t5_base-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[flan_t5_large-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[whisper_large_v3-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[bart_large_cnn-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
   - perf/test_perf.py::test_perf[mbart_large_50_many_to_one_mmt-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[llama_7b-cppmanager-exe-plugin_ifb-bfloat16-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-input_output_len:128,128+512,32]
+  - perf/test_perf.py::test_perf[bart_large_cnn-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[flan_t5_base-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[flan_t5_large-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-bfloat16-input_output_len:128,128+512,32]
 
   # Test list validation
   - test_list_validation.py::test_list_validation
@@ -41,22 +38,21 @@ trt_llm_release_perf_sanity_test:
         gt: 25000
   tests:
   # E2E gptManagerBenchmark IFB
-  - perf/test_perf.py::test_perf[llama_7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128,+512,32]
-  - perf/test_perf.py::test_perf[llama_v2_7b-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v2_7b-bench-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-static_batching-plugin_ifb-float16-bs:8+64-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-gwp:0.0-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[qwen2_7b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-cppmanager-exe-static_batching-plugin_ifb-float16-bs:8+64-input_output_len:128,128+512,32]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-gwp:0.0-input_output_len:128,128+512,32]
+
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-bfloat16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-bfloat16-input_output_len:512,32]
+  - perf/test_perf.py::test_perf[qwen2_7b_instruct-bench-float16-input_output_len:128,128]
 
 # FP8 specific tests
 - condition:
     terms:
       supports_fp8: true
   tests:
-  - perf/test_perf.py::test_perf[llama_7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,32-quant:fp8]
 
 # Tests for systems with 2+ GPUs
 - condition:
@@ -64,16 +60,13 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 2
   tests:
-  - perf/test_perf.py::test_perf[t5_11b-cppmanager-exe-plugin_ifb-float16-bs:1-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[flan_t5_xl-cppmanager-exe-plugin_ifb-float16-bs:1-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[flan_t5_xxl-cppmanager-exe-plugin_ifb-float16-bs:1-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v2_7b-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,128-pp:2]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:int8_sq_per_tensor-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:int8_sq_per_token_channel-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128-gpu:2]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128-gpus:2]
-  - perf/test_perf.py::test_perf[llama_13b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:2]
-  - perf/test_perf.py::test_perf[llama_30b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
+  - perf/test_perf.py::test_perf[t5-bench-float16-maxbs:1-input_output_len:128,20-gpus:2]
+  - perf/test_perf.py::test_perf[flan_t5_large-bench-float16-maxbs:1-input_output_len:128,20-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:int8_sq-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:int8_wo-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:128,128-gpu:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-streaming-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
 
 # FP8 tests for systems with 2+ GPUs
 - condition:
@@ -83,10 +76,10 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 2
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:fp8-gpus:2]
   - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,128-quant:fp8-gpus:2]
-  - perf/test_perf.py::test_perf[llama_30b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:fp8-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
 
 # Tests for systems with 2+ GPUs and high memory
 - condition:
@@ -96,9 +89,8 @@ trt_llm_release_perf_sanity_test:
       gpu_memory:
         gt: 80000
   tests:
-  - perf/test_perf.py::test_perf[llama_70b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
-  - perf/test_perf.py::test_perf[llama_70b_sq_per_tensor-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-gpus:2]
 
 # Tests for systems with 4+ GPUs
 - condition:
@@ -106,10 +98,9 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 4
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:4]
-  - perf/test_perf.py::test_perf[llama_70b_sq_per_tensor-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:4]
-  - perf/test_perf.py::test_perf[qwen_14b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:4]
-  - perf/test_perf.py::test_perf[starcoder_15.5b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:512,200-reqs:10-gpus:4]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:4]
+  - perf/test_perf.py::test_perf[qwen_14b_chat-bench-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[starcoder_15b-bench-float16-maxbs:1-input_output_len:512,200-reqs:10-gpus:4]
 
 # Tests for systems with 8+ GPUs
 - condition:
@@ -117,8 +108,8 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[llama_70b_long_context-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:2000,200-reqs:10-gpus:8]
-  - perf/test_perf.py::test_perf[llama_70b_long_generation-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:200,2000-reqs:10-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:2000,200-reqs:10-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:200,2000-reqs:10-gpus:8]
 
 # Tests for systems with 8+ GPUs and high memory
 - condition:
@@ -128,8 +119,7 @@ trt_llm_release_perf_sanity_test:
       gpu_memory:
         gt: 46000
   tests:
-  - perf/test_perf.py::test_perf[bloom_176b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:8]
-  - perf/test_perf.py::test_perf[gpt_175b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:8]
+  - perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:1-input_output_len:128,128-reqs:10-gpus:8]
 
 # FP8 tests for systems with 8+ GPUs
 - condition:
@@ -139,4 +129,5 @@ trt_llm_release_perf_sanity_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-maxbs:1-input_output_len:128,128+512,32-quant:fp8-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:128,128-quant:fp8-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:1-input_output_len:512,32-quant:fp8-gpus:8]
diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_test.yml b/tests/integration/test_lists/qa/trt_llm_release_perf_test.yml
index 73622d32ed86..92298619e8ab 100644
--- a/tests/integration/test_lists/qa/trt_llm_release_perf_test.yml
+++ b/tests/integration/test_lists/qa/trt_llm_release_perf_test.yml
@@ -13,34 +13,37 @@ trt_llm_release_perf_test:
   - perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32+64-input_len:128+512]
 
   # E2E gptManagerBenchmark IFB
-  - perf/test_perf.py::test_perf[llama_7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b-bench-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b_hf-bench-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[mistral_7b_v0.1_hf-bench-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128+512,32]
   - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-static_batching-plugin_ifb-float16-bs:8+64-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32]
   - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-input_output_len:128,128+512,32]
   - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-gwp:0.0-input_output_len:128,128+512,32]
   - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-bfloat16-gwp:0.5-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[qwen2_7b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128]
-  - perf/test_perf.py::test_perf[starcoder2_3b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,200]
 
   # E2E ENC-DEC
-  - perf/test_perf.py::test_perf[t5_base-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[t5_large-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
   - perf/test_perf.py::test_perf[t5_large-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[t5_3b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[flan_t5_base-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[flan_t5_large-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[whisper_large_v3-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
+
+  # E2E trtllm-bench
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-streaming-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-streaming-float16-input_output_len:512,32]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,32]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-bfloat16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:512,32]
+  - perf/test_perf.py::test_perf[qwen2_7b_instruct-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[starcoder2_3b-bench-float16-input_output_len:512,200]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:128,128]
+
+  # E2E ENC-DEC
   - perf/test_perf.py::test_perf[bart_large_cnn-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
   - perf/test_perf.py::test_perf[mbart_large_50_many_to_one_mmt-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20]
-  - perf/test_perf.py::test_perf[mamba_370m-cppmanager-exe-plugin-float16-input_output_len:128,128+512,32]
-  - perf/test_perf.py::test_perf[mamba_2.8b-cppmanager-exe-plugin-float16-input_output_len:128,128+512,32]
-
+  - perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[flan_t5_base-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[flan_t5_large-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20]
+  - perf/test_perf.py::test_perf[mamba_370m-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[mamba_370m-bench-float16-input_output_len:512,32]
+  - perf/test_perf.py::test_perf[mamba_2.8b-bench-float16-input_output_len:128,128]
+  - perf/test_perf.py::test_perf[mamba_2.8b-bench-float16-input_output_len:512,32]
   # Test list validation
   - test_list_validation.py::test_list_validation
 
@@ -49,18 +52,18 @@ trt_llm_release_perf_test:
       system_gpu_count:
         gte: 2
   tests:
-  - perf/test_perf.py::test_perf[t5_11b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[flan_t5_xl-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[flan_t5_xxl-cppmanager-exe-plugin_ifb-float16-input_output_len:128,20-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:int8_sq_per_tensor-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:int8_sq_per_token_channel-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128-gpus:2]
   - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-mp-maxbs:256-input_output_len:128,128-pp:2]
-  - perf/test_perf.py::test_perf[llama_v2_13b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-loras:8-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-loras:8-gpus:2]
-  - perf/test_perf.py::test_perf[llama_13b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[llama_30b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-tp:2]
+  - perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20-gpus:2]
+  - perf/test_perf.py::test_perf[flan_t5_large-bench-float16-input_output_len:128,20-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:int8_sq-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:int8_wo-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-streaming-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:128,128-loras:8-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-loras:8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:512,32-gpus:2]
 
 - condition:
     ranges:
@@ -70,37 +73,24 @@ trt_llm_release_perf_test:
         gt: 80000
   tests:
   - perf/test_perf.py::test_perf[llama_70b_sq_per_tensor-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:1024,1024-tp:2]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-tp:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:2]
   - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-plugin_ifb-float16-mp-input_output_len:128,128+512,32-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-maxbs:256-input_output_len:128,128+512,32-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128+512,32-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-streaming-float16-input_output_len:128,128-gpus:2]
 
 - condition:
     ranges:
       system_gpu_count:
         gte: 4
   tests:
-  - perf/test_perf.py::test_perf[flan_t5_xxl-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:4]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-streaming-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:4]
-  - perf/test_perf.py::test_perf[llama_70b_sq_per_tensor-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:4]
-  - perf/test_perf.py::test_perf[qwen_14b_chat-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-gpus:4]
-  - perf/test_perf.py::test_perf[starcoder_15.5b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,200-gpus:4]
-
-- condition:
-    ranges:
-      system_gpu_count:
-        gte: 8
-  tests:
-  - perf/test_perf.py::test_perf[gpt_175b-cppmanager-exe-plugin_ifb-float16-maxbs:8-input_output_len:128,128+512,32-reqs:80-gpus:8]
-  - perf/test_perf.py::test_perf[llama_70b_long_context-cppmanager-exe-plugin_ifb-float16-input_output_len:2000,200-reqs:64-gpus:8]
-  - perf/test_perf.py::test_perf[llama_70b_long_generation-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-gpus:8]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-gpus:8]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-con:200-gpus:8]
-  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-con:1-gpus:8]
-  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-reqs:64-gpus:8]
-  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-reqs:64-con:50-gpus:8]
+  - perf/test_perf.py::test_perf[flan_t5_xxl-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[flan_t5_xxl-cppmanager-exe-plugin_ifb-float16-input_output_len:512,32-gpus:4]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-streaming-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:512,32-gpus:4]
+  - perf/test_perf.py::test_perf[qwen_14b_chat-bench-float16-input_output_len:128,128-gpus:4]
+  - perf/test_perf.py::test_perf[qwen_14b_chat-bench-float16-input_output_len:512,32-gpus:4]
+  - perf/test_perf.py::test_perf[starcoder_15b-bench-float16-input_output_len:512,200-gpus:4]
 
 - condition:
     ranges:
@@ -109,10 +99,22 @@ trt_llm_release_perf_test:
       gpu_memory:
         gt: 80000
   tests:
-  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-reqs:64-con:1-gpus:8]
+  # E2E trtllm-bench
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-tp:2-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-loras:8-tp:2-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-bench-float16-input_output_len:128,128-reqs:64-gpus:8]
   - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-bench-float16-input_output_len:128,128-reqs:64-con:50-gpus:8]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1_instruct-bench-float16-input_output_len:128,128-reqs:64-con:1-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:1024,1024-tp:2-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:200,2000-reqs:64-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-bench-float16-input_output_len:200,2000-reqs:64-con:200-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-bench-float16-input_output_len:200,2000-reqs:64-con:1-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b_instruct-bench-float16-input_output_len:2000,200-reqs:64-gpus:8]
+  - perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:128,128-reqs:80-gpus:8]
+  - perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:512,32-reqs:80-gpus:8]
 
 - condition:
+    # for 8xH200
     ranges:
       system_gpu_count:
         gte: 8
@@ -121,31 +123,29 @@ trt_llm_release_perf_test:
   tests:
   - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:32-input_output_len:128,128-quant:fp8-ep:8-tp:8-gpus:8]
   - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-streaming-float16-maxbs:32-input_output_len:128,128-quant:fp8-ep:8-tp:8-gpus:8]
+  - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:1-input_output_len:1000,2000-reqs:10-quant:fp8-con:1-ep:4-tp:8-gpus:8] #min latency test
+  - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:128-maxnt:1127-input_output_len:1000,2000-reqs:5120-quant:fp8-con:1024-ep:8-tp:8-gpus:8] #max throughput test
 
 # FP8 specific tests
 - condition:
     terms:
       supports_fp8: true
   tests:
-  - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:fp8]
   - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128-beams:4-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:int4_awq]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:w4a8_awq]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-maxbs:256-input_output_len:128,128+512,32-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct_hf-bench-float16-input_output_len:128,128-quant:fp8]
-  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct_hf-bench-pytorch-float16-input_output_len:128,128-quant:fp8]
-  - perf/test_perf.py::test_perf[mistral_7b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000+500,2000-quant:fp8]
-  - perf/test_perf.py::test_perf[phi_3_mini_4k_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000+500,2000-quant:fp8]
 
-- condition:
-    terms:
-      supports_fp8: true
-    ranges:
-      gpu_memory:
-        gt: 80000
-  tests:
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:int4_awq]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:w4a8_awq]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-maxbs:256-input_output_len:512,32-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:1000,1000-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-quant:fp8]
   - perf/test_perf.py::test_perf[deepseek_v3_lite_fp8-bench-pytorch-float16-input_output_len:128,128-quant:fp8]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:1000,1000-quant:fp8]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:500,2000-quant:fp8]
+  - perf/test_perf.py::test_perf[phi_3_mini_4k_instruct-bench-float16-input_output_len:1000,1000-quant:fp8]
+  - perf/test_perf.py::test_perf[phi_3_mini_4k_instruct-bench-float16-input_output_len:500,2000-quant:fp8]
 
 - condition:
     terms:
@@ -154,11 +154,15 @@ trt_llm_release_perf_test:
       system_gpu_count:
         gte: 2
   tests:
-  - perf/test_perf.py::test_perf[llama_13b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-quant:fp8-gpus:2]
-  - perf/test_perf.py::test_perf[llama_30b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,200-quant:fp8-gpus:2]
-  - perf/test_perf.py::test_perf[mixtral_8x7b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-quant:fp8-gpus:2]
-  - perf/test_perf.py::test_perf[mistral_7b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000+500,2000-quant:fp8-tp:2]
-  - perf/test_perf.py::test_perf[phi_3_mini_128k_instruct-cppmanager-exe-plugin_ifb-float16-input_output_len:1000,1000+500,2000-quant:fp8-tp:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:512,32-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:512,200-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:512,32-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:1000,1000-quant:fp8-tp:2]
+  - perf/test_perf.py::test_perf[mistral_7b_v0.1-bench-float16-input_output_len:500,2000-quant:fp8-tp:2]
+  - perf/test_perf.py::test_perf[phi_3_mini_128k_instruct-bench-float16-input_output_len:1000,1000-quant:fp8-tp:2]
+  - perf/test_perf.py::test_perf[phi_3_mini_128k_instruct-bench-float16-input_output_len:500,2000-quant:fp8-tp:2]
 
 - condition:
     terms:
@@ -170,6 +174,8 @@ trt_llm_release_perf_test:
         gt: 80000
   tests:
   - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:128,128-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[mixtral_8x7b_v0.1-bench-float16-input_output_len:512,32-quant:fp8-gpus:2]
+  - perf/test_perf.py::test_perf[llama_v3.2_11b-bench-float16-input_output_len:512,200-quant:fp8-gpus:2]
   - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:128,128-quant:fp8-tp:2]
 
 - condition:
@@ -179,9 +185,9 @@ trt_llm_release_perf_test:
       system_gpu_count:
         gte: 4
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,200-quant:fp8-tp:4]
-  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:512,512-quant:fp8-tp:4]
-  - perf/test_perf.py::test_perf[starcoder_15.5b-cppmanager-exe-plugin_ifb-float16-input_output_len:512,512-quant:fp8-tp:4]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:512,200-quant:fp8-tp:4]
+  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-bench-float16-input_output_len:512,512-quant:fp8-tp:4]
+  - perf/test_perf.py::test_perf[starcoder_15b-bench-float16-input_output_len:512,512-quant:fp8-tp:4]
 
 - condition:
     terms:
@@ -190,8 +196,9 @@ trt_llm_release_perf_test:
       system_gpu_count:
         gte: 8
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_70b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128+512,32-quant:fp8-gpus:8]
-  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-cppmanager-exe-plugin_ifb-float16-input_output_len:512,512-quant:fp8-tp:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:128,128-quant:fp8-gpus:8]
+  - perf/test_perf.py::test_perf[llama_v3.1_70b-bench-float16-input_output_len:512,32-quant:fp8-gpus:8]
+  - perf/test_perf.py::test_perf[mixtral_8x22b_v0.1-bench-float16-input_output_len:512,512-quant:fp8-tp:8]
 
 # GB chip specific tests
 - condition:
@@ -199,7 +206,7 @@ trt_llm_release_perf_test:
       chip: 'gb*'
       linux_distribution_name: '*'
   tests:
-  - perf/test_perf.py::test_perf[llama_v3.1_8b-cppmanager-exe-plugin_ifb-float16-input_output_len:128,128-quant:nvfp4]
+  - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-quant:nvfp4]
   - perf/test_perf.py::test_perf[deepseek_v3_lite_nvfp4-bench-pytorch-float16-input_output_len:128,128-quant:nvfp4]
   - perf/test_perf.py::test_perf[deepseek_v3_lite_fp8-bench-pytorch-float16-input_output_len:128,128-quant:fp8]
 
@@ -215,5 +222,9 @@ trt_llm_release_perf_test:
       linux_distribution_name: '*'
   tests:
   - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:512-input_output_len:128,128-quant:fp8-ep:8-tp:8-gpus:8]
+  - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:1-input_output_len:1000,2000-quant:fp8-reqs:10-ep:4-tp:8-gpus:8] #min latency test
+  - perf/test_perf.py::test_perf[deepseek_r1-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-quant:fp8-reqs:49152-con:3072-ep:8-tp:8-gpus:8] #max throughput test
   - perf/test_perf.py::test_perf[deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:512-input_output_len:128,128-quant:nvfp4-ep:8-tp:8-gpus:8]
   - perf/test_perf.py::test_perf[deepseek_r1_nvfp4-bench-pytorch-streaming-float16-maxbs:512-input_output_len:1000,1000-quant:nvfp4-con:4096-ep:8-tp:8-gpus:8]
+  - perf/test_perf.py::test_perf[deepseek_r1-nvfp4-bench-pytorch-float16-maxbs:1-input_output_len:1000,2000-quant:nvfp4-reqs:10-ep:4-tp:8-gpus:8] #min latency test
+  - perf/test_perf.py::test_perf[deepseek_r1-nvfp4-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-quant:nvfp4-reqs:49152-con:3072-ep:8-tp:8-gpus:8] #max throughput test


Reviews:
user: login: kaiyux
body: Thanks for the changes. Do we have numbers that the changes are not going to change the baseline? That said, no known regression will happen after switch from `gptManagerBenchmark` to `trtllm-bench` on those cases?
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-02T02:20:32Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T05:04:05Z
user: login: kaiyux
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-02T16:07:02Z
user: login: LarryXFly
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-09T12:16:53Z
user: login: kaiyux
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T03:24:49Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T06:59:13Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T06:59:57Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T07:03:32Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T07:04:53Z
user: login: kaiyux
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T08:07:21Z
user: login: kaiyux
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T08:09:11Z
user: login: ruodil
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T04:02:27Z
user: login: kaiyux
body: Thanks for the migration!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-15T08:08:13Z

Review comments:
diff_hunk: @@ -1253,6 +1257,10 @@ def get_commands(self):
                 ]
             else:
                 mpi_cmd = ["mpirun", "-n", f"{num_gpus}", "--allow-run-as-root"]
+        elif num_gpus > 1 and num_gpus <= 8 and self._config.runtime == "bench":
+            mpi_cmd = [
+                "mpirun", "-n", "1", "--oversubscribe", "--allow-run-as-root"
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: `trtllm-bench` does not require `mpirun` to launch multi-gpu benchmarking.
created_at: 2025-04-02T02:16:09Z
author_association: MEMBER
diff_hunk: @@ -1253,6 +1257,10 @@ def get_commands(self):
                 ]
             else:
                 mpi_cmd = ["mpirun", "-n", f"{num_gpus}", "--allow-run-as-root"]
+        elif num_gpus > 1 and num_gpus <= 8 and self._config.runtime == "bench":
+            mpi_cmd = [
+                "mpirun", "-n", "1", "--oversubscribe", "--allow-run-as-root"
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: Thanks for reminder. I'll start a test on this branch to make sure there are no regression between same cases using trtllm-bench and gptmanager benchmark. Marking this commit as draft before validating the result.
created_at: 2025-04-02T05:04:05Z
author_association: COLLABORATOR
diff_hunk: @@ -1253,6 +1257,10 @@ def get_commands(self):
                 ]
             else:
                 mpi_cmd = ["mpirun", "-n", f"{num_gpus}", "--allow-run-as-root"]
+        elif num_gpus > 1 and num_gpus <= 8 and self._config.runtime == "bench":
+            mpi_cmd = [
+                "mpirun", "-n", "1", "--oversubscribe", "--allow-run-as-root"
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: @ruodil Thanks, please let me know if that's too heavy, we should be able to remove some old cases such as falcon and bloom.
created_at: 2025-04-02T16:07:02Z
author_association: MEMBER
diff_hunk: @@ -360,9 +386,13 @@ def to_string(self,
             entries.append(f"mp")
 
         # Add Max batch size.
-        if self.max_batch_size > 0:
+        if self.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: Why do we need this `if`?
created_at: 2025-04-10T03:20:27Z
author_association: MEMBER
diff_hunk: @@ -814,8 +850,10 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
         gpu_percent = self._config.gpu_weights_percent
         if gpu_percent != -1:
             build_cmd += [f"--weight_streaming"]
-        if self._config.max_batch_size > 0:
+        if self._config.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: Why is `if` needed here?
created_at: 2025-04-10T03:21:48Z
author_association: MEMBER
diff_hunk: @@ -1058,10 +1100,84 @@ def get_gpt_session_runtime_benchmark_command(self, engine_dir, bs,
             benchmark_cmd.append(f"--input_len={input_len}")
         return benchmark_cmd
 
+    def get_model_yaml_config(self, model_label: str) -> dict:
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: nit: Maybe we should split this function to a separate file?
created_at: 2025-04-10T03:22:40Z
author_association: MEMBER
diff_hunk: @@ -1288,6 +1401,10 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float:
         if len(metric_values) == 0:
             if self._build_script == "trtllm-build" and metric.metric_type == PerfMetricType.ENGINE_SIZE:
                 metric_values = [0.0]
+            elif self._build_script == "trtllm-bench" and self._config.num_gpus > 1 and metric.metric_type == PerfMetricType.BUILD_TIME:
+                print_info("skip building process for multi-gpu test"
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: Please correct me if my understanding is wrong - this only skip the metric, but not the build process, right?
```suggestion
                print_info("WAR: skip build time metric for multi-gpu test"
```
created_at: 2025-04-10T03:23:55Z
author_association: MEMBER
diff_hunk: @@ -360,9 +386,13 @@ def to_string(self,
             entries.append(f"mp")
 
         # Add Max batch size.
-        if self.max_batch_size > 0:
+        if self.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: the max_batch_size is set to default as 2048 in PerfTestConfig as well as in tekit/tensorrt_llm/commands/build.py, so if it's set to other value, it'll be passed to test_case name and trtllm-bench build params
created_at: 2025-04-10T06:59:13Z
author_association: COLLABORATOR
diff_hunk: @@ -814,8 +850,10 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
         gpu_percent = self._config.gpu_weights_percent
         if gpu_percent != -1:
             build_cmd += [f"--weight_streaming"]
-        if self._config.max_batch_size > 0:
+        if self._config.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: the max_batch_size is set to default as 2048 in PerfTestConfig as well as in tekit/tensorrt_llm/commands/build.py, so if it's set to other value, it'll be passed to test_case name and trtllm-bench build params
created_at: 2025-04-10T06:59:57Z
author_association: COLLABORATOR
diff_hunk: @@ -1058,10 +1100,84 @@ def get_gpt_session_runtime_benchmark_command(self, engine_dir, bs,
             benchmark_cmd.append(f"--input_len={input_len}")
         return benchmark_cmd
 
+    def get_model_yaml_config(self, model_label: str) -> dict:
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: do you think we'd better put it in a new py file(model_configs.py) or put it in utils.py?
created_at: 2025-04-10T07:03:32Z
author_association: COLLABORATOR
diff_hunk: @@ -1288,6 +1401,10 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float:
         if len(metric_values) == 0:
             if self._build_script == "trtllm-build" and metric.metric_type == PerfMetricType.ENGINE_SIZE:
                 metric_values = [0.0]
+            elif self._build_script == "trtllm-bench" and self._config.num_gpus > 1 and metric.metric_type == PerfMetricType.BUILD_TIME:
+                print_info("skip building process for multi-gpu test"
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: yes, thanks for suggestion
created_at: 2025-04-10T07:04:52Z
author_association: COLLABORATOR
diff_hunk: @@ -814,8 +850,10 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
         gpu_percent = self._config.gpu_weights_percent
         if gpu_percent != -1:
             build_cmd += [f"--weight_streaming"]
-        if self._config.max_batch_size > 0:
+        if self._config.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: Yes, I meant if `self._config.max_batch_size` equals to 2048, we can still do `build_cmd.append(f"--max_batch_size={self._config.max_batch_size}")`, no?
created_at: 2025-04-10T08:07:20Z
author_association: MEMBER
diff_hunk: @@ -1058,10 +1100,84 @@ def get_gpt_session_runtime_benchmark_command(self, engine_dir, bs,
             benchmark_cmd.append(f"--input_len={input_len}")
         return benchmark_cmd
 
+    def get_model_yaml_config(self, model_label: str) -> dict:
path: tests/integration/defs/perf/test_perf.py
user: login: kaiyux
body: Since it's just including some naive Python dict, my personal suggestion is to use a separate file like `model_yaml_config.py`, if that makes sense to make code cleaner.
created_at: 2025-04-10T08:09:11Z
author_association: MEMBER
diff_hunk: @@ -814,8 +850,10 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list:
         gpu_percent = self._config.gpu_weights_percent
         if gpu_percent != -1:
             build_cmd += [f"--weight_streaming"]
-        if self._config.max_batch_size > 0:
+        if self._config.max_batch_size != 2048:
path: tests/integration/defs/perf/test_perf.py
user: login: ruodil
body: yes, I'll remove this if condition and pass max_batch_size to build_cmd
created_at: 2025-04-11T04:02:27Z
author_association: COLLABORATOR

PR comments:
user: login: kaiyux
created_at: 2025-04-15T23:55:31Z
body: /bot run
author_association: MEMBER
user: login: tensorrt-cicd
created_at: 2025-04-16T00:01:04Z
body: [PR_Github #2374](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2374/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T01:10:27Z
body: [PR_Github #2374](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2374/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1709](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1709/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: ruodil
created_at: 2025-04-16T03:50:00Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T03:57:04Z
body: [PR_Github #2402](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2402/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T04:04:00Z
body: [PR_Github #2402](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2402/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #1725](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1725/) completed with status: 'ABORTED'

author_association: COLLABORATOR
user: login: ruodil
created_at: 2025-04-16T04:06:42Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T04:12:38Z
body: [PR_Github #2406](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2406/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T04:31:39Z
body: [PR_Github #2406](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2406/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #1729](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1729/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: ruodil
created_at: 2025-04-16T06:03:28Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T06:08:55Z
body: [PR_Github #2427](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2427/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-16T10:14:15Z
body: [PR_Github #2427](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/2427/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1743](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1743/) completed with status: 'FAILURE'

author_association: COLLABORATOR