Repository: NVIDIA/TensorRT-LLM
Pull request #4248: [test] add qa test mentioned in docs
Author: login: crazydemo
Created: 2025-05-13T05:13:40Z | Merged: 2025-05-15T05:37:11Z

This PR added below test cases:
1. TestLlama3_3_70BInstruct bfloat16 accuracy test on mmlu and gsm8k.
2. TestNemotronH-8b bf16 accuracy test  on mmlu and gsm8k. (adjust the max_batch_size, due to [5264431](https://nvbugspro.nvidia.com/bug/5264431)
3. quickstart test case on qwen3 and llm_decoder.


Diff:
diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index 161ca1b0bac6..d9c9980d5e61 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -146,7 +146,8 @@ def get_num_samples_and_threshold(self, **acc_specs):
     def evaluate(self,
                  llm: Union[LLM, PyTorchLLM],
                  extra_acc_spec: Optional[str] = None,
-                 extra_evaluator_kwargs: Optional[dict] = None):
+                 extra_evaluator_kwargs: Optional[dict] = None,
+                 sampling_params: Optional[SamplingParams] = None):
         assert self.EVALUATOR_CLS is not None
 
         if llm.args.speculative_config is None:
@@ -175,9 +176,15 @@ def evaluate(self,
                 spec_dec_algo=spec_dec_algo,
                 extra_acc_spec=extra_acc_spec)
 
-        sampling_params = SamplingParams(
-            max_tokens=self.MAX_OUTPUT_LEN,
-            truncate_prompt_tokens=self.MAX_INPUT_LEN)
+        if sampling_params is None:
+            sampling_params = SamplingParams(
+                max_tokens=self.MAX_OUTPUT_LEN,
+                truncate_prompt_tokens=self.MAX_INPUT_LEN)
+        else:
+            if sampling_params.max_tokens is None:
+                sampling_params.max_tokens = self.MAX_OUTPUT_LEN
+            if sampling_params.truncate_prompt_tokens is None:
+                sampling_params.truncate_prompt_tokens = self.MAX_INPUT_LEN
 
         evaluator_kwargs = {}
         if self.EVALUATOR_KWARGS is not None:
diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index ff41314210b3..0dab8eed350b 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -6,6 +6,7 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 72.85
 meta-llama/Llama-3.3-70B-Instruct:
+  - accuracy: 84.07
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 75.61
@@ -61,3 +62,5 @@ Qwen3/Qwen3-30B-A3B:
     accuracy: 83.43
 nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - accuracy: 92.57
+nvidia/Nemotron-H-8B-Base-8K:
+  - accuracy: 46.20
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 183463d71fc7..86896ae27c28 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -22,10 +22,14 @@ meta-llama/Llama-3.1-8B-Instruct:
   - accuracy: 68.17
   - quant_algo: FP8
     accuracy: 67.93
+  - quant_algo: FP8
+    extra_acc_spec: temperature=0.8,top_p=0.95
+    accuracy: 64.62
   - quant_algo: FP8
     kv_cache_quant_algo: FP8
     accuracy: 67.87
 meta-llama/Llama-3.3-70B-Instruct:
+  - accuracy: 81.28
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 79.31
@@ -114,4 +118,4 @@ nvidia/Llama-3_3-Nemotron-Super-49B-v1:
 nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
   - accuracy: 57.97
 nvidia/Nemotron-H-8B-Base-8K:
-  - accuracy: 87.573
+  - accuracy: 69.590
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 69f3d7e13ae1..faee75251355 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -16,7 +16,8 @@
 
 from tensorrt_llm._torch import LLM
 from tensorrt_llm._torch.pyexecutor.config import PyTorchConfig
-from tensorrt_llm.llmapi import KvCacheConfig, MTPDecodingConfig
+from tensorrt_llm.llmapi import (LLM, KvCacheConfig, MTPDecodingConfig,
+                                 SamplingParams)
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
 
@@ -178,9 +179,36 @@ def test_fp8_4gpus(self, tp_size, pp_size, fp8kv, attn_backend,
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
+    @skip_pre_hopper
+    def test_fp8_llm_decoder(self):
+        model_path = f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct-FP8"
+        pytorch_config = PyTorchConfig(enable_trtllm_decoder=True)
+        llm = LLM(model_path, pytorch_backend_config=pytorch_config)
+        assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+
+        sampling_params = SamplingParams(
+            temperature=0.8,
+            top_p=0.95,
+        )
+
+        with llm:
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm,
+                          sampling_params=sampling_params,
+                          extra_acc_spec="temperature=0.8,top_p=0.95")
+
 
 class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/llama-3.3-models/Llama-3.3-70B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    def test_auto_dtype(self):
+        with LLM(self.MODEL_PATH, tensor_parallel_size=8) as llm:
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm)
+            task = GSM8K(self.MODEL_NAME)
+            task.evaluate(llm)
 
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_device_not_contain(["H100", "B200"])
@@ -730,10 +758,13 @@ class TestNemotronH(LlmapiAccuracyTestHarness):
     MODEL_NAME = "nvidia/Nemotron-H-8B-Base-8K"
     MODEL_PATH = f"{llm_models_root()}/Nemotron-H-8B-Base-8K"
 
-    @pytest.mark.skip(reason="https://nvbugspro.nvidia.com/bug/5264431")
     def test_auto_dtype(self):
+        # TODO: remove max_batch_size after mamba cache manager is supported
+        # ToDo: check 47b and 56b model
         kv_cache_config = KvCacheConfig(enable_block_reuse=False)
-        with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm:
+        with LLM(self.MODEL_PATH,
+                 kv_cache_config=kv_cache_config,
+                 max_batch_size=128) as llm:
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
             task = GSM8K(self.MODEL_NAME)
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 1d8ec3d50b12..35958d705b9b 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1259,6 +1259,7 @@ def test_ptp_quickstart(llm_root, llm_venv):
     ("Llama3.2-11B-BF16", "llama-3.2-models/Llama-3.2-11B-Vision"),
     ("Nemotron4_4B-BF16", "nemotron/Minitron-4B-Base"),
     ("Nemotron-H-8B", "Nemotron-H-8B-Base-8K"),
+    ("Qwen3-30B-A3B", "Qwen3/Qwen3-30B-A3B"),
     pytest.param('Llama3.1-8B-NVFP4',
                  'nvfp4-quantized/Meta-Llama-3.1-8B',
                  marks=skip_pre_blackwell),
@@ -1299,10 +1300,12 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path):
                                          dir="./",
                                          delete=True,
                                          delete_on_close=True) as running_log:
+            kv_cache_fraction = 0.6 if "Qwen3" in model_name else None
             llm_venv.run_cmd([
                 str(example_root / "quickstart_advanced.py"),
                 "--enable_overlap_scheduler",
                 "--enable_chunked_prefill",
+                f"--kv_cache_fraction={kv_cache_fraction}",
                 "--model_dir",
                 f"{llm_models_root()}/{model_path}",
             ],
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 04e3348c27c3..7f774a76461b 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -423,6 +423,8 @@ accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2
 accuracy/test_llm_api.py::TestMixtral8x7B::test_smooth_quant_tp2pp2
 accuracy/test_llm_api.py::TestMixtral8x7BInstruct::test_awq_tp2
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
+accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_decoder
+accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4
 accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype
@@ -472,6 +474,7 @@ test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.2-11B-BF16-llama-3.2-models/Llama-3.2-11B-Vision]
 test_e2e.py::test_ptp_quickstart_advanced[Nemotron4_4B-BF16-nemotron/Minitron-4B-Base]
 test_e2e.py::test_ptp_quickstart_advanced[Nemotron-H-8B-Nemotron-H-8B-Base-8K]
+test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-70B-BF16-llama-3.1-model/Meta-Llama-3.1-70B]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-70B-FP8-llama-3.1-model/Llama-3.1-70B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-405B-FP8-llama-3.1-model/Llama-3.1-405B-Instruct-FP8]


Reviews:
user: login: syuoni
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-13T05:52:57Z
user: login: syuoni
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-13T05:53:11Z
user: login: crazydemo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-13T07:32:43Z
user: login: LarryXFly
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-15T05:36:31Z

Review comments:
diff_hunk: @@ -1219,11 +1219,31 @@ def test_ptp_quickstart(llm_root, llm_venv):
     venv_check_call(llm_venv, [str(example_root / "quickstart.py")])
 
 
+@pytest.mark.parametrize("model_name,model_path", [
+    pytest.param('Llama3.1-8B-FP8',
+                 'llama-3.1-model/Llama-3.1-8B-Instruct-FP8',
+                 marks=skip_pre_hopper),
+])
+def test_ptp_quickstart_advanced_llm_decoder(llm_root, llm_venv, model_name,
+                                             model_path):
+    print(f"Testing {model_name}.")
+    example_root = Path(os.path.join(llm_root, "examples", "pytorch"))
+    llm_venv.run_cmd([
+        str(example_root / "quickstart_advanced.py"),
+        "--max_batch_size=4",
+        "--enable_trtllm_decoder",
+        "--temperature=0.8",
+        "--top_p=0.95",
+        f"{llm_models_root()}/{model_path}",
+    ])
path: tests/integration/defs/test_e2e.py
user: login: syuoni
body: Could we make it as an accuracy test?
created_at: 2025-05-13T05:52:57Z
author_association: COLLABORATOR
diff_hunk: @@ -1219,11 +1219,31 @@ def test_ptp_quickstart(llm_root, llm_venv):
     venv_check_call(llm_venv, [str(example_root / "quickstart.py")])
 
 
+@pytest.mark.parametrize("model_name,model_path", [
+    pytest.param('Llama3.1-8B-FP8',
+                 'llama-3.1-model/Llama-3.1-8B-Instruct-FP8',
+                 marks=skip_pre_hopper),
+])
+def test_ptp_quickstart_advanced_llm_decoder(llm_root, llm_venv, model_name,
+                                             model_path):
+    print(f"Testing {model_name}.")
+    example_root = Path(os.path.join(llm_root, "examples", "pytorch"))
+    llm_venv.run_cmd([
+        str(example_root / "quickstart_advanced.py"),
+        "--max_batch_size=4",
+        "--enable_trtllm_decoder",
+        "--temperature=0.8",
+        "--top_p=0.95",
+        f"{llm_models_root()}/{model_path}",
+    ])
path: tests/integration/defs/test_e2e.py
user: login: crazydemo
body: do `mmlu` check in `test_llm_api_pytorch.py`. Could you please re-review this PR? There's some changes in `accuracy_core.py`.
created_at: 2025-05-13T07:32:43Z
author_association: COLLABORATOR

PR comments:
user: login: crazydemo
created_at: 2025-05-13T05:18:05Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T05:24:29Z
body: [PR_Github #4958](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4958/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T05:59:48Z
body: [PR_Github #4958](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4958/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #3600](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3600/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-05-13T07:32:51Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T07:39:03Z
body: [PR_Github #4978](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4978/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T11:29:20Z
body: [PR_Github #4978](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4978/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3615](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3615/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-05-14T02:46:01Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-14T02:55:26Z
body: [PR_Github #5094](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5094/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-14T05:33:14Z
body: [PR_Github #5094](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5094/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3711](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3711/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-05-14T07:02:44Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-14T07:23:17Z
body: [PR_Github #5132](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5132/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-14T12:32:33Z
body: [PR_Github #5132](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5132/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3739](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3739/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-05-15T02:32:53Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-15T02:39:21Z
body: [PR_Github #5251](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5251/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-05-15T05:14:01Z
body: /bot run --skip_test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-15T05:22:04Z
body: [PR_Github #5269](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5269/) Bot args parsing error: usage: /bot [-h]
            {run,kill,skip,submit,reviewers,reuse-pipeline,reuse-review} ...
/bot: error: unrecognized arguments: --skip_test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-15T08:04:10Z
body: [PR_Github #5251](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5251/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3837](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3837/) completed with status: 'FAILURE'

author_association: COLLABORATOR