Repository: NVIDIA/TensorRT-LLM
Pull request #3404: test: add torch flow test case in qa test list
Author: login: crazydemo
Created: 2025-04-09T06:38:02Z | Merged: 2025-04-11T08:57:41Z

This PR refers to [torch flow support list](https://github.com/NVIDIA/TensorRT-LLM/pull/3179/files).

- [x] verify threshold locally


Diff:
diff --git a/tensorrt_llm/_torch/models/modeling_llama.py b/tensorrt_llm/_torch/models/modeling_llama.py
index d39fdaff7386..d02f92549032 100644
--- a/tensorrt_llm/_torch/models/modeling_llama.py
+++ b/tensorrt_llm/_torch/models/modeling_llama.py
@@ -604,6 +604,14 @@ def __init__(
         self,
         model_config: ModelConfig[LlamaConfig],
     ):
+        # to support MistralConfig
+        if not hasattr(model_config.pretrained_config, 'attention_bias'):
+            model_config.pretrained_config.attention_bias = False
+        if not hasattr(model_config.pretrained_config, 'rope_scaling'):
+            model_config.pretrained_config.rope_scaling = None
+        if not hasattr(model_config.pretrained_config, 'mlp_bias'):
+            model_config.pretrained_config.mlp_bias = False
+
         super().__init__(LlamaModel(model_config),
                          config=model_config,
                          hidden_size=model_config.pretrained_config.hidden_size,
diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
index 942c74341f91..57f28496ee7d 100644
--- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
+++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
@@ -147,6 +147,8 @@ meta-llama/Llama-3.3-70B-Instruct:
     accuracy: 34.383
   - quant_algo: FP8
     accuracy: 34.927
+mistralai/Mistral-7B-v0.1:
+  - accuracy: 25.741
 mistralai/Mixtral-8x7B-v0.1:
   - accuracy: 28.810
   - quant_algo: NVFP4
@@ -227,9 +229,13 @@ Qwen/Qwen2.5-7B-Instruct:
   - quant_algo: FP8
     accuracy: 33.248
 nvidia/Nemotron-Mini-4B-Instruct:
+  - quant_algo: FP8
+    accuracy: 25.247
   - quant_algo: FP8
     kv_cache_quant_algo: FP8
     accuracy: 25.72
+nemotron-nas/Llama-3_1-Nemotron-51B-Instruct:
+  - accuracy: 35.672
 deepseek-ai/DeepSeek-V3-Lite:
   - accuracy: 26.465
   - quant_algo: NVFP4
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index bb63e6e786de..fdc749e53e28 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -201,6 +201,16 @@ def test_nvfp4_tp4(self):
             task.evaluate(llm)
 
 
+class TestMistral7B(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "mistralai/Mistral-7B-v0.1"
+    MODEL_PATH = f"{llm_models_root()}/mistral-7b-v0.1"
+
+    def test_auto_dtype(self):
+        with LLM(self.MODEL_PATH) as llm:
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
 class TestMixtral8x7B(LlmapiAccuracyTestHarness):
     MODEL_NAME = "mistralai/Mixtral-8x7B-v0.1"
     MODEL_PATH = f"{llm_models_root()}/Mixtral-8x7B-v0.1"
@@ -418,3 +428,49 @@ def test_nvfp4_4gpus(self, tp_size, pp_size, ep_size, attention_dp,
             task.evaluate(llm)
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
+
+
+class TestMinitron4BBaseInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-Mini-4B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron/nemotron-mini-4b-instruct_vfp8-fp8-bf16-export"
+
+    @skip_pre_ada
+    def test_fp8_prequantized(self):
+        with LLM(self.MODEL_PATH) as llm:
+            assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestNemotronNas(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    def test_auto_dtype_tp8(self):
+        kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7)
+        pytorch_config = PyTorchConfig(enable_overlap_scheduler=True)
+
+        with LLM(self.MODEL_PATH,
+                 tensor_parallel_size=8,
+                 kv_cache_config=kv_cache_config,
+                 pytorch_backend_config=pytorch_config) as llm:
+
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestQwen2_7BInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/Qwen2-7B-Instruct"
+    EXTRA_EVALUATOR_KWARGS = dict(
+        apply_chat_template=True,
+        system_prompt=
+        "You are a helpful assistant, please summarize the article entered by the user with one or two sentences."
+    )
+
+    def test_auto_dtype(self):
+        with LLM(self.MODEL_PATH) as llm:
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm,
+                          extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS)
diff --git a/tests/integration/defs/examples/test_eagle.py b/tests/integration/defs/examples/test_eagle.py
index 413701d97457..f8bbd578048d 100644
--- a/tests/integration/defs/examples/test_eagle.py
+++ b/tests/integration/defs/examples/test_eagle.py
@@ -309,6 +309,8 @@ def test_mistral_eagle_1gpu(llm_mistral_model_root,
                           llm_rouge_root=llm_rouge_root)
 
 
+@pytest.mark.parametrize("use_dynamic_tree", [False, True],
+                         ids=['eagle1', 'eagle2'])
 @pytest.mark.parametrize("llm_qwen_model_root", [
     "qwen_7b_chat", "qwen1.5_7b_chat", "qwen2_7b_instruct",
     "qwen2_0.5b_instruct", "qwen2.5_1.5b_instruct"
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 551c61325717..7b42d4aed646 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -424,11 +424,15 @@ accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4
+accuracy/test_llm_api_pytorch.py::TestMistral_7B::test_auto_dtype_summarization
 accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2
 accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2
 accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[]
+accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized_summarization
+accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_summarization
+accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype_summarization
 
 test_e2e.py::test_benchmark_sanity[bert_base] # 127.18s
 test_e2e.py::test_benchmark_sanity[gpt_350m] # 64.06s


Reviews:
user: login: syuoni
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T07:48:21Z
user: login: crazydemo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T08:11:44Z
user: login: crazydemo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T08:12:24Z
user: login: crazydemo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T08:12:31Z
user: login: crazydemo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T08:12:40Z
user: login: syuoni
body: LGTM. Thanks!
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-10T08:25:24Z

Review comments:
diff_hunk: @@ -193,6 +193,16 @@ def test_nvfp4_tp4(self):
             task.evaluate(llm)
 
 
+class TestMistral_7B(LlmapiAccuracyTestHarness):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: syuoni
body: NIT: `TestMistral_7B` -> `TestMistral7B`. Let's make the test class naming more consistent
created_at: 2025-04-10T07:33:36Z
author_association: COLLABORATOR
diff_hunk: @@ -193,6 +193,16 @@ def test_nvfp4_tp4(self):
             task.evaluate(llm)
 
 
+class TestMistral_7B(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "mistralai/Mistral-7B-v0.1"
+    MODEL_PATH = f"{llm_models_root()}/mistral-7b-v0.1"
+
+    def test_auto_dtype_summarization(self):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: syuoni
body: I'm a bit concerned with naming the test case with task name.
* A test case may run multiple tasks. From the perspective of a test case, datasets/tasks are tools, while its objective is validating the accuracy of a model.
* In the future, we may add/remove tasks of a test case, which requires us to change the test naming. Developers could easily forget to do so, resulting in misleading test case names. This already happens to many cases under `examples/test_xxx.py`.
created_at: 2025-04-10T07:43:49Z
author_association: COLLABORATOR
diff_hunk: @@ -410,3 +420,50 @@ def test_nvfp4_4gpus(self, tp_size, pp_size, ep_size, attention_dp,
             task.evaluate(llm)
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
+
+
+class TestMinitron4BBaseInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-Mini-4B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron/nemotron-mini-4b-instruct_vfp8-fp8-bf16-export"
+
+    @skip_pre_ada
+    def test_fp8_prequantized_summarization(self):
+        with LLM(self.MODEL_PATH) as llm:
+            assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestNemotronNas(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    @pytest.mark.skip_less_device_memory(80000)
+    def test_auto_dtype_summarization(self):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: syuoni
body: Please reflect the parallelism info in the test name.
created_at: 2025-04-10T07:45:55Z
author_association: COLLABORATOR
diff_hunk: @@ -410,3 +420,50 @@ def test_nvfp4_4gpus(self, tp_size, pp_size, ep_size, attention_dp,
             task.evaluate(llm)
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
+
+
+class TestMinitron4BBaseInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-Mini-4B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron/nemotron-mini-4b-instruct_vfp8-fp8-bf16-export"
+
+    @skip_pre_ada
+    def test_fp8_prequantized_summarization(self):
+        with LLM(self.MODEL_PATH) as llm:
+            assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestNemotronNas(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    @pytest.mark.skip_less_device_memory(80000)
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: syuoni
body: I don't think a 51B model requires 8x80GB GPUs. If this test is intended to run tp_size=8, the memory requirement seems not necessary.
created_at: 2025-04-10T07:47:21Z
author_association: COLLABORATOR
diff_hunk: @@ -193,6 +193,16 @@ def test_nvfp4_tp4(self):
             task.evaluate(llm)
 
 
+class TestMistral_7B(LlmapiAccuracyTestHarness):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: crazydemo
body: fixed.
created_at: 2025-04-10T08:11:43Z
author_association: COLLABORATOR
diff_hunk: @@ -193,6 +193,16 @@ def test_nvfp4_tp4(self):
             task.evaluate(llm)
 
 
+class TestMistral_7B(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "mistralai/Mistral-7B-v0.1"
+    MODEL_PATH = f"{llm_models_root()}/mistral-7b-v0.1"
+
+    def test_auto_dtype_summarization(self):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: crazydemo
body: removed task, and appended parallelism strategy in test name
created_at: 2025-04-10T08:12:23Z
author_association: COLLABORATOR
diff_hunk: @@ -410,3 +420,50 @@ def test_nvfp4_4gpus(self, tp_size, pp_size, ep_size, attention_dp,
             task.evaluate(llm)
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
+
+
+class TestMinitron4BBaseInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-Mini-4B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron/nemotron-mini-4b-instruct_vfp8-fp8-bf16-export"
+
+    @skip_pre_ada
+    def test_fp8_prequantized_summarization(self):
+        with LLM(self.MODEL_PATH) as llm:
+            assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestNemotronNas(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    @pytest.mark.skip_less_device_memory(80000)
+    def test_auto_dtype_summarization(self):
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: crazydemo
body: fixed.
created_at: 2025-04-10T08:12:31Z
author_association: COLLABORATOR
diff_hunk: @@ -410,3 +420,50 @@ def test_nvfp4_4gpus(self, tp_size, pp_size, ep_size, attention_dp,
             task.evaluate(llm)
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
+
+
+class TestMinitron4BBaseInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-Mini-4B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron/nemotron-mini-4b-instruct_vfp8-fp8-bf16-export"
+
+    @skip_pre_ada
+    def test_fp8_prequantized_summarization(self):
+        with LLM(self.MODEL_PATH) as llm:
+            assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
+class TestNemotronNas(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    @pytest.mark.skip_less_device_memory(80000)
path: tests/integration/defs/accuracy/test_llm_api_pytorch.py
user: login: crazydemo
body: fixed.
created_at: 2025-04-10T08:12:40Z
author_association: COLLABORATOR

PR comments:
user: login: crazydemo
created_at: 2025-04-10T07:16:57Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T07:22:22Z
body: [PR_Github #1727](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1727/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T07:51:37Z
body: [PR_Github #1727](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1727/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #1289](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1289/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-04-10T08:15:57Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T08:21:22Z
body: [PR_Github #1741](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1741/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T12:35:32Z
body: [PR_Github #1741](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1741/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1293](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1293/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: crazydemo
created_at: 2025-04-11T02:26:12Z
body: /bot reuse-pipeline
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T02:31:32Z
body: [PR_Github #1845](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1845/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T02:36:46Z
body: [PR_Github #1845](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1845/) [ reuse-pipeline ] completed with state `SUCCESS`
Reusing [PR_Github #1741](https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/helpers/job/PR_Github/1741/) for commit `692f7e7`

author_association: COLLABORATOR
user: login: kaiyux
created_at: 2025-04-11T08:46:50Z
body: /bot reuse-pipeline
author_association: MEMBER
user: login: tensorrt-cicd
created_at: 2025-04-11T08:52:10Z
body: [PR_Github #1905](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1905/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T08:57:39Z
body: [PR_Github #1905](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1905/) [ reuse-pipeline ] completed with state `SUCCESS`
Reusing [PR_Github #1741](https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/helpers/job/PR_Github/1741/) for commit `554bfde`

author_association: COLLABORATOR