Repository: NVIDIA/TensorRT-LLM
Pull request #3869: fix: [nvbugs/5066257] serialization improvments
Author: login: coldwaterq
Created: 2025-04-25T16:47:14Z | Merged: 2025-05-23T05:06:29Z

## Description

Add an approve list into the pickle deserialization process to reduce the attack surface of using pickle to a subset of supported objects.

## Test Coverage

if a new object is added this PR will cause a ValueError to be raised that clearly states what module is not supported that is being serialized/deserialized.
Fully utilizing the zmq interfaces would test this code.




Diff:
diff --git a/examples/llm-api/llm_logits_processor.py b/examples/llm-api/llm_logits_processor.py
index 516cd507107b..5fde21a7c9ee 100644
--- a/examples/llm-api/llm_logits_processor.py
+++ b/examples/llm-api/llm_logits_processor.py
@@ -15,6 +15,7 @@
 # This simple callback will output a specific token at each step irrespective of prompt.
 # Refer to ../bindings/executor/example_logits_processor.py for a more
 # sophisticated callback that generates JSON structured output.
+# Please also refer to sampling_params.py for adding subclass to the approved class list for deserialization
 class MyLogitsProcessor(LogitsProcessor):
 
     def __init__(self, allowed_token_id: int):
diff --git a/tensorrt_llm/executor/ipc.py b/tensorrt_llm/executor/ipc.py
index f178596f4972..60192c546a2a 100644
--- a/tensorrt_llm/executor/ipc.py
+++ b/tensorrt_llm/executor/ipc.py
@@ -1,7 +1,6 @@
 import hashlib
 import hmac
 import os
-import pickle  # nosec B403
 import time
 import traceback
 from queue import Queue
@@ -10,6 +9,7 @@
 import zmq
 import zmq.asyncio
 
+import tensorrt_llm.executor.serialization as serialization
 from tensorrt_llm.logger import logger
 
 from .._utils import nvtx_mark, nvtx_range_debug
@@ -41,6 +41,7 @@ def __init__(self,
             is_server (bool): Whether the current process is the server or the client.
             is_async (bool): Whether to use asyncio for the socket. Defaults to False.
             name (str, optional): The name of the queue. Defaults to None.
+            additional_serializable_classes (Dict, optional): Additional classes to be added to the serializable classes.
             use_hmac_encryption (bool): Whether to use HMAC encryption for pickled data. Defaults to True.
         '''
 
@@ -116,26 +117,26 @@ def poll(self, timeout: int) -> bool:
     def put(self, obj: Any):
         self.setup_lazily()
         with nvtx_range_debug("send", color="blue", category="IPC"):
+            data = serialization.dumps(obj)
             if self.use_hmac_encryption:
                 # Send pickled data with HMAC appended
-                data = pickle.dumps(obj)  # nosec B301
                 signed_data = self._sign_data(data)
                 self.socket.send(signed_data)
             else:
                 # Send data without HMAC
-                self.socket.send_pyobj(obj)
+                self.socket.send(data)
 
     async def put_async(self, obj: Any):
         self.setup_lazily()
         try:
+            data = serialization.dumps(obj)
             if self.use_hmac_encryption:
                 # Send pickled data with HMAC appended
-                data = pickle.dumps(obj)  # nosec B301
                 signed_data = self._sign_data(data)
                 await self.socket.send(signed_data)
             else:
                 # Send data without HMAC
-                await self.socket.send_pyobj(obj)
+                await self.socket.send(data)
         except TypeError as e:
             logger.error(f"Cannot pickle {obj}")
             raise e
@@ -161,10 +162,13 @@ def get(self) -> Any:
             if not self._verify_hmac(data, actual_hmac):
                 raise RuntimeError("HMAC verification failed")
 
-            obj = pickle.loads(data)  # nosec B301
+            obj = serialization.loads(
+                data, approved_imports=serialization.BASE_ZMQ_CLASSES)
         else:
             # Receive data without HMAC
-            obj = self.socket.recv_pyobj()
+            data = self.socket.recv()
+            obj = serialization.loads(
+                data, approved_imports=serialization.BASE_ZMQ_CLASSES)
         return obj
 
     async def get_async(self) -> Any:
@@ -182,10 +186,13 @@ async def get_async(self) -> Any:
             if not self._verify_hmac(data, actual_hmac):
                 raise RuntimeError("HMAC verification failed")
 
-            obj = pickle.loads(data)  # nosec B301
+            obj = serialization.loads(
+                data, approved_imports=serialization.BASE_ZMQ_CLASSES)
         else:
             # Receive data without HMAC
-            obj = await self.socket.recv_pyobj()
+            data = await self.socket.recv()
+            obj = serialization.loads(
+                data, approved_imports=serialization.BASE_ZMQ_CLASSES)
         return obj
 
     def close(self):
diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py
index 91460bb35c59..d048136b75e3 100644
--- a/tensorrt_llm/executor/postproc_worker.py
+++ b/tensorrt_llm/executor/postproc_worker.py
@@ -8,6 +8,8 @@
 import zmq
 import zmq.asyncio
 
+import tensorrt_llm.executor.serialization as serialization
+
 from .._utils import nvtx_range_debug
 from ..bindings import executor as tllm
 from ..llmapi.tokenizer import TransformersTokenizer, load_hf_tokenizer
@@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
     ):
         '''
         Args:
@@ -85,8 +88,11 @@ def __init__(
             tokenizer_dir (str): The directory to load tokenizer.
             record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request.
             result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result.
+            BASE_ZMQ_CLASSES (Dict): The base classes for ZMQ serialization.
         '''
-
+        # Passed through from the parent process to ensure
+        # that children processes include any classes added at runtime (such as those from `register_approved_ipc_class`).
+        serialization.BASE_ZMQ_CLASSES = BASE_ZMQ_CLASSES
         self._records: Dict[int, GenerationResult] = {}
         self._record_creator = record_creator
         self._pull_pipe = ZeroMqQueue(address=pull_pipe_addr,
@@ -213,9 +219,11 @@ async def main():
 @print_traceback_on_error
 def postproc_worker_main(feedin_ipc_addr: tuple[str, Optional[bytes]],
                          feedout_ipc_addr: tuple[str, Optional[bytes]],
-                         tokenizer_dir: str, record_creator: Callable):
+                         tokenizer_dir: str, record_creator: Callable,
+                         BASE_ZMQ_CLASSES: Dict):
     worker = PostprocWorker(feedin_ipc_addr,
                             feedout_ipc_addr,
                             tokenizer_dir=tokenizer_dir,
-                            record_creator=record_creator)
+                            record_creator=record_creator,
+                            BASE_ZMQ_CLASSES=BASE_ZMQ_CLASSES)
     worker.start()
diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py
index 64607badaa7e..af7cf0c56811 100644
--- a/tensorrt_llm/executor/proxy.py
+++ b/tensorrt_llm/executor/proxy.py
@@ -9,6 +9,7 @@
 import zmq
 import zmq.asyncio
 
+import tensorrt_llm.executor.serialization as serialization
 from tensorrt_llm.logger import logger
 
 from .._utils import mpi_rank
@@ -297,7 +298,7 @@ def mpi_done_callback(future: concurrent.futures.Future):
             tracer_init_kwargs=tracer_init_kwargs,
             _torch_model_class_mapping=MODEL_CLASS_MAPPING,
             ready_signal=ExecutorBindingsProxy.READY_SIGNAL,
-        )
+            BASE_ZMQ_CLASSES=serialization.BASE_ZMQ_CLASSES)
         for fut in self.mpi_futures:
             fut.add_done_callback(mpi_done_callback)
 
diff --git a/tensorrt_llm/executor/serialization.py b/tensorrt_llm/executor/serialization.py
new file mode 100644
index 000000000000..607ec474081d
--- /dev/null
+++ b/tensorrt_llm/executor/serialization.py
@@ -0,0 +1,173 @@
+import io
+# pickle is not secure, but but this whole file is a wrapper to make it
+# possible to mitigate the primary risk of code injection via pickle.
+import pickle  # nosec B403
+
+# These are the base classes that are generally serialized by the ZeroMQ IPC.
+# If a class is needed by ZMQ routinely it should be added here. If
+# it is only needed in a single instance the class can be added at runtime
+# using register_approved_ipc_class.
+BASE_ZMQ_CLASSES = {
+    "builtins": ["Exception", "ValueError"
+                 ],  # each Exception Error class needs to be added explicitly
+    "collections": ["OrderedDict"],
+    "datetime": ["timedelta"],
+    "pathlib": ["PosixPath"],
+    "llmapi.run_llm_with_postproc": ["perform_faked_oai_postprocess"
+                                     ],  # only used in tests
+    ### starting import of torch models classes. They are used in test_llm_multi_gpu.py.
+    "tensorrt_llm._torch.models.modeling_bert":
+    ["BertForSequenceClassification"],
+    "tensorrt_llm._torch.models.modeling_clip": ["CLIPVisionModel"],
+    "tensorrt_llm._torch.models.modeling_deepseekv3": ["DeepseekV3ForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_gemma3": ["Gemma3ForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_llama": [
+        "Eagle3LlamaForCausalLM",
+        "LlamaForCausalLM",
+        "Llama4ForCausalLM",
+        "Llama4ForConditionalGeneration",
+    ],
+    "tensorrt_llm._torch.models.modeling_llava_next": ["LlavaNextModel"],
+    "tensorrt_llm._torch.models.modeling_mistral": ["MistralForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_mixtral": ["MixtralForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_mllama":
+    ["MllamaForConditionalGeneration"],
+    "tensorrt_llm._torch.models.modeling_nemotron": ["NemotronForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_nemotron_h": ["NemotronHForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_nemotron_nas":
+    ["NemotronNASForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_qwen":
+    ["Qwen2ForCausalLM", "Qwen2ForProcessRewardModel", "Qwen2ForRewardModel"],
+    "tensorrt_llm._torch.models.modeling_qwen2vl":
+    ["Qwen2VLModel", "Qwen2_5_VLModel"],
+    "tensorrt_llm._torch.models.modeling_qwen3": ["Qwen3ForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_qwen3_moe": ["Qwen3MoeForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_qwen_moe": ["Qwen2MoeForCausalLM"],
+    "tensorrt_llm._torch.models.modeling_siglip": ["SiglipVisionModel"],
+    "tensorrt_llm._torch.models.modeling_vila": ["VilaModel"],
+    ### ending import of torch models classes
+    "tensorrt_llm._torch.pyexecutor.llm_request":
+    ["LogitsStorage", "PyResult", "LlmResult", "LlmResponse", "LogProbStorage"],
+    "tensorrt_llm.auto_parallel.config": ["AutoParallelConfig", "CostModel"],
+    "tensorrt_llm.bindings.executor": [
+        "BatchingType", "CapacitySchedulerPolicy", "ContextPhaseParams",
+        "ExecutorConfig", "ExtendedRuntimePerfKnobConfig", "Response", "Result",
+        "FinishReason", "KvCacheConfig", "KvCacheRetentionConfig",
+        "KvCacheRetentionConfig.TokenRangeRetentionConfig", "PeftCacheConfig",
+        "SchedulerConfig"
+    ],
+    "tensorrt_llm.builder": ["BuildConfig"],
+    "tensorrt_llm.disaggregated_params": ["DisaggregatedParams"],
+    "tensorrt_llm.executor.postproc_worker": [
+        "PostprocArgs", "PostprocParams", "PostprocWorkerConfig",
+        "PostprocWorker.Input", "PostprocWorker.Output"
+    ],
+    "tensorrt_llm.executor.request": [
+        "CancellingRequest", "GenerationRequest", "LoRARequest",
+        "PromptAdapterRequest"
+    ],
+    "tensorrt_llm.executor.result": [
+        "CompletionOutput", "DetokenizedGenerationResultBase",
+        "GenerationResult", "GenerationResultBase", "IterationResult",
+        "Logprob", "LogProbsResult", "ResponseWrapper"
+    ],
+    "tensorrt_llm.executor.utils": ["ErrorResponse", "WorkerCommIpcAddrs"],
+    "tensorrt_llm.executor.worker": ["ExecutorBindingsWorker", "worker_main"],
+    "tensorrt_llm.llmapi.llm_args": [
+        "_ModelFormatKind", "_ParallelConfig", "CalibConfig",
+        "CapacitySchedulerPolicy", "KvCacheConfig", "LlmArgs",
+        "LookaheadDecodingConfig", "SchedulerConfig"
+    ],
+    "tensorrt_llm.llmapi.mpi_session": ["RemoteTask"],
+    "tensorrt_llm.llmapi.llm_utils":
+    ["CachedModelLoader._node_build_task", "LlmBuildStats"],
+    "tensorrt_llm.lora_manager": ["LoraConfig"],
+    "tensorrt_llm.mapping": ["Mapping"],
+    "tensorrt_llm.models.modeling_utils":
+    ["QuantConfig", "SpeculativeDecodingMode"],
+    "tensorrt_llm.plugin.plugin": ["PluginConfig"],
+    "tensorrt_llm.sampling_params":
+    ["SamplingParams", "GuidedDecodingParams", "GreedyDecodingParams"],
+    "tensorrt_llm.serve.postprocess_handlers": [
+        "chat_response_post_processor", "chat_stream_post_processor",
+        "completion_stream_post_processor",
+        "completion_response_post_processor", "CompletionPostprocArgs",
+        "ChatPostprocArgs"
+    ],
+    "torch._utils": ["_rebuild_tensor_v2"],
+    "torch.storage": ["_load_from_bytes"],
+}
+
+
+def _register_class(dict, obj):
+    name = getattr(obj, '__qualname__', None)
+    if name is None:
+        name = obj.__name__
+    module = pickle.whichmodule(obj, name)
+    if module not in BASE_ZMQ_CLASSES.keys():
+        BASE_ZMQ_CLASSES[module] = []
+    BASE_ZMQ_CLASSES[module].append(name)
+
+
+def register_approved_ipc_class(obj):
+    _register_class(BASE_ZMQ_CLASSES, obj)
+
+
+class Unpickler(pickle.Unpickler):
+
+    def __init__(self, *args, approved_imports={}, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.approved_imports = approved_imports
+
+    # only import approved classes, this is the security boundary.
+    def find_class(self, module, name):
+        if name not in self.approved_imports.get(module, []):
+            # If this is triggered when it shouldn't be, then the module
+            # and class should be added to the approved_imports. If the class
+            # is being used as part of a routine scenario, then it should be added
+            # to the appropriate base classes above.
+            raise ValueError(f"Import {module} | {name} is not allowed")
+        return super().find_class(module, name)
+
+
+# these are taken from the pickle module to allow for this to be a drop in replacement
+# source: https://github.com/python/cpython/blob/3.13/Lib/pickle.py
+# dump and dumps are just aliases because the serucity controls are on the deserialization
+# side. However they are included here so that in the future if a more secure serialization
+# soliton is identified, it can be added with less impact to the rest of the application.
+dump = pickle.dump  # nosec B301
+dumps = pickle.dumps  # nosec B301
+
+
+def load(file,
+         *,
+         fix_imports=True,
+         encoding="ASCII",
+         errors="strict",
+         buffers=None,
+         approved_imports={}):
+    return Unpickler(file,
+                     fix_imports=fix_imports,
+                     buffers=buffers,
+                     encoding=encoding,
+                     errors=errors,
+                     approved_imports=approved_imports).load()
+
+
+def loads(s,
+          /,
+          *,
+          fix_imports=True,
+          encoding="ASCII",
+          errors="strict",
+          buffers=None,
+          approved_imports={}):
+    if isinstance(s, str):
+        raise TypeError("Can't load pickle from unicode string")
+    file = io.BytesIO(s)
+    return Unpickler(file,
+                     fix_imports=fix_imports,
+                     buffers=buffers,
+                     encoding=encoding,
+                     errors=errors,
+                     approved_imports=approved_imports).load()
diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py
index 5752e9c33316..f4f1064db03c 100644
--- a/tensorrt_llm/executor/worker.py
+++ b/tensorrt_llm/executor/worker.py
@@ -12,6 +12,7 @@
 
 import torch
 
+import tensorrt_llm.executor.serialization as serialization
 from tensorrt_llm.logger import logger
 
 from .._utils import (KVCacheEventSerializer, global_mpi_rank, mpi_comm,
@@ -576,7 +577,11 @@ def worker_main(
     is_llm_executor: Optional[
         bool] = True,  # whether it's the main executor instance
     lora_config: Optional[LoraConfig] = None,
+    BASE_ZMQ_CLASSES: Dict = serialization.BASE_ZMQ_CLASSES,
 ) -> None:
+    # The base classes for ZMQ serialization. Passed through from the parent process to ensure
+    # that children processes include any classes added at runtime (such as those from `register_approved_ipc_class`).
+    serialization.BASE_ZMQ_CLASSES = BASE_ZMQ_CLASSES
     mpi_comm().barrier()
     print_colored_debug(f"Worker {mpi_rank()} entering worker_main...\n",
                         "green")
@@ -671,12 +676,11 @@ def notify_proxy_threads_to_quit():
         assert isinstance(proxy_result_queue, tuple)
         for i in range(postproc_worker_config.num_postprocess_workers):
             fut = postproc_worker_pool.submit(
-                postproc_worker_main,
-                result_queues[i].address,
+                postproc_worker_main, result_queues[i].address,
                 proxy_result_queue,
                 postproc_worker_config.postprocess_tokenizer_dir,
                 PostprocWorker.default_record_creator,
-            )
+                serialization.BASE_ZMQ_CLASSES)
             postprocess_worker_futures.append(fut)
 
     # Error handling in the Worker/MPI process
diff --git a/tensorrt_llm/sampling_params.py b/tensorrt_llm/sampling_params.py
index 7e0c756bed43..0678a1b7e495 100644
--- a/tensorrt_llm/sampling_params.py
+++ b/tensorrt_llm/sampling_params.py
@@ -8,6 +8,7 @@
 from pydantic import BaseModel
 
 from tensorrt_llm.bindings import executor as tllme
+from tensorrt_llm.executor.serialization import register_approved_ipc_class
 
 
 @dataclass(slots=True, kw_only=True)
@@ -71,6 +72,14 @@ def __call__(self, req_id: int, logits: torch.Tensor,
         """
         pass  # noqa
 
+    def __init_subclass__(cls, **kwargs):
+        """
+        This method is called when a class inherits from LogitsProcessor.
+        """
+        # Register subclass as an approved class for deserialization across IPC boundaries.
+        super().__init_subclass__(**kwargs)
+        register_approved_ipc_class(cls)
+
 
 class BatchedLogitsProcessor(ABC):
     """Base class for batched logits processor.
diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py
index bfd0b01371e9..d3c0780f8896 100644
--- a/tensorrt_llm/serve/openai_protocol.py
+++ b/tensorrt_llm/serve/openai_protocol.py
@@ -12,6 +12,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 from typing_extensions import Annotated, Required, TypedDict
 
+from tensorrt_llm.executor.serialization import register_approved_ipc_class
 from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams
 from tensorrt_llm.llmapi import GuidedDecodingParams, SamplingParams
 
@@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
+        """
+        This method is called when a class inherits from OpenAIBaseModel.
+        """
+        # Register subclass as an approved class for deserialization across IPC boundaries.
+        super().__init_subclass__(**kwargs)
+        register_approved_ipc_class(cls)
+
 
 class StreamOptions(OpenAIBaseModel):
     include_usage: Optional[bool] = True
diff --git a/tests/unittest/llmapi/test_executor.py b/tests/unittest/llmapi/test_executor.py
index 8c27225a74ea..b7be69f2a05a 100644
--- a/tests/unittest/llmapi/test_executor.py
+++ b/tests/unittest/llmapi/test_executor.py
@@ -1,4 +1,5 @@
 import asyncio
+import datetime
 import json
 import tempfile
 import threading
@@ -9,6 +10,7 @@
 import torch
 import zmq
 
+import tensorrt_llm.executor.serialization as serialization
 from tensorrt_llm._utils import mpi_world_size
 from tensorrt_llm.bindings import executor as tllm
 from tensorrt_llm.executor import (DetokenizedGenerationResultBase,
@@ -356,6 +358,60 @@ def test_ZeroMqQueue_sync_async():
     push_pipe.close()
 
 
+def _ZeroMqQueue_serialization_complicated_dataclass(addr: str,
+                                                     iterations: int):
+    pull_pipe = ZeroMqQueue(address=addr, is_server=False, is_async=True)
+
+    total = 0
+
+    async def task():
+        print(f"running task")
+        for i in range(iterations):
+            print(f"waiting for msg")
+            msg = await pull_pipe.get_async()
+            # print(f"received: {msg}")
+            nonlocal total
+            try:
+                total += msg.prompt_token_ids[0]
+            except Exception as e:
+                print(f"error: {e}")
+
+    print(f"to run task")
+    asyncio.run(task())
+
+    return total
+
+
+def test_ZeroMqQueue_serialization_complicated_dataclass():
+    # sync send message, async recv message
+    push_pipe = ZeroMqQueue(is_async=False, is_server=True)
+    iterations = 2
+
+    pool = ProcessPoolExecutor(max_workers=1)
+    res = pool.submit(_ZeroMqQueue_serialization_complicated_dataclass,
+                      push_pipe.address, iterations)
+
+    TokenRangeRetentionConfig = tllm.KvCacheRetentionConfig.TokenRangeRetentionConfig
+    kvcache_config = tllm.KvCacheRetentionConfig(
+        [TokenRangeRetentionConfig(0, 2, 30, datetime.timedelta(seconds=30))],
+        80)
+
+    sampling_params = SamplingParams(max_tokens=4,
+                                     embedding_bias=torch.randn(2, 2))
+
+    for i in range(iterations):
+        request = GenerationRequest(prompt_token_ids=[i],
+                                    sampling_params=sampling_params,
+                                    kv_cache_retention_config=kvcache_config)
+        # print(f"put with msg: {request}")
+        push_pipe.put(request)
+
+    print(res.result())
+    assert res.result() == iterations * (iterations - 1) / 2
+    pool.shutdown()
+    push_pipe.close()
+
+
 Input = PostprocWorker.Input
 Output = PostprocWorker.Output
 
@@ -374,7 +430,8 @@ def ResponsePostprocessWorker_worker_task(pull_pipe_addr, push_pipe_addr,
         pull_pipe_addr=pull_pipe_addr,
         push_pipe_addr=push_pipe_addr,
         tokenizer_dir=tokenizer_dir,
-        record_creator=ResponsePostprocessWorker_record_creator)
+        record_creator=ResponsePostprocessWorker_record_creator,
+        BASE_ZMQ_CLASSES=serialization.BASE_ZMQ_CLASSES)
     worker.start()
 
 
diff --git a/tests/unittest/llmapi/test_serialization.py b/tests/unittest/llmapi/test_serialization.py
new file mode 100644
index 000000000000..1859e4ab632e
--- /dev/null
+++ b/tests/unittest/llmapi/test_serialization.py
@@ -0,0 +1,78 @@
+import torch
+
+import tensorrt_llm.executor.serialization as serialization
+
+
+class TestClass:
+
+    def __init__(self, name: str):
+        self.name = name
+
+
+def test_serialization_allowed_class():
+    obj = TestClass("test")
+    serialization.register_approved_ipc_class(TestClass)
+    module = TestClass.__module__
+    assert module in serialization.BASE_ZMQ_CLASSES
+    assert "TestClass" in serialization.BASE_ZMQ_CLASSES[module]
+    a = serialization.dumps(obj)
+    b = serialization.loads(a, approved_imports=serialization.BASE_ZMQ_CLASSES)
+    assert type(obj) == type(b) and obj.name == b.name
+
+
+def test_serialization_disallowed_class():
+    obj = TestClass("test")
+    a = serialization.dumps(obj)
+    excep = None
+    try:
+        serialization.loads(a, approved_imports={})
+    except Exception as e:
+        excep = e
+        print(excep)
+    assert isinstance(excep, ValueError) and str(
+        excep) == "Import llmapi.test_serialization | TestClass is not allowed"
+
+
+def test_serialization_basic_object():
+    obj = {"test": "test"}
+    a = serialization.dumps(obj)
+    b = serialization.loads(a, approved_imports=serialization.BASE_ZMQ_CLASSES)
+    assert obj == b
+
+
+def test_serialization_complex_object_allowed_class():
+    obj = torch.tensor([1, 2, 3])
+    a = serialization.dumps(obj)
+    b = serialization.loads(a, approved_imports=serialization.BASE_ZMQ_CLASSES)
+    assert torch.all(obj == b)
+
+
+def test_serialization_complex_object_partially_allowed_class():
+    obj = torch.tensor([1, 2, 3])
+    a = serialization.dumps(obj)
+    excep = None
+    try:
+        b = serialization.loads(a,
+                                approved_imports={
+                                    'torch._utils': ['_rebuild_tensor_v2'],
+                                })
+    except Exception as e:
+        excep = e
+    assert isinstance(excep, ValueError) and str(
+        excep) == "Import torch.storage | _load_from_bytes is not allowed"
+
+
+def test_serialization_complex_object_disallowed_class():
+    obj = torch.tensor([1, 2, 3])
+    a = serialization.dumps(obj)
+    excep = None
+    try:
+        serialization.loads(a)
+    except Exception as e:
+        excep = e
+    assert isinstance(excep, ValueError) and str(
+        excep) == "Import torch._utils | _rebuild_tensor_v2 is not allowed"
+
+
+if __name__ == "__main__":
+    test_serialization_allowed_class()


Reviews:
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-25T21:00:04Z
user: login: Superjomn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-25T23:40:58Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T18:45:24Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T18:48:04Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T18:49:01Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T20:08:08Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-28T22:47:44Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-01T18:32:38Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-01T19:08:55Z
user: login: kaiyux
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-06T12:29:12Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-06T16:04:05Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T17:22:56Z
user: login: Superjomn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-07T06:26:26Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T20:20:32Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-07T20:59:48Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T18:08:44Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-08T18:38:50Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-09T16:59:21Z
user: login: Superjomn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T06:46:43Z
user: login: LinPoly
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T07:27:51Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T16:19:09Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T16:55:17Z
user: login: Superjomn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-14T05:21:27Z
user: login: LinPoly
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-14T08:48:37Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-14T17:34:43Z
user: login: yibinl-nvidia
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-14T17:38:13Z
user: login: coldwaterq
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T20:00:20Z
user: login: Superjomn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-16T07:36:01Z
user: login: Superjomn
body: LGTM as the first milestone. 

IIUC, there are the following TODOs:
1. Cleanup the existing HMAC logic @yibinl-nvidia 
2. Simplify the passing for BASE_ZMQ_CLASSES in intra-process scenarios @Superjomn 
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-16T07:40:34Z
user: login: yibinl-nvidia
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-22T19:11:42Z
user: login: yibinl-nvidia
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-23T04:27:49Z

Review comments:
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
+        "builtins": ["Exception"],
+        "tensorrt_llm.executor.request":
+        ["CancellingRequest", "GenerationRequest"],
+        "tensorrt_llm.sampling_params": ["SamplingParams"],
+        "tensorrt_llm.executor.postproc_worker":
+        ["PostprocParams", "PostprocWorker.Input", "PostprocWorker.Output"],
+        "tensorrt_llm.serve.postprocess_handlers":
+        ["CompletionPostprocArgs", "completion_response_post_processor"],
+        "tensorrt_llm.bindings.executor": [
+            "Response", "Result", "FinishReason", "KvCacheRetentionConfig",
+            "KvCacheRetentionConfig.TokenRangeRetentionConfig"
+        ],
+        "tensorrt_llm.serve.openai_protocol":
+        ["CompletionResponse", "CompletionResponseChoice", "UsageInfo"],
+        "torch._utils": ["_rebuild_tensor_v2"],
+        "torch.storage": ["_load_from_bytes"],
+        "collections": ["OrderedDict"],
+        "datetime": ["timedelta"],
+    }
+
     def __init__(self,
                  address: Optional[tuple[str, Optional[bytes]]] = None,
                  *,
                  socket_type: int = zmq.PAIR,
                  is_server: bool,
                  is_async: bool = False,
                  name: Optional[str] = None,
+                 additional_serializable_types: Optional[Dict] = None,
                  use_hmac_encryption: bool = True):
path: tensorrt_llm/executor/ipc.py
user: login: yibinl-nvidia
body: Do you think we can remove hmac encryption if we use the approved list?
created_at: 2025-04-25T20:51:46Z
author_association: COLLABORATOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
path: tensorrt_llm/executor/ipc.py
user: login: yibinl-nvidia
body: Could we move BASE_SERIALIZABLE_TYPES outside ZeroMqQueue? Each ZeroMqQueue instance can have a different set of approved class. Also please add comments on why we need this in IPC.  
created_at: 2025-04-25T20:57:17Z
author_association: COLLABORATOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
path: tensorrt_llm/executor/ipc.py
user: login: Superjomn
body: Can you add more descriptions here to guide some developers in updating this? They may introduce a new type, break the tests, and when they finally switch here and the comments should help them to briefly know the background.
created_at: 2025-04-25T23:30:27Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,101 @@
+import io
+import pickle
path: tensorrt_llm/executor/serialization.py
user: login: Superjomn
body: How about adding some unittests for this file, as it is the core of the feature, you can add a `executor/test_serialization.py` in the [unittest](https://github.com/Superjomn/TensorRT-LLM/tree/57944206ba403acf6ac2b3834b33ad15464ee6d5/tests/unittest) directory.
created_at: 2025-04-25T23:40:53Z
author_association: COLLABORATOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
+        "builtins": ["Exception"],
+        "tensorrt_llm.executor.request":
+        ["CancellingRequest", "GenerationRequest"],
+        "tensorrt_llm.sampling_params": ["SamplingParams"],
+        "tensorrt_llm.executor.postproc_worker":
+        ["PostprocParams", "PostprocWorker.Input", "PostprocWorker.Output"],
+        "tensorrt_llm.serve.postprocess_handlers":
+        ["CompletionPostprocArgs", "completion_response_post_processor"],
+        "tensorrt_llm.bindings.executor": [
+            "Response", "Result", "FinishReason", "KvCacheRetentionConfig",
+            "KvCacheRetentionConfig.TokenRangeRetentionConfig"
+        ],
+        "tensorrt_llm.serve.openai_protocol":
+        ["CompletionResponse", "CompletionResponseChoice", "UsageInfo"],
+        "torch._utils": ["_rebuild_tensor_v2"],
+        "torch.storage": ["_load_from_bytes"],
+        "collections": ["OrderedDict"],
+        "datetime": ["timedelta"],
+    }
+
     def __init__(self,
                  address: Optional[tuple[str, Optional[bytes]]] = None,
                  *,
                  socket_type: int = zmq.PAIR,
                  is_server: bool,
                  is_async: bool = False,
                  name: Optional[str] = None,
+                 additional_serializable_types: Optional[Dict] = None,
                  use_hmac_encryption: bool = True):
path: tensorrt_llm/executor/ipc.py
user: login: coldwaterq
body: you could, however it's already implemented and provides an additional layer of security if anyone wants to use it. I'll change the flag to default to False in my commit though.
created_at: 2025-04-28T18:45:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
path: tensorrt_llm/executor/ipc.py
user: login: coldwaterq
body: In the commit I'm testing now I moved this into the serialization file as well. Since any error that would result in people updating the file would also originate from there.
created_at: 2025-04-28T18:48:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
path: tensorrt_llm/executor/ipc.py
user: login: coldwaterq
body: the commit I'm working on adresses this by adding more context and moving the definition into the serialization file so the error will originate where the update has to be made as well.
created_at: 2025-04-28T18:49:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,101 @@
+import io
+import pickle
path: tensorrt_llm/executor/serialization.py
user: login: coldwaterq
body: added in the latest commit
created_at: 2025-04-28T20:08:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,18 +26,42 @@ class ZeroMqQueue:
         zmq.PUSH: "PUSH",
     }
 
+    # Base serializable types and their handlers
+    BASE_SERIALIZABLE_TYPES = {
+        "builtins": ["Exception"],
+        "tensorrt_llm.executor.request":
+        ["CancellingRequest", "GenerationRequest"],
+        "tensorrt_llm.sampling_params": ["SamplingParams"],
+        "tensorrt_llm.executor.postproc_worker":
+        ["PostprocParams", "PostprocWorker.Input", "PostprocWorker.Output"],
+        "tensorrt_llm.serve.postprocess_handlers":
+        ["CompletionPostprocArgs", "completion_response_post_processor"],
+        "tensorrt_llm.bindings.executor": [
+            "Response", "Result", "FinishReason", "KvCacheRetentionConfig",
+            "KvCacheRetentionConfig.TokenRangeRetentionConfig"
+        ],
+        "tensorrt_llm.serve.openai_protocol":
+        ["CompletionResponse", "CompletionResponseChoice", "UsageInfo"],
+        "torch._utils": ["_rebuild_tensor_v2"],
+        "torch.storage": ["_load_from_bytes"],
+        "collections": ["OrderedDict"],
+        "datetime": ["timedelta"],
+    }
+
     def __init__(self,
                  address: Optional[tuple[str, Optional[bytes]]] = None,
                  *,
                  socket_type: int = zmq.PAIR,
                  is_server: bool,
                  is_async: bool = False,
                  name: Optional[str] = None,
+                 additional_serializable_types: Optional[Dict] = None,
                  use_hmac_encryption: bool = True):
path: tensorrt_llm/executor/ipc.py
user: login: yibinl-nvidia
body: We didn't communicate to the user explicitly about this flag so most of them are unaware of this option. Let's keep this option on and I will consider to default to False if necessary in the future. 
created_at: 2025-04-28T22:47:44Z
author_association: COLLABORATOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: coldwaterq
body: @yibinl-nvidia there are still a couple of changes like this in order to pass the updated approved list across process boundaries, but I think that's unavoidable and it should be invisible to the end user. Do foresee any issue with this? 
created_at: 2025-05-01T18:32:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: yibinl-nvidia
body: This should be fine as long as it is not directly exposed to the user. 
created_at: 2025-05-01T19:08:36Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: kaiyux
body: Should we somehow hide this line inside of the LLM API?

@Superjomn Thoughts on this line?
created_at: 2025-05-06T12:28:28Z
author_association: MEMBER
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: yibinl-nvidia
body: This is the design changes I discussed with @coldwaterq. The original idea was to append LLM API with an additional optional arg for the supplementary approved class, but this approach means API breaking change and it seems that the example above is the only place needed such change, so we ought to add `serialization.register_approved_ipc_class` instead. And I think it is necessary to highlight the changes in this example because user can define their own `LogitsProcessor` class and they would confuse why serialization fail without proper example.
created_at: 2025-05-06T16:04:04Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: coldwaterq
body: Yep, this solution also allows the creator of the object to simply pass it into this function which resolves how pickle will identify it even if the module of the object changes between runs. Such as if the script is run directly sometimes and used as a module in others.
created_at: 2025-05-06T17:22:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: Superjomn
body: Is it possible to make all the inheritors of `BatchedLogitsProcessor` be registered automatically? To use a logits-processor, users should always inherit that class. 

Got the following example from Cursor:
```python
# Assume this function is defined elsewhere
def register_approved_ipc_class(dtype):
  """
  Registers the given class type.
  (This is a placeholder for your actual registration logic)
  """
  print(f"Registering IPC class: {dtype.__name__}")
  # In a real scenario, you would add dtype to a list, dictionary,
  # or perform some other registration action.


class BaseClass:
  def __init_subclass__(cls, **kwargs):
    """
    This method is called when a class inherits from BaseClass.
    """
    super().__init_subclass__(**kwargs) # Call the superclass's __init_subclass__
    register_approved_ipc_class(cls)   # Register the new subclass


# Now, any class that inherits from BaseClass will automatically be registered.

class ChildClassA(BaseClass):
  pass

class ChildClassB(BaseClass):
  def __init__(self, name):
    self.name = name

class GrandChildClass(ChildClassA):
  pass

# Example of creating instances (not strictly necessary to show registration)
# child_a_instance = ChildClassA()
# child_b_instance = ChildClassB("Test")
# grand_child_instance = GrandChildClass()
```

Ideally, users should have little chance to call `register_approved_ipc_class` explicitly.
created_at: 2025-05-07T06:23:58Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: coldwaterq
body: That should work, however, I would propose that ideally, users should call register_approved_ipc_class anytime they create a custom sublcass that needs it. Because this is effectively our last warning that the class will be a part of the security boundary and them confirming they believe that is not a security risk.

If we want to hide the register_approved_ipc_class call from the developers creating the classes, then we should consider adding in warnings in some other way.
created_at: 2025-05-07T20:20:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,6 +68,8 @@ def __call__(self, req_ids: List[int], logits: List[torch.Tensor],
 
 
 def main():
+    # Register MyLogitsProcessor as an approved class for deserialization across IPC boundaries.
+    serialization.register_approved_ipc_class(MyLogitsProcessor)
path: examples/llm-api/llm_logits_processor.py
user: login: yibinl-nvidia
body: > Because this is effectively our last warning that the class will be a part of the security boundary and them confirming they believe that is not a security risk.

The argument I would have is that we should try our best to make our own codebase to be safe, but there is no way we can enforce any safety rules to the user who creates their own classes. As such, even we put a warning about this security boundary, the user needs to understand the context about how we do serialization in IPC. I doubt it is necessary to disclose this.


created_at: 2025-05-07T20:59:48Z
author_association: COLLABORATOR
diff_hunk: @@ -69,6 +70,13 @@ def __call__(self, req_id: int, logits: torch.Tensor,
         """
         pass  # noqa
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/sampling_params.py
user: login: coldwaterq
body: `super().__init_subclass__(**kwargs)` should probably be added to this as well.
created_at: 2025-05-08T18:08:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -69,6 +70,13 @@ def __call__(self, req_id: int, logits: torch.Tensor,
         """
         pass  # noqa
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/sampling_params.py
user: login: yibinl-nvidia
body: good catch!
created_at: 2025-05-08T18:38:50Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/serve/openai_protocol.py
user: login: yibinl-nvidia
body: @LinPoly @Superjomn could you please review the change for `OpenaiBaseModel`? My understanding is that we don't expect to add new subclass of `OpenaiBaseModel` so allow all subclass to be deserializable is safe to do.
created_at: 2025-05-09T16:59:20Z
author_association: COLLABORATOR
diff_hunk: @@ -116,26 +116,26 @@ def poll(self, timeout: int) -> bool:
     def put(self, obj: Any):
         self.setup_lazily()
         with nvtx_range_debug("send", color="blue", category="IPC"):
+            data = serialization.dumps(obj)
             if self.use_hmac_encryption:
path: tensorrt_llm/executor/ipc.py
user: login: Superjomn
body: @yibinl-nvidia should we eliminate the HMAC code? Is the serialization enhancement already sufficient for safety? If HMAC contributes little based on this PR, we can clean up the code to reduce the complexity.
created_at: 2025-05-12T06:46:43Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/serve/openai_protocol.py
user: login: LinPoly
body: IIUC, `register_approved_ipc_class` will automatically add all sub-classes of `OpenaiBaseModel` into collections of legal IPC object, if we add new subclass, it will also be registered, why would it be unsafe if new sub-class is added?
created_at: 2025-05-12T07:27:51Z
author_association: COLLABORATOR
diff_hunk: @@ -116,26 +116,26 @@ def poll(self, timeout: int) -> bool:
     def put(self, obj: Any):
         self.setup_lazily()
         with nvtx_range_debug("send", color="blue", category="IPC"):
+            data = serialization.dumps(obj)
             if self.use_hmac_encryption:
path: tensorrt_llm/executor/ipc.py
user: login: yibinl-nvidia
body: Sure, I can file a separate PR to remove HMAC.
created_at: 2025-05-12T16:19:09Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/serve/openai_protocol.py
user: login: yibinl-nvidia
body: @LinPoly The purpose of this PR is to only allow deserialization for approved class so that hacker cannot inject malicious class in deserialization. It would be unsafe if the new subclass is prone to attack. 

Put the question slightly differently, do you know if we plan to add new subclass to `OpenaiBaseModel`? If so, what kind of subclass will be added?
created_at: 2025-05-12T16:55:17Z
author_association: COLLABORATOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: Superjomn
body: Can we use the central ·BASE_ZMQ_CLASSES· without exposing it in the argument lists of classes and functions? Custom classes can be registered to this central dictionary as well. If all registrations happen at class definition time, we won’t need to worry about syncing the dictionary across processes. 
created_at: 2025-05-14T05:21:27Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/serve/openai_protocol.py
user: login: LinPoly
body: So our current subclasses are not prone to attack? If so I think it would be safe to register in this way since new subclasses (if there would be) will be very similar to what we have for now.
created_at: 2025-05-14T08:48:37Z
author_association: COLLABORATOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: yibinl-nvidia
body: @coldwaterq I vaguely remember the reason we do this in class initialization is that each process / worker might have different approved list, but it seems unnecessary since BASE_ZMQ_CLASSES is going to be static throughout the execution. And I agree with Chunwei's point here that it simplifies the code. Thoughts?
created_at: 2025-05-14T17:34:43Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +21,14 @@ class OpenAIBaseModel(BaseModel):
     # OpenAI API does not allow extra fields & allow to initialize by both alias and field name
     model_config = ConfigDict(extra="forbid", populate_by_name=True)
 
+    def __init_subclass__(cls, **kwargs):
path: tensorrt_llm/serve/openai_protocol.py
user: login: yibinl-nvidia
body: Yes, we did assessment on the current subclass and they are safe. Thanks for the confirmation.
created_at: 2025-05-14T17:38:13Z
author_association: COLLABORATOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: coldwaterq
body: This is a subprocess call, so the static list that was dynamically modified by subclass initialization has to be passed into the subprocess otherwise any class initialized only by the parent process will be blocked by the child processes deserialization
created_at: 2025-05-14T20:00:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,6 +79,7 @@ def __init__(
         tokenizer_dir: str,
         record_creator: Callable[
             ["PostprocWorker.Input", TransformersTokenizer], Any],
+        BASE_ZMQ_CLASSES: Dict,
path: tensorrt_llm/executor/postproc_worker.py
user: login: Superjomn
body: Got it, thanks for the explanation.
created_at: 2025-05-16T07:36:01Z
author_association: COLLABORATOR

PR comments:
user: login: yibinl-nvidia
created_at: 2025-04-25T20:49:47Z
body: Thanks a lot for making this PR! Several general comments:
1. Could you please follow https://github.com/NVIDIA/TensorRT-LLM/blob/main/CONTRIBUTING.md to sign your commits and run pre-commit hook.
2. Please add the complicated dataclass serialization unit test we discussed offline to `test_executor.py`. Also would be good to have additional tests for `serailization.py`. I can trigger a pipeline test after you added the tests to see if there is any test failure. 
author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-04-25T21:03:12Z
body: @kaiyux This is the "approved list approach" I discussed with you offline. Let me know your thoughts.

cc @Superjomn @litaotju @juney-nvidia 
author_association: COLLABORATOR
user: login: coldwaterq
created_at: 2025-04-28T20:10:12Z
body: @yibinl-nvidia I fixed the DCO and ran the pre-commit hooks, added the tests, and made some improvements based on the comments. let me know if there is anything else I should address.
author_association: CONTRIBUTOR
user: login: yibinl-nvidia
created_at: 2025-04-28T22:49:37Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-28T22:55:03Z
body: [PR_Github #3654](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3654/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-28T23:06:06Z
body: [PR_Github #3654](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3654/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #2585](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2585/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-04-29T03:25:44Z
body: I did a micro-benchmark comparing the custom serialization and deserialization against Pickle. The first result is based on a simple GenerationResult object (same one as in the `test_executor.py`) approximately 2 KB in size. The second result focused on benchmarking message with various size. This shows that the new approach introduces trivial μs-level of perf degradation.

```
Serialization and Deserialization Performance Results for GenerationRequest with 10000 iterations:
Pickle (avg/min/max): 199.038μs / 186.559μs / 1271.870μs
Custom (avg/min/max): 204.601μs / 192.718μs / 363.867μs
Average time difference: 5.56 μs

Size Comparison:
Pickle size: 1920 bytes
Custom size: 1920 bytes
Size ratio (custom/pickle): 1.00x

Serialization Performance Across Different Message Sizes:
Size            Pickle Avg (μs)         Custom Avg (μs)         Diff (μs)       Size Ratio
--------------------------------------------------------------------------------
1.0 KB               200.348                 205.710               5                    1.00x
10.0 KB              279.806                 285.365               5                    1.00x
100.0 KB            1044.078                1051.731               7                    1.00x
1.0 MB              8487.678                8505.488              17                    1.00x
10.0 MB            92523.012               93069.062             546                    1.00x
```

author_association: COLLABORATOR
user: login: coldwaterq
created_at: 2025-04-29T16:26:12Z
body: The last commit should resolve the issue reported by blossom. 
author_association: CONTRIBUTOR
user: login: yibinl-nvidia
created_at: 2025-04-29T16:28:02Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-29T16:33:56Z
body: [PR_Github #3744](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3744/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-29T18:17:19Z
body: [PR_Github #3744](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3744/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2650](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2650/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-04-29T21:46:05Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-29T21:52:12Z
body: [PR_Github #3766](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3766/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: coldwaterq
created_at: 2025-04-30T02:24:34Z
body: the tests that were failed in the basic bot run should be fixed with those new commits.
author_association: CONTRIBUTOR
user: login: tensorrt-cicd
created_at: 2025-04-30T04:50:25Z
body: [PR_Github #3766](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3766/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2667](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2667/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-04-30T16:25:27Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-30T16:31:04Z
body: [PR_Github #3873](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3873/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-30T23:31:10Z
body: [PR_Github #3873](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3873/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2746](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2746/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-01T20:45:24Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-01T20:53:15Z
body: [PR_Github #3941](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3941/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-02T03:55:22Z
body: [PR_Github #3941](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/3941/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2793](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2793/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-03T07:02:28Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-03T07:07:54Z
body: [PR_Github #4000](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4000/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-03T17:03:10Z
body: [PR_Github #4000](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4000/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2841](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2841/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-04T00:46:13Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-04T00:51:59Z
body: [PR_Github #4012](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4012/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-04T07:50:11Z
body: [PR_Github #4012](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4012/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2850](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2850/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-04T16:40:59Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-04T16:46:40Z
body: [PR_Github #4038](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4038/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-04T23:42:03Z
body: [PR_Github #4038](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4038/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2868](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2868/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-05T00:23:28Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-05T00:29:04Z
body: [PR_Github #4042](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4042/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-05T07:24:12Z
body: [PR_Github #4042](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4042/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2871](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2871/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-05T20:25:19Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-05T20:30:44Z
body: [PR_Github #4111](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4111/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T03:26:48Z
body: [PR_Github #4111](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4111/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2928](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2928/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-06T03:36:54Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T03:42:31Z
body: [PR_Github #4140](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4140/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T11:16:58Z
body: [PR_Github #4140](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4140/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #2954](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/2954/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-06T15:54:57Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T16:00:47Z
body: [PR_Github #4236](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4236/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T22:38:01Z
body: [PR_Github #4236](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4236/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3026](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3026/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-06T22:43:19Z
body: /bot run
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-06T22:49:54Z
body: [PR_Github #4267](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4267/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-07T05:28:20Z
body: [PR_Github #4267](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4267/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3054](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3054/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-07T17:00:13Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-07T17:06:11Z
body: [PR_Github #4413](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4413/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T00:03:53Z
body: [PR_Github #4413](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4413/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3175](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3175/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-08T20:40:24Z
body: /bot -h
author_association: COLLABORATOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:40:35Z
body: ## GitHub Bot Help

`/bot [-h] ['run', 'kill', 'skip', 'reuse-pipeline'] ...`

Provide a user friendly way for developers to interact with a Jenkins server.

Run `/bot [-h|--help]` to print this help message.

See details below for each supported subcommand.

<details>

`run  [--disable-fail-fast --skip-test --stage-list "A10-1, xxx" --gpu-type "A30, H100_PCIe" --add-multi-gpu-test --only-multi-gpu-test --disable-multi-gpu-test --post-merge --extra-stage "H100_PCIe-[Post-Merge]-1, xxx"]`

Launch build/test pipelines. All previously running jobs will be killed.

`--disable-fail-fast ` *(OPTIONAL)* : Disable fail fast on build/tests/infra failures.

`--skip-test ` *(OPTIONAL)* : Skip all test stages, but still run build stages, package stages and sanity check stages. Note: Does **NOT** update GitHub check status.

`--stage-list "A10-1, xxx"` *(OPTIONAL)* : Only run the specified test stages. Examples: "A10-1, xxx". Note: Does **NOT** update GitHub check status.

`--gpu-type "A30, H100_PCIe"` *(OPTIONAL)* : Only run the test stages on the specified GPU types. Examples: "A30, H100_PCIe". Note: Does **NOT** update GitHub check status.

`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Note: Does **NOT** update GitHub check status.

`--disable-multi-gpu-test ` *(OPTIONAL)* : Disable the multi-GPU tests. Note: Does **NOT** update GitHub check status.

`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests. Will also run L0 pre-merge pipeline.

`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline.

`--extra-stage "H100_PCIe-[Post-Merge]-1, xxx"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Examples: --extra-stage "H100_PCIe-[Post-Merge]-1, xxx".

### kill

`kill  `

Kill all running builds associated with pull request.

### skip

`skip --comment COMMENT `

Skip testing for latest commit on pull request. `--comment "Reason for skipping build/test"` is required. IMPORTANT NOTE: This is dangerous since lack of user care and validation can cause top of tree to break.

### reuse-pipeline

`reuse-pipeline `

Reuse a previous pipeline to validate current commit. This action will also kill all currently running builds associated with the pull request. IMPORTANT NOTE: This is dangerous since lack of user care and validation can cause top of tree to break.

</details>
author_association: NONE
user: login: yibinl-nvidia
created_at: 2025-05-08T20:44:51Z
body: /bot run --gpu-type "A10"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T20:50:48Z
body: [PR_Github #4603](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4603/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-08T22:42:24Z
body: /bot kill
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T22:45:56Z
body: [PR_Github #4603](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4603/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #3317 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3317/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-08T22:48:44Z
body: /bot run --gpu-type "A10"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T22:53:19Z
body: [PR_Github #4604](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4604/) [ kill ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T22:53:21Z
body: [PR_Github #4604](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4604/) [ kill ] completed with state `SUCCESS`
Successfully killed previous jobs for commit `bfe54ce`

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-08T22:54:18Z
body: /bot run --gpu-type "A10"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-08T22:59:41Z
body: [PR_Github #4605](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4605/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T02:03:42Z
body: [PR_Github #4605](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4605/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3318 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3318/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-09T04:29:26Z
body: /bot run --stage-list "A10-TensorRT-6"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T04:35:12Z
body: [PR_Github #4645](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4645/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T05:05:41Z
body: [PR_Github #4645](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4645/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #3351 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3351/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-09T06:01:36Z
body: /bot run --stage-list "A10-TensorRT-6"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T06:07:10Z
body: [PR_Github #4661](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4661/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T09:09:33Z
body: [PR_Github #4661](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4661/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3362 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3362/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-09T16:55:13Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T17:01:48Z
body: [PR_Github #4732](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4732/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T21:09:56Z
body: [PR_Github #4732](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4732/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3415](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3415/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-09T22:56:03Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-09T23:01:36Z
body: [PR_Github #4738](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4738/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-10T05:02:58Z
body: [PR_Github #4738](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4738/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3420](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3420/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-10T05:49:20Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-10T05:54:48Z
body: [PR_Github #4753](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4753/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-10T09:37:27Z
body: [PR_Github #4753](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4753/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3432 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3432/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-10T21:17:20Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-10T21:22:59Z
body: [PR_Github #4763](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4763/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-11T00:36:53Z
body: [PR_Github #4763](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4763/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3439 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3439/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-11T07:35:57Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-11T07:41:29Z
body: [PR_Github #4770](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4770/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-11T11:04:16Z
body: [PR_Github #4770](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4770/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3446 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3446/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-11T18:39:42Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-11T18:45:24Z
body: [PR_Github #4788](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4788/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-11T22:34:35Z
body: [PR_Github #4788](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4788/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3463 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3463/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-12T00:57:21Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-12T01:02:46Z
body: [PR_Github #4796](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4796/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-12T04:19:57Z
body: [PR_Github #4796](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4796/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3469 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3469/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-12T06:31:30Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-12T06:37:52Z
body: [PR_Github #4838](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4838/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-12T19:18:58Z
body: [PR_Github #4838](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4838/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3504](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3504/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-12T23:50:27Z
body: @Superjomn @kaiyux Could you re-review this PR as pipeline finally passed. Thanks!
author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-13T00:25:47Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T00:31:31Z
body: [PR_Github #4918](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4918/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-13T04:30:42Z
body: [PR_Github #4918](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/4918/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3568 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3568/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-15T06:31:03Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-15T06:37:16Z
body: [PR_Github #5282](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5282/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-15T12:23:04Z
body: [PR_Github #5282](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5282/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3858](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3858/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-16T00:23:26Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-16T00:29:27Z
body: [PR_Github #5423](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5423/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-16T07:53:00Z
body: [PR_Github #5423](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5423/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #3959](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/3959/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-16T15:09:26Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-16T15:15:01Z
body: [PR_Github #5521](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5521/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-16T23:27:50Z
body: [PR_Github #5521](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5521/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4023](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4023/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-19T06:47:27Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-19T06:55:04Z
body: [PR_Github #5685](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5685/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-19T11:47:56Z
body: [PR_Github #5685](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5685/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #4153](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4153/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-20T03:02:27Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T03:09:29Z
body: [PR_Github #5808](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5808/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T06:25:28Z
body: [PR_Github #5808](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5808/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4254](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4254/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-20T09:04:03Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T09:09:33Z
body: [PR_Github #5857](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5857/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-20T20:02:26Z
body: /bot kill
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T20:08:04Z
body: [PR_Github #5908](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5908/) [ kill ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T20:08:06Z
body: [PR_Github #5857](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5857/) [ run ] completed with state `ABORTED`

author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T20:08:36Z
body: [PR_Github #5908](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5908/) [ kill ] completed with state `SUCCESS`
Successfully killed previous jobs for commit `55aedb0`

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-20T20:14:04Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-20T20:19:25Z
body: [PR_Github #5910](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5910/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-21T05:45:03Z
body: [PR_Github #5910](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/5910/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4331 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4331/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-21T16:27:28Z
body: /bot run --only-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-21T16:32:49Z
body: [PR_Github #6046](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6046/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-21T22:09:57Z
body: [PR_Github #6046](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6046/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4418 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4418/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-21T23:35:28Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-21T23:44:20Z
body: [PR_Github #6066](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6066/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-22T03:20:10Z
body: [PR_Github #6066](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6066/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4437](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4437/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-22T04:33:09Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-22T04:38:06Z
body: [PR_Github #6102](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6102/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-22T06:07:35Z
body: [PR_Github #6102](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6102/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4461](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4461/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: yibinl-nvidia
created_at: 2025-05-22T07:02:14Z
body: /bot run --add-multi-gpu-test --disable-fail-fast
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-22T07:07:50Z
body: [PR_Github #6123](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6123/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-22T18:54:19Z
body: [PR_Github #6123](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6123/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #4471](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/4471/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: kaiyux
created_at: 2025-05-23T02:50:22Z
body: /bot reuse-pipeline
author_association: MEMBER
user: login: tensorrt-cicd
created_at: 2025-05-23T02:55:26Z
body: [PR_Github #6217](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6217/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-05-23T03:01:30Z
body: [PR_Github #6217](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/6217/) [ reuse-pipeline ] completed with state `SUCCESS`
Reusing [PR_Github #6123](https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/helpers/job/PR_Github/6123/) for commit `9b9df25`

author_association: COLLABORATOR