Repository: NVIDIA/TensorRT-LLM
Pull request #2985: feat: Add support of chat completion in PD
Author: login: Shunkangz
Created: 2025-03-23T14:54:24Z | Merged: 2025-04-11T09:53:29Z

Add support of chat completion in PD and fix the include_usage option. 


Diff:
diff --git a/examples/disaggregated/clients/disagg_client.py b/examples/disaggregated/clients/disagg_client.py
index e044f36644a4..f74b110bdef0 100644
--- a/examples/disaggregated/clients/disagg_client.py
+++ b/examples/disaggregated/clients/disagg_client.py
@@ -63,6 +63,61 @@ async def send_request(session, server_host, server_port, model, prompt,
             return text
 
 
+async def send_chat_request(session, server_host, server_port, model, prompt,
+                            max_tokens, temperature, streaming):
+    url = f"http://{server_host}:{server_port}/v1/chat/completions"
+    headers = {"Content-Type": "application/json"}
+    data = {
+        "model":
+        model,
+        "messages": [{
+            "role": "system",
+            "content": "You are a helpfule assistant."
+        }, {
+            "role": "user",
+            "content": prompt
+        }],
+        "max_tokens":
+        max_tokens,
+        "temperature":
+        temperature
+    }
+    if streaming:
+        data["stream"] = True
+
+    async with session.post(url, headers=headers, json=data) as response:
+        if response.status != 200:
+            raise Exception(f"Error: {await response.text()}")
+
+        if streaming:
+            text = ""
+            iter = 0
+            async for line in response.content:
+                if line:
+                    line = line.decode('utf-8').strip()
+                    if line == "data: [DONE]":
+                        break
+                    if line.startswith("data: "):
+                        line = line[len("data: "):]
+                        response_json = json.loads(line)
+                        if iter == 0:
+                            text += response_json["choices"][0]["message"][
+                                "content"]
+                        else:
+                            if "content" in response_json["choices"][0][
+                                    "delta"]:
+                                text += response_json["choices"][0]["delta"][
+                                    "content"]
+                        iter += 1
+            logging.info(text)
+            return text
+        else:
+            response_json = await response.json()
+            text = response_json["choices"][0]["message"]["content"]
+            logging.info(text)
+            return text
+
+
 async def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-c",
@@ -85,6 +140,11 @@ async def main():
                         type=int,
                         help="Time to wait for server to start",
                         default=None)
+    parser.add_argument("-e",
+                        "--endpoint",
+                        type=str,
+                        help="Endpoint to use",
+                        default="completions")
     parser.add_argument("-o",
                         "--output-file",
                         type=str,
@@ -111,11 +171,18 @@ async def main():
             await wait_for_server(session, server_host, server_port,
                                   args.server_start_timeout)
 
-        tasks = [
-            send_request(session, server_host, server_port, model, prompt,
-                         args.max_tokens, args.temperature, args.streaming)
-            for prompt in prompts
-        ]
+        if args.endpoint == "completions":
+            tasks = [
+                send_request(session, server_host, server_port, model, prompt,
+                             args.max_tokens, args.temperature, args.streaming)
+                for prompt in prompts
+            ]
+        elif args.endpoint == "chat":
+            tasks = [
+                send_chat_request(session, server_host, server_port, model,
+                                  prompt, args.max_tokens, args.temperature,
+                                  args.streaming) for prompt in prompts
+            ]
 
         responses = await asyncio.gather(*tasks)
 
diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py
index 6fb02460368f..0646679147b4 100644
--- a/tensorrt_llm/serve/openai_disagg_server.py
+++ b/tensorrt_llm/serve/openai_disagg_server.py
@@ -7,7 +7,7 @@
 import signal
 from contextlib import asynccontextmanager
 from http import HTTPStatus
-from typing import List, Optional, Union
+from typing import List, Optional, Type, Union
 
 import aiohttp
 import uvicorn
@@ -17,7 +17,9 @@
 
 # yapf: disable
 from tensorrt_llm.executor import CppExecutorError
-from tensorrt_llm.serve.openai_protocol import (CompletionRequest,
+from tensorrt_llm.serve.openai_protocol import (ChatCompletionRequest,
+                                                ChatCompletionResponse,
+                                                CompletionRequest,
                                                 CompletionResponse,
                                                 DisaggregatedParams,
                                                 ErrorResponse)
@@ -81,6 +83,9 @@ def register_routes(self):
         self.app.add_api_route("/v1/completions",
                                self.openai_completion,
                                methods=["POST"])
+        self.app.add_api_route("/v1/chat/completions",
+                               self.openai_chat_completion,
+                               methods=["POST"])
 
     async def health(self) -> Response:
         return Response(status_code=200)
@@ -89,7 +94,9 @@ async def version(self) -> JSONResponse:
         ver = {"version": VERSION}
         return JSONResponse(content=ver)
 
-    async def merge_streaming_responses(self, ctx_response, gen_server, gen_req):
+    async def merge_streaming_responses(self, ctx_response,
+                                        gen_server: str,
+                                        gen_req: Union[CompletionRequest, ChatCompletionRequest]):
         # First yield the context response if it's not None
         if ctx_response is not None:
             # Remove the disaggregated params from the context response
@@ -99,7 +106,13 @@ async def merge_streaming_responses(self, ctx_response, gen_server, gen_req):
             yield f"data: {data}\n\n".encode('utf-8')
 
         # Then yield the generation responses
-        gen_response = await self.send_request(gen_server, gen_req)
+        if isinstance(gen_req, CompletionRequest):
+            gen_response = await self.send_completion_request(gen_server, gen_req)
+        elif isinstance(gen_req, ChatCompletionRequest):
+            gen_response = await self.send_chat_request(gen_server, gen_req)
+        else:
+            raise TypeError("Invalid request type: {type(gen_req).__name__}")
+
         async for chunk in gen_response.body_iterator:
             yield chunk
 
@@ -113,60 +126,88 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
                 elif not isinstance(req.prompt, list) or not all(isinstance(x, int) for x in req.prompt):
                     raise ValueError("Disaggregated server currently only supports single string prompt or list of integers in request")
 
-            ctx_response = None
-            if os.getenv("TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") == "1":
-                # Hard-code first token, ctx_request_id for testing
-                gen_req.disaggregated_params = DisaggregatedParams(request_type="generation_only", first_gen_tokens=[7], ctx_request_id=1, encoded_opaque_state=None, draft_tokens=None)
-                # Since KV cache for prompt tokens will be uninitialized, need to ignore eos
-                gen_req.ignore_eos = True
-            else:
-                # Pick a context server
-                ctx_server = self.get_next_server(self.ctx_servers, "context")
-                logging.info("Sending request to ctx server: %s", ctx_server)
-
-                # Send request to context server
-                req.max_tokens = 1
-                req.disaggregated_params = DisaggregatedParams(request_type="context_only")
-                # Disable streaming for context server
-                req.stream = False
-                req.stream_options = None
-                ctx_response = await self.send_request(ctx_server, req)
-
-                #TODO: Context server should skip de-tokenization and return raw tokens
-                choices = ctx_response.choices
-                if len(choices) > 1:
-                    raise ValueError("Disagg server returned more than one choice. This is currently not supported in disaggregated server.")
-                if choices[0].disaggregated_params is None:
-                    raise ValueError("Context server did not return disaggregated params")
-
-                # Append disaggregates parameters to generation request
-                gen_req.disaggregated_params = choices[0].disaggregated_params
-
-            gen_req.disaggregated_params.request_type = "generation_only"
-
-            # Pick a generation server and send request
-            gen_server = self.get_next_server(self.gen_servers, "generation")
-            logging.info("Sending request to gen server: %s", gen_server)
-
-            if not gen_req.stream:
-                gen_response = await self.send_request(gen_server, gen_req)
-                return gen_response
-            else:
-                # Return a streaming response that combines both context and generation responses
-                return StreamingResponse(
-                    self.merge_streaming_responses(ctx_response, gen_server, gen_req),
-                    media_type="text/event-stream"
-                )
-
-        except CppExecutorError as e:
-            # If internal executor error is raised, shutdown the server
-            logging.exception(e)
-            signal.raise_signal(signal.SIGINT)
-        except HTTPException as e:
-            raise e  # Re-raise HTTP exceptions properly
+            ctx_response = await self._process_context_server_request(req, "completion")
+
+            return await self._process_generation_server_request(gen_req, ctx_response)
+
+        except Exception as e:
+            await self._handle_exception(e)
+
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
+
+        try:
+            gen_req = copy.deepcopy(req)
+            ctx_response = await self._process_context_server_request(req, "chat")
+
+            return await self._process_generation_server_request(gen_req, ctx_response)
         except Exception as e:
-            logging.exception(e)
-            raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
+            await self._handle_exception(e)
+
+    async def _handle_exception(self, exception):
+        if isinstance(exception, CppExecutorError):
+            logging.exception(exception)
+            signal.raise_signal(signal.SIGINT)
+        elif isinstance(exception, HTTPException):
+            raise exception  # Re-raise HTTP exceptions properly
+        else:
+            logging.exception(exception)
+            raise HTTPException(status_code=500, detail=f"Internal server error {str(exception)}")
+
+    async def _process_context_server_request(self, ctx_req, request_type: str):
+        # No need to send request to context server if we are benchmarking generation only
+        if os.getenv("TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") == "1":
+            return None
+
+        ctx_server = self.get_next_server(self.ctx_servers, "context")
+        logging.info("Sending request to ctx server: %s", ctx_server)
+
+        if request_type == "chat":
+            ctx_req.max_completion_tokens = 1
+        elif request_type == "completion":
+            ctx_req.max_tokens = 1
+        ctx_req.disaggregated_params = DisaggregatedParams(request_type="context_only")
+        ctx_req.stream = False
+        ctx_req.stream_options = None
+
+        if request_type == "chat":
+            return await self.send_chat_request(ctx_server, ctx_req)
+        elif request_type == "completion":
+            return await self.send_completion_request(ctx_server, ctx_req)
+
+    async def _process_generation_server_request(self, gen_req, ctx_response):
+        if os.getenv("TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") == "1":
+            # Hard-code first token, ctx_request_id for testing
+            gen_req.disaggregated_params = DisaggregatedParams(request_type="generation_only", first_gen_tokens=[7], ctx_request_id=1, encoded_opaque_state=None, draft_tokens=None)
+            # Since KV cache for prompt tokens will be uninitialized, need to ignore eos
+            gen_req.ignore_eos = True
+        else:
+            choices = ctx_response.choices
+            if len(choices) > 1:
+                raise ValueError("Disagg server returned more than one choice. This is currently not supported in disaggregated server.")
+            if choices[0].disaggregated_params is None:
+                raise ValueError("Context server did not return disaggregated params")
+
+            # Append disaggregates parameters to generation request
+            gen_req.disaggregated_params = choices[0].disaggregated_params
+        gen_req.disaggregated_params.request_type = "generation_only"
+
+        # Pick a generation server and send request
+        gen_server = self.get_next_server(self.gen_servers, "generation")
+        logging.info("Sending request to gen server: %s", gen_server)
+
+        if not gen_req.stream:
+            if isinstance(gen_req, CompletionRequest):
+                gen_response = await self.send_completion_request(gen_server, gen_req)
+            elif isinstance(gen_req, ChatCompletionRequest):
+                gen_response = await self.send_chat_request(gen_server, gen_req)
+
+            return gen_response
+        else:
+            # Return a streaming response that combines both context and generation responses
+            return StreamingResponse(
+                self.merge_streaming_responses(ctx_response, gen_server, gen_req),
+                media_type="text/event-stream"
+            )
 
     async def __call__(self, host, port):
         config = uvicorn.Config(self.app,
@@ -192,10 +233,8 @@ def get_next_server(self, servers: List[str], server_type: str) -> str:
 
         return server
 
-
-    async def create_completion_generator(self, url: str, request: CompletionRequest):
-        async with self.session.post(url + "/v1/completions", json=request.model_dump(exclude_unset=True)) as response:
-
+    async def create_generator(self, url: str, request: Union[CompletionRequest, ChatCompletionRequest], end_point: str):
+        async with self.session.post(url + end_point, json=request.model_dump(exclude_unset=True)) as response:
             content_type = response.headers.get("Content-Type", "")
             if "text/event-stream" in content_type:
                 if not request.stream:
@@ -206,20 +245,28 @@ async def create_completion_generator(self, url: str, request: CompletionRequest
                         if line:
                             yield line
                             await asyncio.sleep(0)
-
                 except Exception as e:
                     logging.error(f"Unexpected error in stream: {e}")
                     raise
 
-    async def send_request(self, url: str, request: CompletionRequest) -> Union[CompletionResponse, StreamingResponse]:
+    async def create_completion_generator(self, url: str, request: CompletionRequest):
+        async for chunk in self.create_generator(url, request, "/v1/completions"):
+            yield chunk
+
+    async def create_chat_generator(self, url: str, request: ChatCompletionRequest):
+        async for chunk in self.create_generator(url, request, "/v1/chat/completions"):
+            yield chunk
 
+    async def send_request(self, url: str,
+                           request: Union[CompletionRequest, ChatCompletionRequest],
+                           endpoint: str,
+                           response_type: Type[Union[CompletionResponse, ChatCompletionResponse]],
+                           create_generator: callable) -> Union[CompletionResponse, ChatCompletionResponse, StreamingResponse]:
         if request.stream:
-            response_generator = self.create_completion_generator(url, request)
+            response_generator = create_generator(url, request)
             return StreamingResponse(content=response_generator, media_type="text/event-stream")
         else:
-            """Send an asynchronous request and return JSON response"""
-            async with self.session.post(url + "/v1/completions", json=request.model_dump(exclude_unset=True)) as response:
-
+            async with self.session.post(url + endpoint, json=request.model_dump(exclude_unset=True)) as response:
                 content_type = response.headers.get("Content-Type", "")
                 if "text/event-stream" in content_type:
                     raise ValueError("Received an event-stream although request stream was False")
@@ -228,7 +275,13 @@ async def send_request(self, url: str, request: CompletionRequest) -> Union[Comp
                 if not response.ok:
                     logging.error(f"Received failed response {response_dict}")
                     response.raise_for_status()
-                return CompletionResponse(**response_dict)
+                return response_type(**response_dict)
+
+    async def send_completion_request(self, url: str, request: CompletionRequest) -> Union[CompletionResponse, StreamingResponse]:
+        return await self.send_request(url, request, "/v1/completions", CompletionResponse, self.create_completion_generator)
+
+    async def send_chat_request(self, url: str, request: ChatCompletionRequest) -> ChatCompletionResponse:
+        return await self.send_request(url, request, "/v1/chat/completions", ChatCompletionResponse, self.create_chat_generator)
 
     async def check_server_ready(self, server_url: str) -> bool:
         try:
diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py
index faaef5906a79..9d4196eef7c5 100644
--- a/tensorrt_llm/serve/openai_protocol.py
+++ b/tensorrt_llm/serve/openai_protocol.py
@@ -88,22 +88,6 @@ class CompletionResponseChoice(OpenAIBaseModel):
     )
     disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
 
-    @staticmethod
-    def to_disaggregated_params(
-            tllm_disagg_params: LlmDisaggregatedParams) -> DisaggregatedParams:
-        if tllm_disagg_params is None:
-            return None
-        else:
-            encoded_opaque_state = base64.b64encode(
-                tllm_disagg_params.opaque_state).decode(
-                    "utf-8") if tllm_disagg_params is not None else None
-            return DisaggregatedParams(
-                request_type=tllm_disagg_params.request_type,
-                first_gen_tokens=tllm_disagg_params.first_gen_tokens,
-                ctx_request_id=tllm_disagg_params.ctx_request_id,
-                encoded_opaque_state=encoded_opaque_state,
-                draft_tokens=tllm_disagg_params.draft_tokens)
-
 
 class CompletionResponse(OpenAIBaseModel):
     id: str = Field(default_factory=lambda: f"cmpl-{str(uuid.uuid4().hex)}")
@@ -234,21 +218,6 @@ def to_sampling_params(self) -> SamplingParams:
         )
         return sampling_params
 
-    def to_llm_disaggregated_params(self) -> LlmDisaggregatedParams:
-        if self.disaggregated_params is None:
-            return None
-        else:
-            opaque_state = base64.b64decode(
-                self.disaggregated_params.encoded_opaque_state
-            ) if self.disaggregated_params.encoded_opaque_state is not None else None
-
-            return LlmDisaggregatedParams(
-                request_type=self.disaggregated_params.request_type,
-                first_gen_tokens=self.disaggregated_params.first_gen_tokens,
-                ctx_request_id=self.disaggregated_params.ctx_request_id,
-                opaque_state=opaque_state,
-                draft_tokens=self.disaggregated_params.draft_tokens)
-
     def model_post_init(self, __context: Any) -> None:
         if self.best_of is None:
             self.best_of = self.n
@@ -379,6 +348,8 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
     finish_reason: Optional[str] = None
     stop_reason: Optional[Union[int, str]] = None
 
+    disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
+
 
 class ChatCompletionResponse(OpenAIBaseModel):
     id: str = Field(default_factory=lambda: f"chatcmpl-{str(uuid.uuid4().hex)}")
@@ -520,6 +491,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
                      "Will be accessible by the chat template."),
     )
 
+    disaggregated_params: Optional[DisaggregatedParams] = Field(
+        default=None,
+        description=("Parameters for disaggregated serving"),
+    )
+
     # doc: end-chat-completion-extra-params
 
     def to_sampling_params(self) -> SamplingParams:
@@ -632,3 +608,41 @@ def check_special_tokens(cls, data):
             raise ValueError(
                 "special_tokens related settings are not supported")
         return data
+
+
+def encode_opaque_state(opaque_state: Optional[bytes]) -> Optional[str]:
+    if opaque_state is None:
+        return None
+    return base64.b64encode(opaque_state).decode("utf-8")
+
+
+def decode_opaque_state(encoded_opaque_state: Optional[str]) -> Optional[bytes]:
+    if encoded_opaque_state is None:
+        return None
+    return base64.b64decode(encoded_opaque_state)
+
+
+def to_disaggregated_params(
+        tllm_disagg_params: LlmDisaggregatedParams) -> DisaggregatedParams:
+    if tllm_disagg_params is None:
+        return None
+    return DisaggregatedParams(
+        request_type=tllm_disagg_params.request_type,
+        first_gen_tokens=tllm_disagg_params.first_gen_tokens,
+        ctx_request_id=tllm_disagg_params.ctx_request_id,
+        encoded_opaque_state=encode_opaque_state(
+            tllm_disagg_params.opaque_state),
+        draft_tokens=tllm_disagg_params.draft_tokens)
+
+
+def to_llm_disaggregated_params(
+        disaggregated_params: DisaggregatedParams) -> LlmDisaggregatedParams:
+    if disaggregated_params is None:
+        return None
+    return LlmDisaggregatedParams(
+        request_type=disaggregated_params.request_type,
+        first_gen_tokens=disaggregated_params.first_gen_tokens,
+        ctx_request_id=disaggregated_params.ctx_request_id,
+        opaque_state=decode_opaque_state(
+            disaggregated_params.encoded_opaque_state),
+        draft_tokens=disaggregated_params.draft_tokens)
diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py
index 4d7b7255a881..332d244c8a15 100644
--- a/tensorrt_llm/serve/openai_server.py
+++ b/tensorrt_llm/serve/openai_server.py
@@ -27,7 +27,8 @@
                                                 CompletionResponse,
                                                 CompletionResponseChoice,
                                                 ErrorResponse, ModelCard,
-                                                ModelList, UsageInfo)
+                                                ModelList, UsageInfo,
+                                                to_llm_disaggregated_params)
 from tensorrt_llm.serve.postprocess_handlers import (
     ChatPostprocArgs, CompletionPostprocArgs, chat_response_post_processor,
     chat_stream_post_processor, completion_response_post_processor,
@@ -194,6 +195,7 @@ async def create_chat_response(
                 **(request.chat_template_kwargs or {}),
             )
             sampling_params = request.to_sampling_params()
+            disaggregated_params = to_llm_disaggregated_params(request.disaggregated_params)
             postproc_args = ChatPostprocArgs.from_request(request)
             if conversation and conversation[-1].get(
                     "content") and conversation[-1].get("role") == get_role():
@@ -209,6 +211,7 @@ async def create_chat_response(
                 sampling_params=sampling_params,
                 _postproc_params=postproc_params if self.postproc_worker_enabled else None,
                 streaming=request.stream,
+                disaggregated_params=disaggregated_params
             )
             asyncio.create_task(self.await_disconnected(raw_request, promise))
             if not self.postproc_worker_enabled:
@@ -306,7 +309,7 @@ async def create_completion_response(
             promises: List[RequestOutput] = []
             postproc_params_collection: List[Optional[PostprocParams]] = []
             sampling_params = request.to_sampling_params()
-            disaggregated_params = request.to_llm_disaggregated_params()
+            disaggregated_params = to_llm_disaggregated_params(request.disaggregated_params)
             for idx, prompt in enumerate(prompts):
                 postproc_args = CompletionPostprocArgs.from_request(request)
                 postproc_args.prompt_idx = idx
diff --git a/tensorrt_llm/serve/postprocess_handlers.py b/tensorrt_llm/serve/postprocess_handlers.py
index 7ae44191eee6..fe3a79c12405 100644
--- a/tensorrt_llm/serve/postprocess_handlers.py
+++ b/tensorrt_llm/serve/postprocess_handlers.py
@@ -19,7 +19,8 @@
                               CompletionResponseChoice,
                               CompletionResponseStreamChoice,
                               CompletionStreamResponse, DeltaMessage,
-                              FunctionCall, StreamOptions, ToolCall, UsageInfo)
+                              FunctionCall, StreamOptions, ToolCall, UsageInfo,
+                              to_disaggregated_params)
 
 # yapf: enale
 
@@ -175,11 +176,13 @@ def chat_response_post_processor(rsp: GenerationResultBase, args: ChatPostprocAr
                 ])
         else:
             message = ChatMessage(role=role, content=output.text)
+        disaggregated_params = to_disaggregated_params(output.disaggregated_params)
         choice = ChatCompletionResponseChoice(
             index=output.index,
             message=message,
             finish_reason=output.finish_reason,
             stop_reason=output.stop_reason,
+            disaggregated_params=disaggregated_params,
         )
 
         if args.return_logprobs:
@@ -281,8 +284,7 @@ def completion_response_post_processor(rsp: GenerationResult, args: CompletionPo
         text = output.text
         if args.echo:
             text = args.prompt + text
-        disaggregated_params = CompletionResponseChoice.to_disaggregated_params(
-            output.disaggregated_params)
+        disaggregated_params = to_disaggregated_params(output.disaggregated_params)
         choice = CompletionResponseChoice(
             text=text,
             index=args.prompt_idx * args.num_choices + output.index,
diff --git a/tests/integration/defs/disaggregated/sanity_check.sh b/tests/integration/defs/disaggregated/sanity_check.sh
index d2919f528ac3..979a2d3f229b 100644
--- a/tests/integration/defs/disaggregated/sanity_check.sh
+++ b/tests/integration/defs/disaggregated/sanity_check.sh
@@ -76,6 +76,12 @@ trtllm-serve disaggregated --server_start_timeout 900 -c ${CONFIG_FILE} &> outpu
 for i in $(seq 1 ${NUM_ITERS}); do
     python3 ${CLIENT_DIR}/disagg_client.py -c ${EXAMPLE_DIR}/disagg_config.yaml -p ${CLIENT_DIR}/prompts.json --server-start-timeout 950
     python3 ${CLIENT_DIR}/disagg_client.py -c ${EXAMPLE_DIR}/disagg_config.yaml -p ${CLIENT_DIR}/prompts.json --server-start-timeout 950 --streaming -o output_streaming.json
+
+    # Run the chat completion endpoint test only for TinyLlama
+    if [[ "${TEST_DESC}" == "overlap" ]]; then
+      python3 ${CLIENT_DIR}/disagg_client.py -c ${EXAMPLE_DIR}/disagg_config.yaml -p ${CLIENT_DIR}/prompts.json -e chat --server-start-timeout 950 -o output_chat.json
+      python3 ${CLIENT_DIR}/disagg_client.py -c ${EXAMPLE_DIR}/disagg_config.yaml -p ${CLIENT_DIR}/prompts.json -e chat --server-start-timeout 950 --streaming -o output_streaming_chat.json
+    fi
 done
 
 echo "------------------"
@@ -118,5 +124,10 @@ if [[ "${TEST_DESC}" != "gen_only" ]]; then
   for expected_string in "${expected_strings[@]}"; do
       grep "${expected_string}" output.json
       grep "${expected_string}" output_streaming.json
+
+      if [[ "${TEST_DESC}" == "overlap" ]]; then
+        grep "${expected_string}" output_chat.json
+        grep "${expected_string}" output_streaming_chat.json
+      fi
   done
 fi


Reviews:
user: login: pcastonguay
body: There are no tests for the chat endpoint. Could we add one please? Thanks.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T14:09:43Z
user: login: Shunkangz
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T07:41:00Z
user: login: Shunkangz
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T07:41:24Z
user: login: Shunkangz
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T07:41:45Z
user: login: Shunkangz
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T15:13:39Z
user: login: pcastonguay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T18:43:24Z
user: login: pcastonguay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T12:46:09Z
user: login: Shunkangz
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-09T14:57:55Z
user: login: pcastonguay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-09T23:22:43Z
user: login: pcastonguay
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-09T23:23:12Z

Review comments:
diff_hunk: @@ -379,6 +379,24 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
     finish_reason: Optional[str] = None
     stop_reason: Optional[Union[int, str]] = None
 
+    disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
+
+    @staticmethod
+    def to_disaggregated_params(
+            tllm_disagg_params: LlmDisaggregatedParams) -> DisaggregatedParams:
+        if tllm_disagg_params is None:
+            return None
+        else:
+            encoded_opaque_state = base64.b64encode(
+                tllm_disagg_params.opaque_state).decode(
+                    "utf-8") if tllm_disagg_params is not None else None
+            return DisaggregatedParams(
+                request_type=tllm_disagg_params.request_type,
+                first_gen_tokens=tllm_disagg_params.first_gen_tokens,
+                ctx_request_id=tllm_disagg_params.ctx_request_id,
+                encoded_opaque_state=encoded_opaque_state,
+                draft_tokens=tllm_disagg_params.draft_tokens)
path: tensorrt_llm/serve/openai_protocol.py
user: login: pcastonguay
body: Looks like this is copy pasted from above. Can we refactor to avoid code duplication?
created_at: 2025-03-24T14:01:37Z
author_association: COLLABORATOR
diff_hunk: @@ -557,6 +580,21 @@ def to_sampling_params(self) -> SamplingParams:
         )
         return sampling_params
 
+    def to_llm_disaggregated_params(self) -> LlmDisaggregatedParams:
+        if self.disaggregated_params is None:
+            return None
+        else:
+            opaque_state = base64.b64decode(
+                self.disaggregated_params.encoded_opaque_state
+            ) if self.disaggregated_params.encoded_opaque_state is not None else None
+
+            return LlmDisaggregatedParams(
+                request_type=self.disaggregated_params.request_type,
+                first_gen_tokens=self.disaggregated_params.first_gen_tokens,
+                ctx_request_id=self.disaggregated_params.ctx_request_id,
+                opaque_state=opaque_state,
+                draft_tokens=self.disaggregated_params.draft_tokens)
path: tensorrt_llm/serve/openai_protocol.py
user: login: pcastonguay
body: Looks like this is copy pasted from above. Can we refactor to avoid code duplication.
created_at: 2025-03-24T14:02:40Z
author_association: COLLABORATOR
diff_hunk: @@ -134,6 +140,50 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.exception(e)
             raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
 
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
+
+        try:
+            gen_req = copy.deepcopy(req)
+
+            # Pick a context server
+            ctx_server = self.get_next_server(self.ctx_servers, "context")
+            logging.info("Sending request to ctx server: %s", ctx_server)
+
+            # Send request to context server
+            req.max_completion_tokens = 1
+            req.disaggregated_params = DisaggregatedParams(request_type="context_only")
+            # Disable streaming for context server
+            req.stream = False
+            req.stream_options = None
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: pcastonguay
body: Default should be None, do we need to have this line?
created_at: 2025-03-24T14:06:56Z
author_association: COLLABORATOR
diff_hunk: @@ -134,6 +140,50 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.exception(e)
             raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
 
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
+
+        try:
+            gen_req = copy.deepcopy(req)
+
+            # Pick a context server
+            ctx_server = self.get_next_server(self.ctx_servers, "context")
+            logging.info("Sending request to ctx server: %s", ctx_server)
+
+            # Send request to context server
+            req.max_completion_tokens = 1
+            req.disaggregated_params = DisaggregatedParams(request_type="context_only")
+            # Disable streaming for context server
+            req.stream = False
+            req.stream_options = None
+            ctx_response = await self.send_chat_request(ctx_server, req)
+
+            #TODO: Context server should skip de-tokenization and return raw tokens
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: pcastonguay
body: This can be removed since now the context server skips detokenization. Simiarly, can remove for completions endpoint.
created_at: 2025-03-24T14:08:25Z
author_association: COLLABORATOR
diff_hunk: @@ -134,6 +140,50 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.exception(e)
             raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
 
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
+
+        try:
+            gen_req = copy.deepcopy(req)
+
+            # Pick a context server
+            ctx_server = self.get_next_server(self.ctx_servers, "context")
+            logging.info("Sending request to ctx server: %s", ctx_server)
+
+            # Send request to context server
+            req.max_completion_tokens = 1
+            req.disaggregated_params = DisaggregatedParams(request_type="context_only")
+            # Disable streaming for context server
+            req.stream = False
+            req.stream_options = None
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: Shunkangz
body: I think so. The user might enable include_usage in their HTTP request. In this case, both of stream and stream_options need to be set as None. 
created_at: 2025-03-25T07:41:00Z
author_association: COLLABORATOR
diff_hunk: @@ -134,6 +140,50 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.exception(e)
             raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
 
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
+
+        try:
+            gen_req = copy.deepcopy(req)
+
+            # Pick a context server
+            ctx_server = self.get_next_server(self.ctx_servers, "context")
+            logging.info("Sending request to ctx server: %s", ctx_server)
+
+            # Send request to context server
+            req.max_completion_tokens = 1
+            req.disaggregated_params = DisaggregatedParams(request_type="context_only")
+            # Disable streaming for context server
+            req.stream = False
+            req.stream_options = None
+            ctx_response = await self.send_chat_request(ctx_server, req)
+
+            #TODO: Context server should skip de-tokenization and return raw tokens
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: Shunkangz
body: Sure. 
created_at: 2025-03-25T07:41:24Z
author_association: COLLABORATOR
diff_hunk: @@ -379,6 +379,24 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
     finish_reason: Optional[str] = None
     stop_reason: Optional[Union[int, str]] = None
 
+    disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
+
+    @staticmethod
+    def to_disaggregated_params(
+            tllm_disagg_params: LlmDisaggregatedParams) -> DisaggregatedParams:
+        if tllm_disagg_params is None:
+            return None
+        else:
+            encoded_opaque_state = base64.b64encode(
+                tllm_disagg_params.opaque_state).decode(
+                    "utf-8") if tllm_disagg_params is not None else None
+            return DisaggregatedParams(
+                request_type=tllm_disagg_params.request_type,
+                first_gen_tokens=tllm_disagg_params.first_gen_tokens,
+                ctx_request_id=tllm_disagg_params.ctx_request_id,
+                encoded_opaque_state=encoded_opaque_state,
+                draft_tokens=tllm_disagg_params.draft_tokens)
path: tensorrt_llm/serve/openai_protocol.py
user: login: Shunkangz
body: No problem. 
created_at: 2025-03-25T07:41:45Z
author_association: COLLABORATOR
diff_hunk: @@ -557,6 +580,21 @@ def to_sampling_params(self) -> SamplingParams:
         )
         return sampling_params
 
+    def to_llm_disaggregated_params(self) -> LlmDisaggregatedParams:
+        if self.disaggregated_params is None:
+            return None
+        else:
+            opaque_state = base64.b64decode(
+                self.disaggregated_params.encoded_opaque_state
+            ) if self.disaggregated_params.encoded_opaque_state is not None else None
+
+            return LlmDisaggregatedParams(
+                request_type=self.disaggregated_params.request_type,
+                first_gen_tokens=self.disaggregated_params.first_gen_tokens,
+                ctx_request_id=self.disaggregated_params.ctx_request_id,
+                opaque_state=opaque_state,
+                draft_tokens=self.disaggregated_params.draft_tokens)
path: tensorrt_llm/serve/openai_protocol.py
user: login: Shunkangz
body: I extract them as helper functions. Do you think that I can just place them in the same file?
created_at: 2025-03-25T15:13:38Z
author_association: COLLABORATOR
diff_hunk: @@ -557,6 +580,21 @@ def to_sampling_params(self) -> SamplingParams:
         )
         return sampling_params
 
+    def to_llm_disaggregated_params(self) -> LlmDisaggregatedParams:
+        if self.disaggregated_params is None:
+            return None
+        else:
+            opaque_state = base64.b64decode(
+                self.disaggregated_params.encoded_opaque_state
+            ) if self.disaggregated_params.encoded_opaque_state is not None else None
+
+            return LlmDisaggregatedParams(
+                request_type=self.disaggregated_params.request_type,
+                first_gen_tokens=self.disaggregated_params.first_gen_tokens,
+                ctx_request_id=self.disaggregated_params.ctx_request_id,
+                opaque_state=opaque_state,
+                draft_tokens=self.disaggregated_params.draft_tokens)
path: tensorrt_llm/serve/openai_protocol.py
user: login: pcastonguay
body: Yes I think that's fine. Thanks.
created_at: 2025-03-25T18:43:24Z
author_association: COLLABORATOR
diff_hunk: @@ -149,7 +162,58 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.info("Sending request to gen server: %s", gen_server)
 
             if not gen_req.stream:
-                gen_response = await self.send_request(gen_server, gen_req)
+                gen_response = await self.send_completion_request(gen_server, gen_req)
+                return gen_response
+            else:
+                # Return a streaming response that combines both context and generation responses
+                return StreamingResponse(
+                    self.merge_streaming_responses(ctx_response, gen_server, gen_req),
+                    media_type="text/event-stream"
+                )
+
+        except CppExecutorError as e:
+            # If internal executor error is raised, shutdown the server
+            logging.exception(e)
+            signal.raise_signal(signal.SIGINT)
+        except HTTPException as e:
+            raise e  # Re-raise HTTP exceptions properly
+        except Exception as e:
+            logging.exception(e)
+            raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
+
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: pcastonguay
body: Looks like most of the code is copied from `openai_completion` except for call to `send_chat_request` instead of `send_completion_request`. Can we refactor to avoid duplicating so much code?
created_at: 2025-04-08T12:44:33Z
author_association: COLLABORATOR
diff_hunk: @@ -149,7 +162,58 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.info("Sending request to gen server: %s", gen_server)
 
             if not gen_req.stream:
-                gen_response = await self.send_request(gen_server, gen_req)
+                gen_response = await self.send_completion_request(gen_server, gen_req)
+                return gen_response
+            else:
+                # Return a streaming response that combines both context and generation responses
+                return StreamingResponse(
+                    self.merge_streaming_responses(ctx_response, gen_server, gen_req),
+                    media_type="text/event-stream"
+                )
+
+        except CppExecutorError as e:
+            # If internal executor error is raised, shutdown the server
+            logging.exception(e)
+            signal.raise_signal(signal.SIGINT)
+        except HTTPException as e:
+            raise e  # Re-raise HTTP exceptions properly
+        except Exception as e:
+            logging.exception(e)
+            raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
+
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: Shunkangz
body: I refactor the code of proxy server. Could you please review it again? Thank you!
created_at: 2025-04-09T14:57:55Z
author_association: COLLABORATOR
diff_hunk: @@ -149,7 +162,58 @@ async def openai_completion(self, req: CompletionRequest) -> Response:
             logging.info("Sending request to gen server: %s", gen_server)
 
             if not gen_req.stream:
-                gen_response = await self.send_request(gen_server, gen_req)
+                gen_response = await self.send_completion_request(gen_server, gen_req)
+                return gen_response
+            else:
+                # Return a streaming response that combines both context and generation responses
+                return StreamingResponse(
+                    self.merge_streaming_responses(ctx_response, gen_server, gen_req),
+                    media_type="text/event-stream"
+                )
+
+        except CppExecutorError as e:
+            # If internal executor error is raised, shutdown the server
+            logging.exception(e)
+            signal.raise_signal(signal.SIGINT)
+        except HTTPException as e:
+            raise e  # Re-raise HTTP exceptions properly
+        except Exception as e:
+            logging.exception(e)
+            raise HTTPException(status_code=500, detail=f"Internal server error {str(e)}")
+
+    async def openai_chat_completion(self, req: ChatCompletionRequest) -> Response:
path: tensorrt_llm/serve/openai_disagg_server.py
user: login: pcastonguay
body: Thanks, look much better now.
created_at: 2025-04-09T23:22:43Z
author_association: COLLABORATOR

PR comments:
user: login: Shunkangz
created_at: 2025-03-23T14:59:32Z
body: /bot run
author_association: COLLABORATOR
user: login: juney-nvidia
created_at: 2025-03-23T22:48:42Z
body: @chuangz0 @xiaoweiw-nv @pcastonguay pls help review this MR.

Thanks
June
author_association: COLLABORATOR
user: login: Shixiaowei02
created_at: 2025-03-24T08:21:48Z
body: /bot run
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-24T08:27:11Z
body: [PR_Github #269](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/269/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-24T12:30:08Z
body: [PR_Github #269](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/269/) [ run ] completed with state `FAILURE`
[/LLM/main/L0_MergeRequest_PR pipeline #259](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/259/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-03-26T06:58:45Z
body: > There are no tests for the chat endpoint. Could we add one please? Thanks.

Sure. I think that the modification of chat endpoint should be orthogonal to the parallel strategy of context and generation workers. Do we need to double all existing tests for chat or just add single test should be enough? @pcastonguay 
author_association: COLLABORATOR
user: login: pcastonguay
created_at: 2025-03-26T13:55:31Z
body: > > There are no tests for the chat endpoint. Could we add one please? Thanks.
> 
> Sure. I think that the modification of chat endpoint should be orthogonal to the parallel strategy of context and generation workers. Do we need to double all existing tests for chat or just add single test should be enough? @pcastonguay

Single test should be enough.
author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-08T09:31:39Z
body: /bot run --add-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-08T09:36:59Z
body: [PR_Github #1438](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1438/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-08T11:30:57Z
body: [PR_Github #1438](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1438/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1079](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1079/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-09T14:57:22Z
body: /bot run --add-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-09T15:02:37Z
body: [PR_Github #1629](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1629/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-10T01:12:07Z
body: /bot run --add-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:17:38Z
body: [PR_Github #1668](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1668/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:18:45Z
body: [PR_Github #1629](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1629/) [ run ] completed with state `ABORTED`
[/LLM/main/L0_MergeRequest_PR pipeline #1219](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1219/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-10T01:19:48Z
body: /bot run --add-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:24:55Z
body: [PR_Github #1669](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1669/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:26:11Z
body: [PR_Github #1668](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1668/) [ run ] completed with state `ABORTED`

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-10T01:40:46Z
body: /bot run --add-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:46:26Z
body: [PR_Github #1671](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1671/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T01:47:54Z
body: [PR_Github #1669](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1669/) [ run ] completed with state `ABORTED`

author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T05:57:48Z
body: [PR_Github #1671](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1671/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1249](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1249/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-10T07:16:29Z
body: /bot run --only-multi-gpu-test
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T07:22:05Z
body: [PR_Github #1726](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1726/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T13:54:59Z
body: [PR_Github #1726](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1726/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1288 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1288/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: byshiue
created_at: 2025-04-10T14:05:39Z
body: /bot run --stage-list "DGX_H100-4_GPUs-PyTorch-1, DGX_H100-4_GPUs-PyTorch-2, DGX_H100-4_GPUs-TensorRT-1, DGX_H100-4_GPUs-TensorRT-2"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-10T14:10:57Z
body: [PR_Github #1785](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1785/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T02:00:41Z
body: [PR_Github #1785](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1785/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1323 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1323/) completed with status: 'FAILURE'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-11T02:06:19Z
body: /bot run --stage-list "DGX_H100-4_GPUs-PyTorch-1, DGX_H100-4_GPUs-PyTorch-2"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T02:11:34Z
body: [PR_Github #1838](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1838/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-11T05:20:49Z
body: /bot run --stage-list "DGX_H100-4_GPUs-PyTorch-1, DGX_H100-4_GPUs-PyTorch-2"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T05:26:04Z
body: [PR_Github #1866](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1866/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T05:26:57Z
body: [PR_Github #1838](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1838/) [ run ] completed with state `ABORTED`

author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T07:48:31Z
body: [PR_Github #1866](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1866/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #1383 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/1383/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-11T08:19:01Z
body: /bot reuse-pipeline
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T08:24:58Z
body: [PR_Github #1897](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1897/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T08:30:18Z
body: [PR_Github #1897](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1897/) [ reuse-pipeline ] completed with state `SUCCESS`
Reusing [PR_Github #1866 `(Partly Tested)`](https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/helpers/job/PR_Github/1866/) for commit `f466d3f`

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-11T08:34:44Z
body: /bot reuse-pipeline
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T08:41:03Z
body: [PR_Github #1902](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1902/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T08:46:18Z
body: [PR_Github #1902](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1902/) [ reuse-pipeline ] completed with state `SUCCESS`
Reusing [PR_Github #1866 `(Partly Tested)`](https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/helpers/job/PR_Github/1866/) for commit `d506af6`

author_association: COLLABORATOR
user: login: Shunkangz
created_at: 2025-04-11T09:32:19Z
body: /bot reuse-pipeline
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T09:38:03Z
body: [PR_Github #1912](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1912/) [ reuse-pipeline ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T09:39:06Z
body: [PR_Github #1912](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1912/) [ reuse-pipeline ] completed with state `FAILURE`
Can't reuse [PR_Github #0]() with status: `UNKNOWN`

author_association: COLLABORATOR
user: login: byshiue
created_at: 2025-04-11T09:41:50Z
body: /bot skip --comment "the ci has passed while the reuse-pipeline fail"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T09:48:10Z
body: [PR_Github #1915](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1915/) [ skip ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-11T09:53:26Z
body: [PR_Github #1915](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1915/) [ skip ] completed with state `SUCCESS`
Skipping testing for commit `7d219e5`

author_association: COLLABORATOR