Repository: huggingface/transformers
Pull request: Correctly return finish reason length when finished

This PR updates `transfsormers serve` so that it accurately returns the stop reason -> instead of only returning "stop" (stop token), it now accurately returns "length" as well, therefore indicating when the max new token limit has been reached.

Transformers chat is updated to check on this reason, and to offer to continue:

```
Generation stopped after reaching the token limit.

Continue generating? (y/N):
```

This results in chats like the following:
<img width="1256" height="626" alt="image" src="https://github.com/user-attachments/assets/863a0828-6f2d-43b0-a6d6-24cf47d6d070" />

Implemented for both continuous batching and generate.

Changed files:
filename: src/transformers/cli/chat.py
patch: @@ -19,7 +19,7 @@
 import string
 import time
 from collections.abc import AsyncIterator
-from typing import Annotated
+from typing import Annotated, Any, Optional
 
 import click
 import typer
@@ -103,12 +103,14 @@ def __init__(self, model_id: str, user_id: str):
         self.model_id = model_id
         self.user_id = user_id
 
-    async def stream_output(self, stream: AsyncIterator[ChatCompletionStreamOutput]) -> tuple[str, int]:
+    async def stream_output(self, stream: AsyncIterator[ChatCompletionStreamOutput]) -> tuple[str, str | Any | None]:
         self._console.print(f"[bold blue]<{self.model_id}>:")
         with Live(console=self._console, refresh_per_second=4) as live:
             text = ""
+            finish_reason: str | None = None
             async for token in await stream:
                 outputs = token.choices[0].delta.content
+                finish_reason = getattr(token.choices[0], "finish_reason", finish_reason)
 
                 if not outputs:
                     continue
@@ -147,7 +149,7 @@ async def stream_output(self, stream: AsyncIterator[ChatCompletionStreamOutput])
 
         self._console.print()
 
-        return text
+        return text, finish_reason
 
     def input(self) -> str:
         """Gets user input from the console."""
@@ -169,6 +171,18 @@ def print_color(self, text: str, color: str):
         self._console.print(f"[bold {color}]{text}")
         self._console.print()
 
+    def confirm(self, message: str, default: bool = False) -> bool:
+        """Displays a yes/no prompt to the user, returning True for confirmation."""
+        default_hint = "Y/n" if default else "y/N"
+        response = self._console.input(f"[bold yellow]{message} ({default_hint}): ")
+        self._console.print()
+
+        response = response.strip().lower()
+        if not response:
+            return default
+
+        return response in {"y", "yes"}
+
     def print_help(self, minimal: bool = False):
         """Prints the help message to the console."""
         self._console.print(Markdown(HELP_STRING_MINIMAL if minimal else HELP_STRING))
@@ -362,9 +376,15 @@ async def _inner_run(self):
         config = self.config
 
         async with AsyncInferenceClient(base_url=self.base_url) as client:
+            pending_user_input: Optional[str] = None
             while True:
                 try:
-                    user_input = interface.input()
+                    if pending_user_input is not None:
+                        user_input = pending_user_input
+                        pending_user_input = None
+                        interface.print_user_message(user_input)
+                    else:
+                        user_input = interface.input()
 
                     # User commands
                     if user_input == "!exit":
@@ -448,9 +468,15 @@ async def _inner_run(self):
                         },
                     )
 
-                    model_output = await interface.stream_output(stream)
+                    model_output, finish_reason = await interface.stream_output(stream)
 
                     chat.append({"role": "assistant", "content": model_output})
+
+                    if finish_reason == "length":
+                        interface.print_color("Generation stopped after reaching the token limit.", "yellow")
+                        if interface.confirm("Continue generating?"):
+                            pending_user_input = "Please continue. Do not repeat text.”"
+                            continue
                 except KeyboardInterrupt:
                     break
 

filename: src/transformers/cli/serve.py
patch: @@ -835,11 +835,18 @@ def stream_chat_completion(request_id, decode_stream):
                 # they come from the assistant.
                 yield self.build_chat_completion_chunk(request_id, role="assistant", model=model_id_and_revision)
 
+                n_tokens_generated = 0
                 for result in self.running_continuous_batching_manager.request_id_iter(request_id):
+                    n_tokens_generated += 1
+
                     if result.status == RequestStatus.FINISHED:
+                        generated_all_tokens = n_tokens_generated >= generation_config.max_new_tokens
+                        final_token_is_eos = result == tokenizer.eos_token
+                        reason = "length" if (generated_all_tokens and not final_token_is_eos) else "stop"
+
                         yield self.build_chat_completion_chunk(
                             request_id,
-                            finish_reason="stop",
+                            finish_reason=reason,
                             model=model_id_and_revision,
                         )
                         break
@@ -1080,8 +1087,13 @@ def generate_with_cache(**kwargs):
                 # they come from the assistant.
                 yield self.build_chat_completion_chunk(request_id, role="assistant", model=model_id_and_revision)
 
+                result = ""
+                n_tokens_generated = 0
+
                 for result in streamer:
-                    # Temporary hack for GPTOS 3: don't emit the final "
"
+                    n_tokens_generated += 1
+
+                    # Temporary hack for GPT-OSS 3: don't emit the final "
"
                     if "gptoss" in model.config.architectures[0].lower():
                         result = result.removesuffix("
")
                     results += result
@@ -1170,7 +1182,11 @@ def generate_with_cache(**kwargs):
                         yield self.build_chat_completion_chunk(
                             _request_id, content=result, model=model_id_and_revision
                         )
-                yield self.build_chat_completion_chunk(_request_id, finish_reason="stop", model=model_id_and_revision)
+
+                generated_all_tokens = n_tokens_generated >= generation_config.max_new_tokens
+                final_token_is_eos = result == streamer.tokenizer.eos_token
+                reason = "length" if (generated_all_tokens and not final_token_is_eos) else "stop"
+                yield self.build_chat_completion_chunk(_request_id, finish_reason=reason, model=model_id_and_revision)
 
                 thread.join()
             except Exception as e:

filename: tests/cli/test_serve.py
patch: @@ -291,10 +291,9 @@ def test_requests(self, test_name: str, request_flags: dict):
         # NOTE: the output of our server is wrapped by `InferenceClient`, which sends fields even when they
         # are empty.
 
-        # Finish reason: the last payload should have a finish reason of "stop", all others should be empty
-        # TODO: we may add other finish reasons in the future, and this may need more logic
+        # Finish reason: the last payload should have a finish reason of "length" or "stop", all others should be empty
         finish_reasons = [payload.choices[0].finish_reason for payload in all_payloads]
-        self.assertEqual(finish_reasons[-1], "stop")
+        self.assertTrue(finish_reasons[-1] in ["length", "stop"])
         self.assertTrue(all(reason is None for reason in finish_reasons[:-1]))
 
         # Role: the first payload should have a role of "assistant", all others should be empty
@@ -328,6 +327,18 @@ def test_generation_config_in_request(self):
         # sets `do_sample=True`
         self.assertEqual(output_text, '<think>\nOkay, the user just asked, "')
 
+    def test_early_return_due_to_length(self):
+        request = {
+            "model": "Qwen/Qwen3-0.6B",
+            "messages": [{"role": "user", "content": "Hello, how are you?"}],
+            "stream": True,
+            "max_tokens": 3,
+        }
+
+        all_payloads = self.run_server(request)
+        last_payload = all_payloads[-1]
+        self.assertTrue(last_payload.choices[0]["finish_reason"] == "length")
+
     # TODO: one test for each request flag, to confirm it is working as expected
     # TODO: speed-based test to confirm that KV cache is working across requests
 
@@ -549,7 +560,7 @@ def test_tool_call(self):
         # Finally, the last payload should contain a finish reason
         finish_reasons = [payload.choices[0].finish_reason for payload in all_payloads]
         # TODO: I think the finish reason for a tool call is different? double check this
-        self.assertEqual(finish_reasons[-1], "stop")
+        self.assertTrue(finish_reasons[-1] in ["stop", "length"])
         self.assertTrue(all(reason is None for reason in finish_reasons[:-1]))