Repository: huggingface/transformers
Pull request: [pipelines] add support to `skip_special_tokens` in the main text generation pipelines

# What does this PR do?

Useful for thinking models, where we sometimes want to retrieve all generated tokens (including special tokens).

Test script:
```py
from transformers import pipeline

model_id = "Qwen/Qwen3-0.6B"
chat = [{"role": "user", "content": "What's your name?\n\n"}]
generator = pipeline("text-generation", model=model_id)

# normal pipeline use
output = generator(chat, max_new_tokens=1000, do_sample=False)
print(output)
# print ends in "How can I assist you today?'}]}]"

# forcing special tokens to be included in the output
output = generator(chat, max_new_tokens=1000, do_sample=False, skip_special_tokens=False)
print(output)
# print ends in "How can I assist you today?
'}]}]"
```

Changed files:
filename: src/transformers/generation/utils.py
patch: @@ -1754,6 +1754,9 @@ def _prepare_generation_config(
                 for key, model_gen_config_value in model_generation_config.__dict__.items():
                     if key.startswith("_") or key == "transformers_version":  # metadata
                         continue
+                    # Don't set `cache_implementation = 'hybrid'` from the model defaults, see #40135
+                    if key == "cache_implementation" and model_generation_config.cache_implementation == "hybrid":
+                        continue
                     global_default_value = getattr(global_default_generation_config, key, None)
                     custom_gen_config_value = getattr(generation_config, key, None)
                     if (

filename: src/transformers/pipelines/image_text_to_text.py
patch: @@ -203,22 +203,33 @@ def _sanitize_parameters(
         clean_up_tokenization_spaces=None,
         stop_sequence=None,
         continue_final_message=None,
+        skip_special_tokens=None,
         **kwargs: Unpack[ProcessingKwargs],
     ):
         forward_kwargs = {}
         preprocess_params = {}
         postprocess_params = {}
-        preprocess_params.update(kwargs)
 
+        # Preprocess params
+        preprocess_params.update(kwargs)
         if timeout is not None:
             preprocess_params["timeout"] = timeout
-
         if continue_final_message is not None:
             preprocess_params["continue_final_message"] = continue_final_message
 
+        # Forward kwargs
+        if generate_kwargs is not None:
+            forward_kwargs["generate_kwargs"] = generate_kwargs
+        if stop_sequence is not None:
+            stop_sequence_ids = self.processor.tokenizer.encode(stop_sequence, add_special_tokens=False)
+            if len(stop_sequence_ids) > 1:
+                logger.warning_once(
+                    "Stopping on a multiple token sequence is not yet supported on transformers. The first token of"
+                    " the stop sequence will be used as the stop sequence string in the interim."
+                )
+            generate_kwargs["eos_token_id"] = stop_sequence_ids[0]
         if generate_kwargs is not None:
             forward_kwargs["generate_kwargs"] = generate_kwargs
-
         if max_new_tokens is not None:
             if "generate_kwargs" not in forward_kwargs:
                 forward_kwargs["generate_kwargs"] = {}
@@ -229,6 +240,7 @@ def _sanitize_parameters(
                 )
             forward_kwargs["generate_kwargs"]["max_new_tokens"] = max_new_tokens
 
+        # Postprocess params
         if return_full_text is not None and return_type is None:
             if return_tensors is not None:
                 raise ValueError("`return_full_text` is mutually exclusive with `return_tensors`")
@@ -241,14 +253,9 @@ def _sanitize_parameters(
             postprocess_params["continue_final_message"] = continue_final_message
         if clean_up_tokenization_spaces is not None:
             postprocess_params["clean_up_tokenization_spaces"] = clean_up_tokenization_spaces
-        if stop_sequence is not None:
-            stop_sequence_ids = self.processor.tokenizer.encode(stop_sequence, add_special_tokens=False)
-            if len(stop_sequence_ids) > 1:
-                logger.warning_once(
-                    "Stopping on a multiple token sequence is not yet supported on transformers. The first token of"
-                    " the stop sequence will be used as the stop sequence string in the interim."
-                )
-            generate_kwargs["eos_token_id"] = stop_sequence_ids[0]
+        if skip_special_tokens is not None:
+            postprocess_params["skip_special_tokens"] = skip_special_tokens
+
         return preprocess_params, forward_kwargs, postprocess_params
 
     @overload
@@ -432,7 +439,12 @@ def _forward(self, model_inputs, generate_kwargs=None):
         return {"generated_sequence": generated_sequence, "prompt_text": prompt_text, "input_ids": input_ids}
 
     def postprocess(
-        self, model_outputs, return_type=ReturnType.FULL_TEXT, continue_final_message=None, **postprocess_kwargs
+        self,
+        model_outputs,
+        return_type=ReturnType.FULL_TEXT,
+        continue_final_message=None,
+        skip_special_tokens=None,
+        **postprocess_kwargs,
     ):
         input_texts = model_outputs["prompt_text"]
         input_texts = [input_texts] if isinstance(input_texts, (str, Chat)) else input_texts
@@ -445,8 +457,13 @@ def postprocess(
             ]
 
         # Decode inputs and outputs the same way to remove input text from generated text if present
-        generated_texts = self.processor.post_process_image_text_to_text(generated_sequence, **postprocess_kwargs)
-        decoded_inputs = self.processor.post_process_image_text_to_text(input_ids, **postprocess_kwargs)
+        skip_special_tokens = skip_special_tokens if skip_special_tokens is not None else True
+        generated_texts = self.processor.post_process_image_text_to_text(
+            generated_sequence, skip_special_tokens=skip_special_tokens, **postprocess_kwargs
+        )
+        decoded_inputs = self.processor.post_process_image_text_to_text(
+            input_ids, skip_special_tokens=skip_special_tokens, **postprocess_kwargs
+        )
 
         # Force consistent behavior for including the input text in the output
         if return_type in {ReturnType.NEW_TEXT, ReturnType.FULL_TEXT}:

filename: src/transformers/pipelines/text_generation.py
patch: @@ -157,10 +157,11 @@ def _sanitize_parameters(
         truncation=None,
         max_length=None,
         continue_final_message=None,
+        skip_special_tokens=None,
         **generate_kwargs,
     ):
+        # preprocess kwargs
         preprocess_params = {}
-
         add_special_tokens = False
         if "add_special_tokens" in generate_kwargs:
             add_special_tokens = preprocess_params["add_special_tokens"] = generate_kwargs.pop("add_special_tokens")
@@ -193,10 +194,20 @@ def _sanitize_parameters(
 
         if continue_final_message is not None:
             preprocess_params["continue_final_message"] = continue_final_message
-
         preprocess_params.update(generate_kwargs)
+
+        # forward kwargs
+        if stop_sequence is not None:
+            stop_sequence_ids = self.tokenizer.encode(stop_sequence, add_special_tokens=False)
+            generate_kwargs["eos_token_id"] = stop_sequence_ids
         forward_params = generate_kwargs
+        if self.assistant_model is not None:
+            forward_params["assistant_model"] = self.assistant_model
+        if self.assistant_tokenizer is not None:
+            forward_params["tokenizer"] = self.tokenizer
+            forward_params["assistant_tokenizer"] = self.assistant_tokenizer
 
+        # postprocess kwargs
         postprocess_params = {}
         if return_full_text is not None and return_type is None:
             if return_text is not None:
@@ -214,16 +225,8 @@ def _sanitize_parameters(
             postprocess_params["clean_up_tokenization_spaces"] = clean_up_tokenization_spaces
         if continue_final_message is not None:
             postprocess_params["continue_final_message"] = continue_final_message
-
-        if stop_sequence is not None:
-            stop_sequence_ids = self.tokenizer.encode(stop_sequence, add_special_tokens=False)
-            generate_kwargs["eos_token_id"] = stop_sequence_ids
-
-        if self.assistant_model is not None:
-            forward_params["assistant_model"] = self.assistant_model
-        if self.assistant_tokenizer is not None:
-            forward_params["tokenizer"] = self.tokenizer
-            forward_params["assistant_tokenizer"] = self.assistant_tokenizer
+        if skip_special_tokens is not None:
+            postprocess_params["skip_special_tokens"] = skip_special_tokens
 
         return preprocess_params, forward_params, postprocess_params
 
@@ -462,6 +465,7 @@ def postprocess(
         return_type=ReturnType.FULL_TEXT,
         clean_up_tokenization_spaces=True,
         continue_final_message=None,
+        skip_special_tokens=None,
     ):
         generated_sequence = model_outputs["generated_sequence"][0]
         input_ids = model_outputs["input_ids"]
@@ -480,14 +484,15 @@ def postprocess(
                     if isinstance(v, tf.Tensor) and v.shape[0] == len(generated_sequence):
                         splitted_keys[k] = v.numpy().tolist()
 
+        skip_special_tokens = skip_special_tokens if skip_special_tokens is not None else True
         for idx, sequence in enumerate(generated_sequence):
             if return_type == ReturnType.TENSORS:
                 record = {"generated_token_ids": sequence}
             elif return_type in {ReturnType.NEW_TEXT, ReturnType.FULL_TEXT}:
                 # Decode text
                 text = self.tokenizer.decode(
                     sequence,
-                    skip_special_tokens=True,
+                    skip_special_tokens=skip_special_tokens,
                     clean_up_tokenization_spaces=clean_up_tokenization_spaces,
                 )
 
@@ -498,7 +503,7 @@ def postprocess(
                     prompt_length = len(
                         self.tokenizer.decode(
                             input_ids[0],
-                            skip_special_tokens=True,
+                            skip_special_tokens=skip_special_tokens,
                             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
                         )
                     )

filename: tests/pipelines/test_pipelines_text_generation.py
patch: @@ -540,3 +540,18 @@ def test_pipeline_assisted_generation(self):
         # It is running assisted generation under the hood (e.g. flags incompatible with assisted gen will crash)
         with self.assertRaises(ValueError):
             _ = pipe(prompt, generate_kwargs={"num_beams": 2})
+
+    @require_torch
+    def test_pipeline_skip_special_tokens(self):
+        """Tests that we can use `skip_special_tokens=False` to get the special tokens in the output"""
+        model_id = "google/gemma-3-270m-it"
+        chat = [{"role": "user", "content": "What's your name?"}]
+        generator = pipeline("text-generation", model=model_id)
+
+        # normal pipeline use
+        output = generator(chat, max_new_tokens=20, do_sample=False)
+        self.assertNotIn("<end_of_turn>", str(output[0]["generated_text"]))
+
+        # forcing special tokens to be included in the output
+        output = generator(chat, max_new_tokens=1000, do_sample=False, skip_special_tokens=False)
+        self.assertIn("<end_of_turn>", str(output[0]["generated_text"]))