Repository: huggingface/transformers
Pull request: Better typehints for `apply_chat_template`

# What does this PR do?
Make better typehints for `apply_chat_template`. Now IDEs and type checkers will correctly infer types for this code.

```py
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")

# Returns raw string when tokenize=False
raw_prompt: str = tokenizer.apply_chat_template(
    [{"role": "user", "content": "Hello!"}],
    tokenize=False,
)

# Returns token ids when tokenize=True (default) and return_dict=False
token_ids: list[int] = tokenizer.apply_chat_template(
    [{"role": "user", "content": "Hello!"}],
)

# Returns BatchEncoding when return_dict=True
batch_encoding: BatchEncoding = tokenizer.apply_chat_template(
    [{"role": "user", "content": "Hello!"}],
    return_dict=True,
)
```

Similarly, can be done for [`ProcessorMixin.apply_chat_template`](https://github.com/huggingface/transformers/blob/e11a00a16f925b7d3b52f5007bdce3464edb361f/src/transformers/processing_utils.py#L1559), but I'm not sure how to add to it, because it's using kwargs instead of arguments   


## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [X] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [ ] Did you write any new necessary tests?


## Who can review?

@ArthurZucker and @itazap

Changed files:
filename: src/transformers/models/auto/tokenization_auto.py
patch: @@ -940,7 +940,9 @@ def __init__(self):
 
     @classmethod
     @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
-    def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
+    def from_pretrained(
+        cls, pretrained_model_name_or_path, *inputs, **kwargs
+    ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
         r"""
         Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.
 

filename: src/transformers/tokenization_utils_base.py
patch: @@ -28,7 +28,7 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Literal, NamedTuple, Optional, Union, overload
 
 import numpy as np
 from packaging import version
@@ -1507,6 +1507,69 @@ def get_vocab(self) -> dict[str, int]:
         """
         raise NotImplementedError()
 
+    # Case: tokenize=False → returns rendered string
+    @overload
+    def apply_chat_template(
+        self,
+        conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],
+        tools: Optional[list[Union[dict, Callable]]] = None,
+        documents: Optional[list[dict[str, str]]] = None,
+        chat_template: Optional[str] = None,
+        add_generation_prompt: bool = False,
+        continue_final_message: bool = False,
+        tokenize: Literal[False] = False,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: bool = False,
+        max_length: Optional[int] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_dict: bool = False,
+        return_assistant_tokens_mask: bool = False,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
+        **kwargs,
+    ) -> str: ...
+
+    # Case: tokenize=True, return_dict=False, return_tensors=None → returns ids
+    @overload
+    def apply_chat_template(
+        self,
+        conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],
+        tools: Optional[list[Union[dict, Callable]]] = None,
+        documents: Optional[list[dict[str, str]]] = None,
+        chat_template: Optional[str] = None,
+        add_generation_prompt: bool = False,
+        continue_final_message: bool = False,
+        tokenize: Literal[True] = True,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: bool = False,
+        max_length: Optional[int] = None,
+        return_tensors: Literal[None] = None,
+        return_dict: Literal[False] = False,
+        return_assistant_tokens_mask: bool = False,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
+        **kwargs,
+    ) -> Union[list[int], list[list[int]]]: ...
+
+    # Case: tokenize=True, return_dict=True → returns BatchEncoding
+    @overload
+    def apply_chat_template(
+        self,
+        conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],
+        tools: Optional[list[Union[dict, Callable]]] = None,
+        documents: Optional[list[dict[str, str]]] = None,
+        chat_template: Optional[str] = None,
+        add_generation_prompt: bool = False,
+        continue_final_message: bool = False,
+        tokenize: Literal[True] = True,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: bool = False,
+        max_length: Optional[int] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_dict: Literal[True] = True,
+        return_assistant_tokens_mask: bool = False,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
+        **kwargs,
+    ) -> BatchEncoding: ...
+
     def apply_chat_template(
         self,
         conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],